From 124fecd22103f45aab7a4003c462d93fd5fa9a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 10 Apr 2018 10:15:31 +0200 Subject: [PATCH] [Docs] Add rankEval method for Jva HL client This change adds documentation about using the ranking evaluation API from the high level Java REST client. Closes #28694 --- .../documentation/SearchDocumentationIT.java | 84 +++++++++++++++++ .../high-level/search/rank-eval.asciidoc | 89 +++++++++++++++++++ .../high-level/supported-apis.asciidoc | 2 + 3 files changed, 175 insertions(+) create mode 100644 docs/java-rest/high-level/search/rank-eval.asciidoc diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index bd1cf48f141..a8dde67fa94 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.unit.TimeValue; @@ -44,6 +45,16 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.rankeval.EvalQueryQuality; +import org.elasticsearch.index.rankeval.EvaluationMetric; +import org.elasticsearch.index.rankeval.MetricDetail; +import org.elasticsearch.index.rankeval.PrecisionAtK; +import org.elasticsearch.index.rankeval.RankEvalRequest; +import org.elasticsearch.index.rankeval.RankEvalResponse; +import org.elasticsearch.index.rankeval.RankEvalSpec; +import org.elasticsearch.index.rankeval.RatedDocument; +import org.elasticsearch.index.rankeval.RatedRequest; +import org.elasticsearch.index.rankeval.RatedSearchHit; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; @@ -74,6 +85,7 @@ import org.elasticsearch.search.suggest.SuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestion; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -688,6 +700,78 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase { } } + public void testRankEval() throws Exception { + indexSearchTestData(); + RestHighLevelClient client = highLevelClient(); + { + // tag::rank-eval-request-basic + EvaluationMetric metric = new PrecisionAtK(); // <1> + List ratedDocs = new ArrayList<>(); + ratedDocs.add(new RatedDocument("posts", "1", 1)); // <2> + SearchSourceBuilder searchQuery = new SearchSourceBuilder(); + searchQuery.query(QueryBuilders.matchQuery("user", "kimchy"));// <3> + RatedRequest ratedRequest = // <4> + new RatedRequest("kimchy_query", ratedDocs, searchQuery); + List ratedRequests = Arrays.asList(ratedRequest); + RankEvalSpec specification = + new RankEvalSpec(ratedRequests, metric); // <5> + RankEvalRequest request = // <6> + new RankEvalRequest(specification, new String[] { "posts" }); + // end::rank-eval-request-basic + + // tag::rank-eval-execute + RankEvalResponse response = client.rankEval(request); + // end::rank-eval-execute + + logger.warn(Strings.toString(response)); + // tag::rank-eval-response + double evaluationResult = response.getEvaluationResult(); // <1> + assertEquals(1.0 / 3.0, evaluationResult, 0.0); + Map partialResults = + response.getPartialResults(); + EvalQueryQuality evalQuality = + partialResults.get("kimchy_query"); // <2> + assertEquals("kimchy_query", evalQuality.getId()); + double qualityLevel = evalQuality.getQualityLevel(); // <3> + assertEquals(1.0 / 3.0, qualityLevel, 0.0); + List hitsAndRatings = evalQuality.getHitsAndRatings(); + RatedSearchHit ratedSearchHit = hitsAndRatings.get(0); + assertEquals("3", ratedSearchHit.getSearchHit().getId()); // <4> + assertFalse(ratedSearchHit.getRating().isPresent()); // <5> + MetricDetail metricDetails = evalQuality.getMetricDetails(); + String metricName = metricDetails.getMetricName(); + assertEquals(PrecisionAtK.NAME, metricName); // <6> + PrecisionAtK.Detail detail = (PrecisionAtK.Detail) metricDetails; + assertEquals(1, detail.getRelevantRetrieved()); // <7> + assertEquals(3, detail.getRetrieved()); + // end::rank-eval-response + + // tag::rank-eval-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(RankEvalResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::rank-eval-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::rank-eval-execute-async + client.rankEvalAsync(request, listener); // <1> + // end::rank-eval-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + public void testMultiSearch() throws Exception { indexSearchTestData(); RestHighLevelClient client = highLevelClient(); diff --git a/docs/java-rest/high-level/search/rank-eval.asciidoc b/docs/java-rest/high-level/search/rank-eval.asciidoc new file mode 100644 index 00000000000..6db0dadd00e --- /dev/null +++ b/docs/java-rest/high-level/search/rank-eval.asciidoc @@ -0,0 +1,89 @@ +[[java-rest-high-rank-eval]] +=== Ranking Evaluation API + +The `rankEval` method allows to evaluate the quality of ranked search +results over a set of search request. Given sets of manually rated +documents for each search request, ranking evaluation performs a +<> request and calculates +information retrieval metrics like _mean reciprocal rank_, _precision_ +or _discounted cumulative gain_ on the returned results. + +[[java-rest-high-rank-eval-request]] +==== Ranking Evaluation Request + +In order to build a `RankEvalRequest`, you first need to create an +evaluation specification (`RankEvalSpec`). This specification requires +to define the evaluation metric that is going to be calculated, as well +as a list of rated documents per search requests. Creating the ranking +evaluation request then takes the specification and a list of target +indices as arguments: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-request-basic] +-------------------------------------------------- +<1> Define the metric used in the evaluation +<2> Add rated documents, specified by index name, id and rating +<3> Create the search query to evaluate +<4> Combine the three former parts into a `RatedRequest` +<5> Create the ranking evaluation specification +<6> Create the ranking evaluation request + +[[java-rest-high-rank-eval-sync]] +==== Synchronous Execution + +The `rankEval` method executes `RankEvalRequest`s synchronously: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-execute] +-------------------------------------------------- + +[[java-rest-high-rank-eval-async]] +==== Asynchronous Execution + +The `rankEvalAsync` method executes `RankEvalRequest`s asynchronously, +calling the provided `ActionListener` when the response is ready. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-execute-async] +-------------------------------------------------- +<1> The `RankEvalRequest` to execute and the `ActionListener` to use when +the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for `RankEvalResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. +<2> Called when the whole `RankEvalRequest` fails. + +==== RankEvalResponse + +The `RankEvalResponse` that is returned by executing the request +contains information about the overall evaluation score, the +scores of each individual search request in the set of queries and +detailed information about search hits and details about the metric +calculation per partial result. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-response] +-------------------------------------------------- +<1> The overall evaluation result +<2> Partial results that are keyed by their query id +<3> The metric score for each partial result +<4> Rated search hits contain a fully fledged `SearchHit` +<5> Rated search hits also contain an `Optional` rating that +is not present if the document did not get a rating in the request +<6> Metric details are named after the metric used in the request +<7> After casting to the metric used in the request, the +metric details offers insight into parts of the metric calculation \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 29052171cdd..1f3d7a37443 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -32,10 +32,12 @@ The Java High Level REST Client supports the following Search APIs: * <> * <> * <> +* <> include::search/search.asciidoc[] include::search/scroll.asciidoc[] include::search/multi-search.asciidoc[] +include::search/rank-eval.asciidoc[] == Miscellaneous APIs