Validate API: support for verbose explanation of succesfully validated queries

This commit adds a `rewrite` parameter to the validate API in order to shown
how the given query is re-written into primitive queries. For example, an MLT
query is re-written into a disjunction of the selected terms. Other use cases
include `fuzzy`, `common_terms`, or `match` query especially with a
`cutoff_frequency` parameter. Note that the explanation is only given for a
single randomly chosen shard only, so the output may vary from one shard to
another.

Relates #1412
Closes #10147
This commit is contained in:
Alex Ksikes 2015-03-18 12:28:20 -07:00
parent c13e604697
commit c347dfe91c
7 changed files with 221 additions and 29 deletions

View File

@ -75,3 +75,84 @@ curl -XGET 'http://localhost:9200/twitter/tweet/_validate/query?q=post_date:foo&
} ]
}
--------------------------------------------------
coming[1.6] When the query is valid, the explanation defaults to the string
representation of that query. With `rewrite` set to `true`, the explanation
is more detailed showing the actual Lucene query that will be executed.
For Fuzzy Queries:
[source,js]
--------------------------------------------------
curl -XGET 'http://localhost:9200/imdb/movies/_validate/query?rewrite=true'
{
"query": {
"fuzzy": {
"actors": "kyle"
}
}
}
--------------------------------------------------
Response:
[source,js]
--------------------------------------------------
{
"valid": true,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"explanations": [
{
"index": "imdb",
"valid": true,
"explanation": "filtered(plot:kyle plot:kylie^0.75 plot:kyne^0.75 plot:lyle^0.75 plot:pyle^0.75)->cache(_type:movies)"
}
]
}
--------------------------------------------------
For More Like This:
[source,js]
--------------------------------------------------
curl -XGET 'http://localhost:9200/imdb/movies/_validate/query?rewrite=true'
{
"query": {
"more_like_this": {
"like": {
"_id": "88247"
},
"boost_terms": 1
}
}
}
--------------------------------------------------
Response:
[source,js]
--------------------------------------------------
{
"valid": true,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"explanations": [
{
"index": "imdb",
"valid": true,
"explanation": "filtered(((title:terminator^3.71334 plot:future^2.763601 plot:human^2.8415773 plot:sarah^3.4193945 plot:kyle^3.8244398 plot:cyborg^3.9177752 plot:connor^4.040236 plot:reese^4.7133346 ... )~6) -ConstantScore(_uid:movies#88247))->cache(_type:movies)"
}
]
}
--------------------------------------------------
CAUTION: The request is executed on a single shard only, which is randomly
selected. The detailed explanation of the query may depend on which shard is
being hit, and therefore may vary from one request to another.

View File

@ -37,6 +37,7 @@ class ShardValidateQueryRequest extends BroadcastShardOperationRequest {
private BytesReference source;
private String[] types = Strings.EMPTY_ARRAY;
private boolean explain;
private boolean rewrite;
private long nowInMillis;
@Nullable
@ -51,6 +52,7 @@ class ShardValidateQueryRequest extends BroadcastShardOperationRequest {
this.source = request.source();
this.types = request.types();
this.explain = request.explain();
this.rewrite = request.rewrite();
this.filteringAliases = filteringAliases;
this.nowInMillis = request.nowInMillis;
}
@ -67,6 +69,10 @@ class ShardValidateQueryRequest extends BroadcastShardOperationRequest {
return this.explain;
}
public boolean rewrite() {
return this.rewrite;
}
public String[] filteringAliases() {
return filteringAliases;
}
@ -96,6 +102,7 @@ class ShardValidateQueryRequest extends BroadcastShardOperationRequest {
}
explain = in.readBoolean();
rewrite = in.readBoolean();
nowInMillis = in.readVLong();
}
@ -118,6 +125,7 @@ class ShardValidateQueryRequest extends BroadcastShardOperationRequest {
}
out.writeBoolean(explain);
out.writeBoolean(rewrite);
out.writeVLong(nowInMillis);
}
}

View File

@ -19,6 +19,8 @@
package org.elasticsearch.action.admin.indices.validate.query;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ShardOperationFailedException;
@ -34,11 +36,14 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.routing.GroupShardsIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.MatchNoDocsFilter;
import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.query.QueryParsingException;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.script.ScriptService;
@ -48,6 +53,7 @@ import org.elasticsearch.search.internal.ShardSearchLocalRequest;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -147,7 +153,7 @@ public class TransportValidateQueryAction extends TransportBroadcastOperationAct
} else {
ShardValidateQueryResponse validateQueryResponse = (ShardValidateQueryResponse) shardResponse;
valid = valid && validateQueryResponse.isValid();
if (request.explain()) {
if (request.explain() || request.rewrite()) {
if (queryExplanations == null) {
queryExplanations = newArrayList();
}
@ -173,10 +179,11 @@ public class TransportValidateQueryAction extends TransportBroadcastOperationAct
boolean valid;
String explanation = null;
String error = null;
Engine.Searcher searcher = indexShard.acquireSearcher("validate_query");
DefaultSearchContext searchContext = new DefaultSearchContext(0,
new ShardSearchLocalRequest(request.types(), request.nowInMillis(), request.filteringAliases()),
null, indexShard.acquireSearcher("validate_query"), indexService, indexShard,
null, searcher, indexService, indexShard,
scriptService, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter()
);
SearchContext.setCurrent(searchContext);
@ -190,12 +197,18 @@ public class TransportValidateQueryAction extends TransportBroadcastOperationAct
if (request.explain()) {
explanation = searchContext.query().toString();
}
if (request.rewrite()) {
explanation = getRewrittenQuery(searcher.searcher(), searchContext.query());
}
} catch (QueryParsingException e) {
valid = false;
error = e.getDetailedMessage();
} catch (AssertionError e) {
valid = false;
error = e.getMessage();
} catch (IOException e) {
valid = false;
error = e.getMessage();
} finally {
SearchContext.current().close();
SearchContext.removeCurrent();
@ -203,4 +216,13 @@ public class TransportValidateQueryAction extends TransportBroadcastOperationAct
return new ShardValidateQueryResponse(request.shardId(), valid, explanation, error);
}
private String getRewrittenQuery(IndexSearcher searcher, Query query) throws IOException {
Query queryRewrite = searcher.rewrite(query);
if (queryRewrite instanceof MatchNoDocsQuery || queryRewrite instanceof MatchNoDocsFilter) {
return query.toString();
} else {
return queryRewrite.toString();
}
}
}

View File

@ -49,6 +49,7 @@ public class ValidateQueryRequest extends BroadcastOperationRequest<ValidateQuer
private BytesReference source;
private boolean explain;
private boolean rewrite;
private String[] types = Strings.EMPTY_ARRAY;
@ -163,6 +164,20 @@ public class ValidateQueryRequest extends BroadcastOperationRequest<ValidateQuer
return explain;
}
/**
* Indicates whether the query should be rewritten into primitive queries
*/
public void rewrite(boolean rewrite) {
this.rewrite = rewrite;
}
/**
* Indicates whether the query should be rewritten into primitive queries
*/
public boolean rewrite() {
return rewrite;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
@ -178,7 +193,7 @@ public class ValidateQueryRequest extends BroadcastOperationRequest<ValidateQuer
}
explain = in.readBoolean();
rewrite = in.readBoolean();
}
@Override
@ -193,6 +208,7 @@ public class ValidateQueryRequest extends BroadcastOperationRequest<ValidateQuer
}
out.writeBoolean(explain);
out.writeBoolean(rewrite);
}
@Override
@ -203,6 +219,7 @@ public class ValidateQueryRequest extends BroadcastOperationRequest<ValidateQuer
} catch (Exception e) {
// ignore
}
return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", source[" + sSource + "], explain:" + explain;
return "[" + Arrays.toString(indices) + "]" + Arrays.toString(types) + ", source[" + sSource + "], explain:" + explain +
", rewrite:" + rewrite;
}
}

View File

@ -85,6 +85,14 @@ public class ValidateQueryRequestBuilder extends BroadcastOperationRequestBuilde
return this;
}
/**
* Indicates whether the query should be rewritten into primitive queries
*/
public ValidateQueryRequestBuilder setRewrite(boolean rewrite) {
request.rewrite(rewrite);
return this;
}
@Override
protected void doExecute(ActionListener<ValidateQueryResponse> listener) {
if (sourceBuilder != null) {

View File

@ -78,6 +78,11 @@ public class RestValidateQueryAction extends BaseRestHandler {
} else {
validateQueryRequest.explain(false);
}
if (request.paramAsBoolean("rewrite", false)) {
validateQueryRequest.rewrite(true);
} else {
validateQueryRequest.rewrite(false);
}
client.admin().indices().validateQuery(validateQueryRequest, new RestBuilderListener<ValidateQueryResponse>(channel) {
@Override

View File

@ -19,7 +19,6 @@
package org.elasticsearch.validate;
import com.google.common.base.Charsets;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
import org.elasticsearch.client.Client;
@ -28,6 +27,7 @@ import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.cache.filter.AutoFilterCachingPolicy;
import org.elasticsearch.index.cache.filter.FilterCacheModule;
@ -49,12 +49,11 @@ import org.junit.Test;
import java.io.IOException;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.*;
/**
*
@ -116,6 +115,10 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
return filter;
}
private String filtered(String query) {
return "filtered(" + query + ")";
}
@Test
public void explainValidateQuery() throws Exception {
createIndex("test");
@ -149,13 +152,13 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
assertThat(response.getQueryExplanation().get(0).getError(), containsString("Failed to parse"));
assertThat(response.getQueryExplanation().get(0).getExplanation(), nullValue());
final String typeFilter = filter("_type:type1");
assertExplanation(QueryBuilders.queryStringQuery("_id:1"), equalTo("filtered(ConstantScore(_uid:type1#1))->" + typeFilter));
final String typeFilter = "->" + filter("_type:type1");
assertExplanation(QueryBuilders.queryStringQuery("_id:1"), equalTo(filtered("ConstantScore(_uid:type1#1)") + typeFilter));
assertExplanation(QueryBuilders.idsQuery("type1").addIds("1").addIds("2"),
equalTo("filtered(ConstantScore(_uid:type1#1 _uid:type1#2))->" + typeFilter));
equalTo(filtered("ConstantScore(_uid:type1#1 _uid:type1#2)") + typeFilter));
assertExplanation(QueryBuilders.queryStringQuery("foo"), equalTo("filtered(_all:foo)->" + typeFilter));
assertExplanation(QueryBuilders.queryStringQuery("foo"), equalTo(filtered("_all:foo") + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
@ -163,14 +166,14 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
FilterBuilders.termFilter("bar", "2"),
FilterBuilders.termFilter("baz", "3")
)
), equalTo("filtered(filtered(foo:1)->" + filter(filter("bar:[2 TO 2]") + " " + filter("baz:3")) + ")->" + typeFilter));
), equalTo(filtered("filtered(foo:1)->" + filter(filter("bar:[2 TO 2]") + " " + filter("baz:3"))) + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
FilterBuilders.orFilter(
FilterBuilders.termFilter("bar", "2")
)
), equalTo("filtered(filtered(foo:1)->" + filter(filter("bar:[2 TO 2]")) + ")->" + typeFilter));
), equalTo(filtered("filtered(foo:1)->" + filter(filter("bar:[2 TO 2]"))) + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.matchAllQuery(),
@ -179,28 +182,28 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
.addPoint(30, -80)
.addPoint(20, -90)
.addPoint(40, -70) // closing polygon
), equalTo("filtered(ConstantScore(" + filter("GeoPolygonFilter(pin.location, [[40.0, -70.0], [30.0, -80.0], [20.0, -90.0], [40.0, -70.0]]))") + ")->" + typeFilter));
), equalTo(filtered("ConstantScore(" + filter("GeoPolygonFilter(pin.location, [[40.0, -70.0], [30.0, -80.0], [20.0, -90.0], [40.0, -70.0]]))")) + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoBoundingBoxFilter("pin.location")
.topLeft(40, -80)
.bottomRight(20, -70)
), equalTo("filtered(ConstantScore(" + filter("GeoBoundingBoxFilter(pin.location, [40.0, -80.0], [20.0, -70.0]))") + ")->" + typeFilter));
), equalTo(filtered("ConstantScore(" + filter("GeoBoundingBoxFilter(pin.location, [40.0, -80.0], [20.0, -70.0]))")) + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceFilter("pin.location")
.lat(10).lon(20).distance(15, DistanceUnit.DEFAULT).geoDistance(GeoDistance.PLANE)
), equalTo("filtered(ConstantScore(" + filter("GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0))") + ")->" + typeFilter));
), equalTo(filtered("ConstantScore(" + filter("GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0))")) + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceFilter("pin.location")
.lat(10).lon(20).distance(15, DistanceUnit.DEFAULT).geoDistance(GeoDistance.PLANE)
), equalTo("filtered(ConstantScore(" + filter("GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0))") + ")->" + typeFilter));
), equalTo(filtered("ConstantScore(" + filter("GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0))")) + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceRangeFilter("pin.location")
.lat(10).lon(20).from("15m").to("25m").geoDistance(GeoDistance.PLANE)
), equalTo("filtered(ConstantScore(" + filter("GeoDistanceRangeFilter(pin.location, PLANE, [15.0 - 25.0], 10.0, 20.0))") + ")->" + typeFilter));
), equalTo(filtered("ConstantScore(" + filter("GeoDistanceRangeFilter(pin.location, PLANE, [15.0 - 25.0], 10.0, 20.0))")) + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceRangeFilter("pin.location")
.lat(10).lon(20).from("15miles").to("25miles").geoDistance(GeoDistance.PLANE)
), equalTo("filtered(ConstantScore(" + filter("GeoDistanceRangeFilter(pin.location, PLANE, [" + DistanceUnit.DEFAULT.convert(15.0, DistanceUnit.MILES) + " - " + DistanceUnit.DEFAULT.convert(25.0, DistanceUnit.MILES) + "], 10.0, 20.0))") + ")->" + typeFilter));
), equalTo(filtered("ConstantScore(" + filter("GeoDistanceRangeFilter(pin.location, PLANE, [" + DistanceUnit.DEFAULT.convert(15.0, DistanceUnit.MILES) + " - " + DistanceUnit.DEFAULT.convert(25.0, DistanceUnit.MILES) + "], 10.0, 20.0))")) + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
@ -208,13 +211,13 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
FilterBuilders.termFilter("bar", "2"),
FilterBuilders.termFilter("baz", "3")
)
), equalTo("filtered(filtered(foo:1)->" + filter("+" + filter("bar:[2 TO 2]") + " +" + filter("baz:3")) + ")->" + typeFilter));
), equalTo(filtered("filtered(foo:1)->" + filter("+" + filter("bar:[2 TO 2]") + " +" + filter("baz:3"))) + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.termsFilter("foo", "1", "2", "3")),
equalTo("filtered(ConstantScore(" + filter("foo:1 foo:2 foo:3") + "))->" + typeFilter));
equalTo(filtered("ConstantScore(" + filter("foo:1 foo:2 foo:3") + ")") + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.notFilter(FilterBuilders.termFilter("foo", "bar"))),
equalTo("filtered(ConstantScore(" + filter("NotFilter(" + filter("foo:bar") + ")") + "))->" + typeFilter));
equalTo(filtered("ConstantScore(" + filter("NotFilter(" + filter("foo:bar") + ")") + ")") + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
@ -222,13 +225,12 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
"child-type",
QueryBuilders.matchQuery("foo", "1")
)
), equalTo("filtered(filtered(foo:1)->CustomQueryWrappingFilter(child_filter[child-type/type1](filtered(foo:1)->" + filter("_type:child-type") + ")))->" + typeFilter));
), equalTo(filtered("filtered(foo:1)->CustomQueryWrappingFilter(child_filter[child-type/type1](filtered(foo:1)->" + filter("_type:child-type") + "))") + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
FilterBuilders.scriptFilter("true")
), equalTo("filtered(filtered(foo:1)->" + filter("ScriptFilter(true)") + ")->" + typeFilter));
), equalTo(filtered("filtered(foo:1)->" + filter("ScriptFilter(true)")) + typeFilter));
}
@Test
@ -365,6 +367,50 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
assertThat(validateQueryResponse.getQueryExplanation().get(0).getExplanation(), containsString("field:\"foo (one* two*)\""));
}
@Test
public void explainWithRewriteValidateQuery() throws Exception {
client().admin().indices().prepareCreate("test")
.addMapping("type1", "field", "type=string,analyzer=whitespace")
.setSettings(SETTING_NUMBER_OF_SHARDS, 1).get();
client().prepareIndex("test", "type1", "1").setSource("field", "quick lazy huge brown pidgin").get();
client().prepareIndex("test", "type1", "2").setSource("field", "the quick brown fox").get();
client().prepareIndex("test", "type1", "3").setSource("field", "the quick lazy huge brown fox jumps over the tree").get();
client().prepareIndex("test", "type1", "4").setSource("field", "the lazy dog quacks like a duck").get();
refresh();
// prefix queries
assertExplanation(QueryBuilders.matchPhrasePrefixQuery("field", "qu"),
containsString("field:quick"), true);
assertExplanation(QueryBuilders.matchPhrasePrefixQuery("field", "ju"),
containsString("field:jumps"), true);
// common terms queries
assertExplanation(QueryBuilders.commonTermsQuery("field", "huge brown pidgin").cutoffFrequency(1),
containsString("(field:huge field:brown) +field:pidgin"), true);
assertExplanation(QueryBuilders.commonTermsQuery("field", "the brown").analyzer("stop"),
containsString("field:brown"), true);
// match queries with cutoff frequency
assertExplanation(QueryBuilders.matchQuery("field", "huge brown pidgin").cutoffFrequency(1),
containsString("(field:huge field:brown) +field:pidgin"), true);
assertExplanation(QueryBuilders.matchQuery("field", "the brown").analyzer("stop"),
containsString("field:brown"), true);
// fuzzy queries
assertExplanation(QueryBuilders.fuzzyQuery("field", "the").fuzziness(Fuzziness.fromEdits(2)),
containsString("field:the field:tree^0.3333333"), true);
assertExplanation(QueryBuilders.fuzzyQuery("field", "jump"),
containsString("field:jumps^0.75"), true);
// more like this queries
assertExplanation(QueryBuilders.moreLikeThisQuery("field").ids("1")
.include(true).minTermFreq(1).minDocFreq(1).maxQueryTerms(2),
containsString("field:huge field:pidgin"), true);
assertExplanation(QueryBuilders.moreLikeThisQuery("field").like("the huge pidgin")
.minTermFreq(1).minDocFreq(1).maxQueryTerms(2),
containsString("field:huge field:pidgin"), true);
}
@Test
public void irrelevantPropertiesBeforeQuery() throws IOException {
createIndex("test");
@ -384,10 +430,15 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
}
private void assertExplanation(QueryBuilder queryBuilder, Matcher<String> matcher) {
assertExplanation(queryBuilder, matcher, false);
}
private void assertExplanation(QueryBuilder queryBuilder, Matcher<String> matcher, boolean withRewrite) {
ValidateQueryResponse response = client().admin().indices().prepareValidateQuery("test")
.setTypes("type1")
.setQuery(queryBuilder)
.setExplain(true)
.setRewrite(withRewrite)
.execute().actionGet();
assertThat(response.getQueryExplanation().size(), equalTo(1));
assertThat(response.getQueryExplanation().get(0).getError(), nullValue());