Indices query/filter skip parsing altogether for irrelevant indices when possible

Closes #2416
This commit is contained in:
Olivier Favre 2013-11-06 15:52:29 +01:00 committed by Luca Cavanna
parent c8020760d7
commit fa80ca97b2
7 changed files with 364 additions and 35 deletions

View File

@ -0,0 +1,38 @@
[[query-dsl-indices-filter]]
=== Indices Filter
The `indices` filter can be used when executed across multiple indices,
allowing to have a filter that executes only when executed on an index
that matches a specific list of indices, and another filter that executes
when it is executed on an index that does not match the listed indices.
[source,js]
--------------------------------------------------
{
"indices" : {
"indices" : ["index1", "index2"],
"filter" : {
"term" : { "tag" : "wow" }
},
"no_match_filter" : {
"term" : { "tag" : "kow" }
}
}
}
--------------------------------------------------
You can use the `index` field to provide a single index.
`no_match_filter` can also have "string" value of `none` (to match no
documents), and `all` (to match all). Defaults to `all`.
`filter` is mandatory. You must provide the indices.
It is forbidden to omit or to give `indices` or `index` multiple times,
or to give both.
Please note that the fields order is important: If the indices are
provided before `filter` or `no_match_filter`, the filter parsing is
skipped altogether.
For instance, this feature is useful to prevent a query that runs
against multiple indices to fail because of a missing type.
See `has_child`, `has_parent`, `top_children` and `nested`.

View File

@ -21,5 +21,18 @@ when it is executed on an index that does not match the listed indices.
}
--------------------------------------------------
You can use the `index` field to provide a single index.
`no_match_query` can also have "string" value of `none` (to match no
documents), and `all` (to match all).
documents), and `all` (to match all). Defaults to `all`.
`query` is mandatory. You must provide the indices.
It is forbidden to omit or to give `indices` or `index` multiple times,
or to give both.
Please note that the fields order is important: If the indices are
provided before `query` or `no_match_query`, the query parsing is
skipped altogether.
For instance, this feature is useful to prevent a query that runs
against multiple indices to fail because of a missing type.
See `has_child`, `has_parent`, `top_children` and `nested`.

View File

@ -70,9 +70,9 @@ public class IndicesFilterBuilder extends BaseFilterBuilder {
@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(IndicesFilterParser.NAME);
builder.field("indices", indices);
builder.field("filter");
filterBuilder.toXContent(builder, params);
builder.field("indices", indices);
if (noMatchFilter != null) {
builder.field("no_match_filter");
noMatchFilter.toXContent(builder, params);

View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.query;
import com.google.common.collect.Sets;
import org.apache.lucene.search.Filter;
import org.elasticsearch.action.support.IgnoreIndices;
import org.elasticsearch.cluster.ClusterService;
@ -31,7 +30,9 @@ import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.Set;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
/**
*/
@ -57,26 +58,55 @@ public class IndicesFilterParser implements FilterParser {
XContentParser parser = parseContext.parser();
Filter filter = null;
Filter noMatchFilter = Queries.MATCH_ALL_FILTER;
Filter chosenFilter = null;
boolean filterFound = false;
Set<String> indices = Sets.newHashSet();
boolean indicesFound = false;
boolean matchesConcreteIndices = false;
String currentFieldName = null;
XContentParser.Token token;
Filter noMatchFilter = Queries.MATCH_ALL_FILTER;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("filter".equals(currentFieldName)) {
filterFound = true;
filter = parseContext.parseInnerFilter();
if (indicesFound) {
// Because we know the indices, we can either skip, or parse and use the query
if (matchesConcreteIndices) {
filter = parseContext.parseInnerFilter();
chosenFilter = filter;
} else {
parseContext.parser().skipChildren(); // skip the filter object without parsing it into a Filter
}
} else {
// We do not know the indices, we must parse the query
filter = parseContext.parseInnerFilter();
}
} else if ("no_match_filter".equals(currentFieldName)) {
noMatchFilter = parseContext.parseInnerFilter();
if (indicesFound) {
// Because we know the indices, we can either skip, or parse and use the query
if (!matchesConcreteIndices) {
noMatchFilter = parseContext.parseInnerFilter();
chosenFilter = noMatchFilter;
} else {
parseContext.parser().skipChildren(); // skip the filter object without parsing it into a Filter
}
} else {
// We do not know the indices, we must parse the query
noMatchFilter = parseContext.parseInnerFilter();
}
} else {
throw new QueryParsingException(parseContext.index(), "[indices] filter does not support [" + currentFieldName + "]");
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("indices".equals(currentFieldName)) {
if (indicesFound) {
throw new QueryParsingException(parseContext.index(), "[indices] indices already specified");
}
indicesFound = true;
Collection<String> indices = new ArrayList<String>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
String value = parser.textOrNull();
if (value == null) {
@ -84,12 +114,17 @@ public class IndicesFilterParser implements FilterParser {
}
indices.add(value);
}
matchesConcreteIndices = matchesIndices(parseContext, getConcreteIndices(indices));
} else {
throw new QueryParsingException(parseContext.index(), "[indices] filter does not support [" + currentFieldName + "]");
}
} else if (token.isValue()) {
if ("index".equals(currentFieldName)) {
indices.add(parser.text());
if (indicesFound) {
throw new QueryParsingException(parseContext.index(), "[indices] indices already specified");
}
indicesFound = true;
matchesConcreteIndices = matchesIndices(parseContext, getConcreteIndices(Arrays.asList(parser.text())));
} else if ("no_match_filter".equals(currentFieldName)) {
String type = parser.text();
if ("all".equals(type)) {
@ -97,6 +132,11 @@ public class IndicesFilterParser implements FilterParser {
} else if ("none".equals(type)) {
noMatchFilter = Queries.MATCH_NO_FILTER;
}
if (indicesFound) {
if (!matchesConcreteIndices) {
chosenFilter = noMatchFilter;
}
}
} else {
throw new QueryParsingException(parseContext.index(), "[indices] filter does not support [" + currentFieldName + "]");
}
@ -105,25 +145,38 @@ public class IndicesFilterParser implements FilterParser {
if (!filterFound) {
throw new QueryParsingException(parseContext.index(), "[indices] requires 'filter' element");
}
if (indices.isEmpty()) {
if (!indicesFound) {
throw new QueryParsingException(parseContext.index(), "[indices] requires 'indices' element");
}
if (filter == null) {
return null;
if (chosenFilter == null) {
// Indices were not provided before we encountered the queries, which we hence parsed
// We must now make a choice
if (matchesConcreteIndices) {
chosenFilter = filter;
} else {
chosenFilter = noMatchFilter;
}
}
return chosenFilter;
}
protected String[] getConcreteIndices(Collection<String> indices) {
String[] concreteIndices = indices.toArray(new String[indices.size()]);
if (clusterService != null) {
MetaData metaData = clusterService.state().metaData();
concreteIndices = metaData.concreteIndices(indices.toArray(new String[indices.size()]), IgnoreIndices.MISSING, true);
}
return concreteIndices;
}
protected boolean matchesIndices(QueryParseContext parseContext, String[] concreteIndices) {
for (String index : concreteIndices) {
if (Regex.simpleMatch(index, parseContext.index().name())) {
return filter;
return true;
}
}
return noMatchFilter;
return false;
}
}

View File

@ -70,9 +70,9 @@ public class IndicesQueryBuilder extends BaseQueryBuilder {
@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(IndicesQueryParser.NAME);
builder.field("indices", indices);
builder.field("query");
queryBuilder.toXContent(builder, params);
builder.field("indices", indices);
if (noMatchQuery != null) {
builder.field("no_match_query");
noMatchQuery.toXContent(builder, params);

View File

@ -19,20 +19,20 @@
package org.elasticsearch.index.query;
import com.google.common.collect.Sets;
import org.apache.lucene.search.Query;
import org.elasticsearch.action.support.IgnoreIndices;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.Set;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
/**
*/
@ -58,27 +58,56 @@ public class IndicesQueryParser implements QueryParser {
XContentParser parser = parseContext.parser();
Query query = null;
Query noMatchQuery = Queries.newMatchAllQuery();
Query chosenQuery = null;
boolean queryFound = false;
Set<String> indices = Sets.newHashSet();
boolean indicesFound = false;
boolean matchesConcreteIndices = false;
String queryName = null;
String currentFieldName = null;
XContentParser.Token token;
Query noMatchQuery = Queries.newMatchAllQuery();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("query".equals(currentFieldName)) {
query = parseContext.parseInnerQuery();
queryFound = true;
if (indicesFound) {
// Because we know the indices, we can either skip, or parse and use the query
if (matchesConcreteIndices) {
query = parseContext.parseInnerQuery();
chosenQuery = query;
} else {
parseContext.parser().skipChildren(); // skip the query object without parsing it into a Query
}
} else {
// We do not know the indices, we must parse the query
query = parseContext.parseInnerQuery();
}
} else if ("no_match_query".equals(currentFieldName)) {
noMatchQuery = parseContext.parseInnerQuery();
if (indicesFound) {
// Because we know the indices, we can either skip, or parse and use the query
if (!matchesConcreteIndices) {
noMatchQuery = parseContext.parseInnerQuery();
chosenQuery = noMatchQuery;
} else {
parseContext.parser().skipChildren(); // skip the query object without parsing it into a Query
}
} else {
// We do not know the indices, we must parse the query
noMatchQuery = parseContext.parseInnerQuery();
}
} else {
throw new QueryParsingException(parseContext.index(), "[indices] query does not support [" + currentFieldName + "]");
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("indices".equals(currentFieldName)) {
if (indicesFound) {
throw new QueryParsingException(parseContext.index(), "[indices] indices already specified");
}
indicesFound = true;
Collection<String> indices = new ArrayList<String>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
String value = parser.textOrNull();
if (value == null) {
@ -86,12 +115,17 @@ public class IndicesQueryParser implements QueryParser {
}
indices.add(value);
}
matchesConcreteIndices = matchesIndices(parseContext, getConcreteIndices(indices));
} else {
throw new QueryParsingException(parseContext.index(), "[indices] query does not support [" + currentFieldName + "]");
}
} else if (token.isValue()) {
if ("index".equals(currentFieldName)) {
indices.add(parser.text());
if (indicesFound) {
throw new QueryParsingException(parseContext.index(), "[indices] indices already specified");
}
indicesFound = true;
matchesConcreteIndices = matchesIndices(parseContext, getConcreteIndices(Arrays.asList(parser.text())));
} else if ("no_match_query".equals(currentFieldName)) {
String type = parser.text();
if ("all".equals(type)) {
@ -99,6 +133,11 @@ public class IndicesQueryParser implements QueryParser {
} else if ("none".equals(type)) {
noMatchQuery = Queries.newMatchNoDocsQuery();
}
if (indicesFound) {
if (!matchesConcreteIndices) {
chosenQuery = noMatchQuery;
}
}
} else if ("_name".equals(currentFieldName)) {
queryName = parser.text();
} else {
@ -109,30 +148,42 @@ public class IndicesQueryParser implements QueryParser {
if (!queryFound) {
throw new QueryParsingException(parseContext.index(), "[indices] requires 'query' element");
}
if (query == null) {
return null;
}
if (indices.isEmpty()) {
if (!indicesFound) {
throw new QueryParsingException(parseContext.index(), "[indices] requires 'indices' element");
}
if (chosenQuery == null) {
// Indices were not provided before we encountered the queries, which we hence parsed
// We must now make a choice
if (matchesConcreteIndices) {
chosenQuery = query;
} else {
chosenQuery = noMatchQuery;
}
}
if (queryName != null && chosenQuery != null) {
parseContext.addNamedQuery(queryName, chosenQuery);
}
return chosenQuery;
}
protected String[] getConcreteIndices(Collection<String> indices) {
String[] concreteIndices = indices.toArray(new String[indices.size()]);
if (clusterService != null) {
MetaData metaData = clusterService.state().metaData();
concreteIndices = metaData.concreteIndices(indices.toArray(new String[indices.size()]), IgnoreIndices.MISSING, true);
}
return concreteIndices;
}
protected boolean matchesIndices(QueryParseContext parseContext, String[] concreteIndices) {
for (String index : concreteIndices) {
if (Regex.simpleMatch(index, parseContext.index().name())) {
if (queryName != null) {
parseContext.addNamedQuery(queryName, query);
}
return query;
return true;
}
}
if (queryName != null) {
parseContext.addNamedQuery(queryName, noMatchQuery);
}
return noMatchQuery;
return false;
}
}

View File

@ -1890,4 +1890,178 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
.setQuery(QueryBuilders.multiMatchQuery("value2", "field2^2").lenient(true)).get();
assertHitCount(searchResponse, 1l);
}
@Test
public void testIndicesQuery() throws Exception {
createIndex("index1", "index2");
ensureGreen();
client().prepareIndex("index1", "type1").setId("1").setSource("text", "value").get();
client().prepareIndex("index2", "type2").setId("2").setSource("text", "value").get();
refresh();
SearchResponse response = client().prepareSearch("index1", "index2")
.setQuery(indicesQuery(matchQuery("text", "value"), "index1")
.noMatchQuery(matchQuery("text", "value"))).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
response = client().prepareSearch("index1", "index2")
.setQuery(indicesQuery(matchQuery("text", "value"), "index1")).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
response = client().prepareSearch("index1", "index2")
.setQuery(indicesQuery(matchQuery("text", "value"), "index1")
.noMatchQuery("all")).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
response = client().prepareSearch("index1", "index2")
.setQuery(indicesQuery(matchQuery("text", "value"), "index1")
.noMatchQuery("none")).get();
assertHitCount(response, 1l);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
}
@Test
public void testIndicesFilter() throws Exception {
createIndex("index1", "index2");
ensureGreen();
client().prepareIndex("index1", "type1").setId("1").setSource("text", "value").get();
client().prepareIndex("index2", "type2").setId("2").setSource("text", "value").get();
refresh();
SearchResponse response = client().prepareSearch("index1", "index2")
.setFilter(indicesFilter(termFilter("text", "value"), "index1")
.noMatchFilter(termFilter("text", "value"))).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
response = client().prepareSearch("index1", "index2")
.setFilter(indicesFilter(termFilter("text", "value"), "index1")).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
response = client().prepareSearch("index1", "index2")
.setFilter(indicesFilter(termFilter("text", "value"), "index1")
.noMatchFilter("all")).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
response = client().prepareSearch("index1", "index2")
.setFilter(indicesFilter(termFilter("text", "value"), "index1")
.noMatchFilter("none")).get();
assertHitCount(response, 1l);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
}
@Test // https://github.com/elasticsearch/elasticsearch/issues/2416
public void testIndicesQueryHideParsingExceptions() throws Exception {
client().admin().indices().prepareCreate("simple")
.addMapping("lone", jsonBuilder().startObject().startObject("lone").endObject().endObject())
.get();
client().admin().indices().prepareCreate("related")
.addMapping("parent", jsonBuilder().startObject().startObject("parent").endObject().endObject())
.addMapping("child", jsonBuilder().startObject().startObject("child").startObject("_parent").field("type", "parent")
.endObject().endObject().endObject())
.get();
ensureGreen();
client().prepareIndex("simple", "lone").setId("1").setSource("text", "value").get();
client().prepareIndex("related", "parent").setId("2").setSource("text", "parent").get();
client().prepareIndex("related", "child").setId("3").setParent("2").setSource("text", "value").get();
refresh();
SearchResponse response = client().prepareSearch("related")
.setQuery(hasChildQuery("child", matchQuery("text", "value"))).get();
assertHitCount(response, 1l);
assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
response = client().prepareSearch("simple")
.setQuery(matchQuery("text", "value")).get();
assertHitCount(response, 1l);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
try {
client().prepareSearch("simple")
.setQuery(hasChildQuery("child", matchQuery("text", "value"))).get();
fail("Should have failed with a SearchPhaseExecutionException because all shards failed with a nested QueryParsingException");
// If no failure happens, the HasChildQuery may have changed behavior when provided with wrong types
} catch (SearchPhaseExecutionException e) {
// There is no easy way to ensure we got a QueryParsingException
}
response = client().prepareSearch("related", "simple")
.setQuery(indicesQuery(matchQuery("text", "parent"), "related")
.noMatchQuery(matchQuery("text", "value"))).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
response = client().prepareSearch("related", "simple")
.setQuery(indicesQuery(hasChildQuery("child", matchQuery("text", "value")), "related")
.noMatchQuery(matchQuery("text", "value"))).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
}
@Test // https://github.com/elasticsearch/elasticsearch/issues/2416
public void testIndicesFilterHideParsingExceptions() throws Exception {
client().admin().indices().prepareCreate("simple")
.addMapping("lone", jsonBuilder().startObject().startObject("lone").endObject().endObject())
.get();
client().admin().indices().prepareCreate("related")
.addMapping("parent", jsonBuilder().startObject().startObject("parent").endObject().endObject())
.addMapping("child", jsonBuilder().startObject().startObject("child").startObject("_parent").field("type", "parent")
.endObject().endObject().endObject())
.get();
ensureGreen();
client().prepareIndex("simple", "lone").setId("1").setSource("text", "value").get();
client().prepareIndex("related", "parent").setId("2").setSource("text", "parent").get();
client().prepareIndex("related", "child").setId("3").setParent("2").setSource("text", "value").get();
refresh();
SearchResponse response = client().prepareSearch("related")
.setFilter(hasChildFilter("child", termFilter("text", "value"))).get();
assertHitCount(response, 1l);
assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
response = client().prepareSearch("simple")
.setFilter(termFilter("text", "value")).get();
assertHitCount(response, 1l);
assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
try {
client().prepareSearch("simple")
.setFilter(hasChildFilter("child", termFilter("text", "value"))).get();
fail("Should have failed with a SearchPhaseExecutionException because all shards failed with a nested QueryParsingException");
// If no failure happens, the HasChildQuery may have changed behavior when provided with wrong types
} catch (SearchPhaseExecutionException e) {
// There is no easy way to ensure we got a QueryParsingException
}
response = client().prepareSearch("related", "simple")
.setFilter(indicesFilter(termFilter("text", "parent"), "related")
.noMatchFilter(termFilter("text", "value"))).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
response = client().prepareSearch("related", "simple")
.setFilter(indicesFilter(hasChildFilter("child", termFilter("text", "value")), "related")
.noMatchFilter(termFilter("text", "value"))).get();
assertHitCount(response, 2l);
assertThat(response.getHits().getAt(0).getId(), either(equalTo("1")).or(equalTo("2")));
assertThat(response.getHits().getAt(1).getId(), either(equalTo("1")).or(equalTo("2")));
}
}