From ef3cb2d4368495220dff454b1321810793668115 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 6 Feb 2015 08:41:32 +0100 Subject: [PATCH] percolator: Take filters from index aliases into account when selecting queries to run on a document. The filter from an indexed alias is as if you would filter on the metadata of a percolator query, but then the filter is defined in the index alias instead of the percolate request. Closes #6241 --- .../percolator/PercolateContext.java | 8 +- .../percolator/PercolatorService.java | 50 ++++++------ .../percolator/PercolatorTests.java | 79 ++++++++++++++++++- 3 files changed, 109 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/elasticsearch/percolator/PercolateContext.java b/src/main/java/org/elasticsearch/percolator/PercolateContext.java index 471e58c13e1..da239114c7e 100644 --- a/src/main/java/org/elasticsearch/percolator/PercolateContext.java +++ b/src/main/java/org/elasticsearch/percolator/PercolateContext.java @@ -90,6 +90,7 @@ public class PercolateContext extends SearchContext { private final ScriptService scriptService; private final ConcurrentMap percolateQueries; private final int numberOfShards; + private final Filter aliasFilter; private String[] types; private Engine.Searcher docSearcher; @@ -109,7 +110,7 @@ public class PercolateContext extends SearchContext { public PercolateContext(PercolateShardRequest request, SearchShardTarget searchShardTarget, IndexShard indexShard, IndexService indexService, PageCacheRecycler pageCacheRecycler, - BigArrays bigArrays, ScriptService scriptService) { + BigArrays bigArrays, ScriptService scriptService, Filter aliasFilter) { this.indexShard = indexShard; this.indexService = indexService; this.fieldDataService = indexService.fieldData(); @@ -123,6 +124,7 @@ public class PercolateContext extends SearchContext { this.searcher = new ContextIndexSearcher(this, engineSearcher); this.scriptService = scriptService; this.numberOfShards = request.getNumberOfShards(); + this.aliasFilter = aliasFilter; } public IndexSearcher docSearcher() { @@ -277,7 +279,7 @@ public class PercolateContext extends SearchContext { @Override public Filter searchFilter(String[] types) { - throw new UnsupportedOperationException(); + return aliasFilter(); } @Override @@ -509,7 +511,7 @@ public class PercolateContext extends SearchContext { @Override public Filter aliasFilter() { - throw new UnsupportedOperationException(); + return aliasFilter; } @Override diff --git a/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/src/main/java/org/elasticsearch/percolator/PercolatorService.java index ac29b5d754f..f19b3b076e7 100644 --- a/src/main/java/org/elasticsearch/percolator/PercolatorService.java +++ b/src/main/java/org/elasticsearch/percolator/PercolatorService.java @@ -23,13 +23,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.memory.ExtendedMemoryIndex; import org.apache.lucene.index.memory.MemoryIndex; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FilteredQuery; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.*; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CloseableThreadLocal; import org.elasticsearch.ElasticsearchException; @@ -48,6 +42,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.lucene.search.XBooleanFilter; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.BytesText; import org.elasticsearch.common.text.StringText; @@ -63,22 +58,14 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.Uid; -import org.elasticsearch.index.mapper.internal.IdFieldMapper; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.percolator.stats.ShardPercolateService; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.percolator.QueryCollector.Count; -import org.elasticsearch.percolator.QueryCollector.Match; -import org.elasticsearch.percolator.QueryCollector.MatchAndScore; -import org.elasticsearch.percolator.QueryCollector.MatchAndSort; +import org.elasticsearch.percolator.QueryCollector.*; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchShardTarget; @@ -96,9 +83,7 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.index.mapper.SourceToParse.source; -import static org.elasticsearch.percolator.QueryCollector.count; -import static org.elasticsearch.percolator.QueryCollector.match; -import static org.elasticsearch.percolator.QueryCollector.matchAndScore; +import static org.elasticsearch.percolator.QueryCollector.*; public class PercolatorService extends AbstractComponent { @@ -174,9 +159,15 @@ public class PercolatorService extends AbstractComponent { shardPercolateService.prePercolate(); long startTime = System.nanoTime(); + String[] filteringAliases = clusterService.state().getMetaData().filteringAliases( + indexShard.shardId().index().name(), + request.indices() + ); + Filter aliasFilter = percolateIndexService.aliasesService().aliasFilter(filteringAliases); + SearchShardTarget searchShardTarget = new SearchShardTarget(clusterService.localNode().id(), request.shardId().getIndex(), request.shardId().id()); final PercolateContext context = new PercolateContext( - request, searchShardTarget, indexShard, percolateIndexService, pageCacheRecycler, bigArrays, scriptService + request, searchShardTarget, indexShard, percolateIndexService, pageCacheRecycler, bigArrays, scriptService, aliasFilter ); try { ParsedDocument parsedDocument = parseRequest(percolateIndexService, request, context); @@ -190,7 +181,7 @@ public class PercolatorService extends AbstractComponent { throw new ElasticsearchIllegalArgumentException("Nothing to percolate"); } - if (context.percolateQuery() == null && (context.trackScores() || context.doSort || context.aggregations() != null)) { + if (context.percolateQuery() == null && (context.trackScores() || context.doSort || context.aggregations() != null) || context.aliasFilter() != null) { context.percolateQuery(new MatchAllDocsQuery()); } @@ -779,8 +770,19 @@ public class PercolatorService extends AbstractComponent { private void queryBasedPercolating(Engine.Searcher percolatorSearcher, PercolateContext context, QueryCollector percolateCollector) throws IOException { Filter percolatorTypeFilter = context.indexService().mapperService().documentMapper(TYPE_NAME).typeFilter(); - percolatorTypeFilter = context.indexService().cache().filter().cache(percolatorTypeFilter, null, context.indexService().queryParserService().autoFilterCachePolicy()); - FilteredQuery query = new FilteredQuery(context.percolateQuery(), percolatorTypeFilter); + percolatorTypeFilter = context.indexService().cache().filter().cache(percolatorTypeFilter, null, context.queryParserService().autoFilterCachePolicy()); + + final Filter filter; + if (context.aliasFilter() != null) { + XBooleanFilter booleanFilter = new XBooleanFilter(); + booleanFilter.add(context.aliasFilter(), BooleanClause.Occur.MUST); + booleanFilter.add(percolatorTypeFilter, BooleanClause.Occur.MUST); + filter = booleanFilter; + } else { + filter = percolatorTypeFilter; + } + + FilteredQuery query = new FilteredQuery(context.percolateQuery(), filter); percolatorSearcher.searcher().search(query, percolateCollector); percolateCollector.aggregatorCollector.postCollection(); if (context.aggregations() != null) { diff --git a/src/test/java/org/elasticsearch/percolator/PercolatorTests.java b/src/test/java/org/elasticsearch/percolator/PercolatorTests.java index c136df36055..d6b9bd852d1 100644 --- a/src/test/java/org/elasticsearch/percolator/PercolatorTests.java +++ b/src/test/java/org/elasticsearch/percolator/PercolatorTests.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicate; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -50,7 +51,6 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.junit.Test; import java.io.IOException; @@ -950,7 +950,84 @@ public class PercolatorTests extends ElasticsearchIntegrationTest { for (PercolateResponse.Match match : response) { assertThat(match.getIndex().string(), equalTo("test2")); } + } + @Test + public void testPercolateWithAliasFilter() throws Exception { + assertAcked(prepareCreate("my-index") + .addMapping(PercolatorService.TYPE_NAME, "a", "type=string,index=not_analyzed") + .addAlias(new Alias("a").filter(FilterBuilders.termFilter("a", "a"))) + .addAlias(new Alias("b").filter(FilterBuilders.termFilter("a", "b"))) + .addAlias(new Alias("c").filter(FilterBuilders.termFilter("a", "c"))) + ); + client().prepareIndex("my-index", PercolatorService.TYPE_NAME, "1") + .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("a", "a").endObject()) + .get(); + client().prepareIndex("my-index", PercolatorService.TYPE_NAME, "2") + .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("a", "b").endObject()) + .get(); + refresh(); + + // Specifying only the document to percolate and no filter, sorting or aggs, the queries are retrieved from + // memory directly. Otherwise we need to retrieve those queries from lucene to be able to execute filters, + // aggregations and sorting on top of them. So this test a different code execution path. + PercolateResponse response = client().preparePercolate() + .setIndices("a") + .setDocumentType("my-type") + .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}")) + .get(); + assertNoFailures(response); + assertThat(response.getCount(), equalTo(1l)); + assertThat(response.getMatches()[0].getId().string(), equalTo("1")); + + response = client().preparePercolate() + .setIndices("b") + .setDocumentType("my-type") + .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}")) + .get(); + assertNoFailures(response); + assertThat(response.getCount(), equalTo(1l)); + assertThat(response.getMatches()[0].getId().string(), equalTo("2")); + + + response = client().preparePercolate() + .setIndices("c") + .setDocumentType("my-type") + .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}")) + .get(); + assertNoFailures(response); + assertThat(response.getCount(), equalTo(0l)); + + // Testing that the alias filter and the filter specified while percolating are both taken into account. + response = client().preparePercolate() + .setIndices("a") + .setDocumentType("my-type") + .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}")) + .setPercolateFilter(FilterBuilders.matchAllFilter()) + .get(); + assertNoFailures(response); + assertThat(response.getCount(), equalTo(1l)); + assertThat(response.getMatches()[0].getId().string(), equalTo("1")); + + response = client().preparePercolate() + .setIndices("b") + .setDocumentType("my-type") + .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}")) + .setPercolateFilter(FilterBuilders.matchAllFilter()) + .get(); + assertNoFailures(response); + assertThat(response.getCount(), equalTo(1l)); + assertThat(response.getMatches()[0].getId().string(), equalTo("2")); + + + response = client().preparePercolate() + .setIndices("c") + .setDocumentType("my-type") + .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}")) + .setPercolateFilter(FilterBuilders.matchAllFilter()) + .get(); + assertNoFailures(response); + assertThat(response.getCount(), equalTo(0l)); } @Test