diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java index 8c0c427beea..9a7bb5c8f3d 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.fielddata.FieldDataStats; -import org.elasticsearch.index.percolator.PercolatorQueryCacheStats; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.search.suggest.completion.CompletionStats; @@ -45,7 +44,6 @@ public class ClusterStatsIndices implements ToXContent { private QueryCacheStats queryCache; private CompletionStats completion; private SegmentsStats segments; - private PercolatorQueryCacheStats percolatorCache; public ClusterStatsIndices(List nodeResponses) { ObjectObjectHashMap countsPerIndex = new ObjectObjectHashMap<>(); @@ -56,7 +54,6 @@ public class ClusterStatsIndices implements ToXContent { this.queryCache = new QueryCacheStats(); this.completion = new CompletionStats(); this.segments = new SegmentsStats(); - this.percolatorCache = new PercolatorQueryCacheStats(); for (ClusterStatsNodeResponse r : nodeResponses) { for (org.elasticsearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { @@ -79,7 +76,6 @@ public class ClusterStatsIndices implements ToXContent { queryCache.add(shardCommonStats.queryCache); completion.add(shardCommonStats.completion); segments.add(shardCommonStats.segments); - percolatorCache.add(shardCommonStats.percolatorCache); } } @@ -122,10 +118,6 @@ public class ClusterStatsIndices implements ToXContent { return segments; } - public PercolatorQueryCacheStats getPercolatorCache() { - return percolatorCache; - } - static final class Fields { static final String COUNT = "count"; } @@ -140,7 +132,6 @@ public class ClusterStatsIndices implements ToXContent { queryCache.toXContent(builder, params); completion.toXContent(builder, params); segments.toXContent(builder, params); - percolatorCache.toXContent(builder, params); return builder; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index bae7b20694d..3a0b1455209 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -55,8 +55,7 @@ public class TransportClusterStatsAction extends TransportNodesAction { private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags(CommonStatsFlags.Flag.Docs, CommonStatsFlags.Flag.Store, - CommonStatsFlags.Flag.FieldData, CommonStatsFlags.Flag.QueryCache, CommonStatsFlags.Flag.Completion, CommonStatsFlags.Flag.Segments, - CommonStatsFlags.Flag.PercolatorCache); + CommonStatsFlags.Flag.FieldData, CommonStatsFlags.Flag.QueryCache, CommonStatsFlags.Flag.Completion, CommonStatsFlags.Flag.Segments); private final NodeService nodeService; private final IndicesService indicesService; @@ -100,7 +99,7 @@ public class TransportClusterStatsAction extends TransportNodesAction { return flags.isSet(Flag.FieldData); } - public IndicesStatsRequest percolate(boolean percolate) { - flags.set(Flag.PercolatorCache, percolate); - return this; - } - - public boolean percolate() { - return flags.isSet(Flag.PercolatorCache); - } - public IndicesStatsRequest segments(boolean segments) { flags.set(Flag.Segments, segments); return this; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java index cad919cbd18..8e7afe3e7e3 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java @@ -127,11 +127,6 @@ public class IndicesStatsRequestBuilder extends BroadcastOperationRequestBuilder return this; } - public IndicesStatsRequestBuilder setPercolate(boolean percolate) { - request.percolate(percolate); - return this; - } - public IndicesStatsRequestBuilder setSegments(boolean segments) { request.segments(segments); return this; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index 8c12dfa9fda..7e8ccd30a8a 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -139,9 +139,6 @@ public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction< flags.set(CommonStatsFlags.Flag.FieldData); flags.fieldDataFields(request.fieldDataFields()); } - if (request.percolate()) { - flags.set(CommonStatsFlags.Flag.PercolatorCache); - } if (request.segments()) { flags.set(CommonStatsFlags.Flag.Segments); flags.includeSegmentFileSizes(request.includeSegmentFileSizes()); @@ -163,6 +160,6 @@ public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction< flags.set(CommonStatsFlags.Flag.Recovery); } - return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexService.cache().getPercolatorQueryCache(), indexShard, flags), indexShard.commitStats()); + return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), indexShard.commitStats()); } } diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 1b795239457..9ebf6853166 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -35,7 +35,7 @@ import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.percolator.PercolatorQueryCache; +import org.elasticsearch.index.percolator.PercolatorFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.index.store.IndexStore; @@ -126,7 +126,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.IGNORE_MALFORMED_SETTING, FieldMapper.COERCE_SETTING, Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING, - PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING, + PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING, MapperService.INDEX_MAPPER_DYNAMIC_SETTING, MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index 60e3250e49d..f5e5ce91d80 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -50,7 +50,6 @@ import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexEventListener; @@ -151,11 +150,9 @@ public final class IndexService extends AbstractIndexComponent implements IndexC this.indexStore = indexStore; indexFieldData.setListener(new FieldDataCacheListener(this)); this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this)); - PercolatorQueryCache percolatorQueryCache = new PercolatorQueryCache(indexSettings, IndexService.this::newQueryShardContext); this.warmer = new IndexWarmer(indexSettings.getSettings(), threadPool, - bitsetFilterCache.createListener(threadPool), - percolatorQueryCache.createListener(threadPool)); - this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache, percolatorQueryCache); + bitsetFilterCache.createListener(threadPool)); + this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache); this.engineFactory = engineFactory; // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE this.searcherWrapper = wrapperFactory.newWrapper(this); @@ -239,8 +236,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC } } } finally { - IOUtils.close(bitsetFilterCache, indexCache, indexFieldData, analysisService, refreshTask, fsyncTask, - cache().getPercolatorQueryCache()); + IOUtils.close(bitsetFilterCache, indexCache, indexFieldData, analysisService, refreshTask, fsyncTask); } } } @@ -443,7 +439,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC return new QueryShardContext( indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(), similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry(), - nodeServicesProvider.getClient(), indexCache.getPercolatorQueryCache(), indexReader, + nodeServicesProvider.getClient(), indexReader, nodeServicesProvider.getClusterService().state() ); } diff --git a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java index b41f5bc0125..61733f24695 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java @@ -24,7 +24,6 @@ import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.cache.query.QueryCache; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import java.io.Closeable; import java.io.IOException; @@ -36,14 +35,11 @@ public class IndexCache extends AbstractIndexComponent implements Closeable { private final QueryCache queryCache; private final BitsetFilterCache bitsetFilterCache; - private final PercolatorQueryCache percolatorQueryCache; - public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache, - PercolatorQueryCache percolatorQueryCache) { + public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache) { super(indexSettings); this.queryCache = queryCache; this.bitsetFilterCache = bitsetFilterCache; - this.percolatorQueryCache = percolatorQueryCache; } public QueryCache query() { @@ -57,13 +53,9 @@ public class IndexCache extends AbstractIndexComponent implements Closeable { return bitsetFilterCache; } - public PercolatorQueryCache getPercolatorQueryCache() { - return percolatorQueryCache; - } - @Override public void close() throws IOException { - IOUtils.close(queryCache, bitsetFilterCache, percolatorQueryCache); + IOUtils.close(queryCache, bitsetFilterCache); } public void clear(String reason) { diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java index 59b980d59a4..decb91e960a 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java @@ -23,11 +23,13 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -36,6 +38,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.core.BinaryFieldMapper; import org.elasticsearch.index.mapper.core.KeywordFieldMapper; +import org.elasticsearch.index.query.PercolateQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; @@ -50,14 +53,17 @@ import java.util.Map; public class PercolatorFieldMapper extends FieldMapper { + public final static XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE; + public final static Setting INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING = + Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope); @Deprecated public static final String LEGACY_TYPE_NAME = ".percolator"; public static final String CONTENT_TYPE = "percolator"; private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType(); - private static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms"; - private static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query"; - static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field"; + public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms"; + public static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query"; + public static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field"; public static class Builder extends FieldMapper.Builder { @@ -172,7 +178,7 @@ public class PercolatorFieldMapper extends FieldMapper { this.queryTermsField = queryTermsField; this.unknownQueryField = unknownQueryField; this.queryBuilderField = queryBuilderField; - this.mapUnmappedFieldAsString = PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings); + this.mapUnmappedFieldAsString = INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings); } @Override @@ -196,7 +202,7 @@ public class PercolatorFieldMapper extends FieldMapper { // Fetching of terms, shapes and indexed scripts happen during this rewrite: queryBuilder = queryBuilder.rewrite(queryShardContext); - try (XContentBuilder builder = XContentFactory.contentBuilder(PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE)) { + try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) { queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap())); builder.flush(); byte[] queryBuilderAsBytes = builder.bytes().toBytes(); diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java index 9fb6934cca4..2c231593860 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java @@ -40,6 +40,7 @@ import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SubSearchContext; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -69,8 +70,8 @@ public class PercolatorHighlightSubFetchPhase implements FetchSubPhase { } List ctxs = context.searcher().getIndexReader().leaves(); - PercolatorQueryCache queriesRegistry = context.percolatorQueryCache(); IndexSearcher percolatorIndexSearcher = percolateQuery.getPercolatorIndexSearcher(); + PercolateQuery.QueryStore queryStore = percolateQuery.getQueryStore(); LeafReaderContext percolatorLeafReaderContext = percolatorIndexSearcher.getIndexReader().leaves().get(0); FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); @@ -78,9 +79,14 @@ public class PercolatorHighlightSubFetchPhase implements FetchSubPhase { createSubSearchContext(context, percolatorLeafReaderContext, percolateQuery.getDocumentSource()); for (InternalSearchHit hit : hits) { - LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs)); - int segmentDocId = hit.docId() - ctx.docBase; - Query query = queriesRegistry.getQueries(ctx).getQuery(segmentDocId); + final Query query; + try { + LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs)); + int segmentDocId = hit.docId() - ctx.docBase; + query = queryStore.getQueries(ctx).getQuery(segmentDocId); + } catch (IOException e) { + throw new RuntimeException(e); + } if (query != null) { subSearchContext.parsedQuery(new ParsedQuery(query)); hitContext.reset( diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java deleted file mode 100644 index e7ca6c3d427..00000000000 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.percolator; - -import com.carrotsearch.hppc.IntObjectHashMap; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.cache.Cache; -import org.elasticsearch.common.cache.CacheBuilder; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexWarmer; -import org.elasticsearch.index.IndexWarmer.TerminationHandle; -import org.elasticsearch.index.engine.Engine.Searcher; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.internal.SourceFieldMapper; -import org.elasticsearch.index.mapper.internal.TypeFieldMapper; -import org.elasticsearch.index.query.PercolateQuery; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.ShardUtils; -import org.elasticsearch.threadpool.ThreadPool; - -import java.io.Closeable; -import java.io.IOException; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.function.Supplier; - -import static org.elasticsearch.index.percolator.PercolatorFieldMapper.LEGACY_TYPE_NAME; -import static org.elasticsearch.index.percolator.PercolatorFieldMapper.PercolatorFieldType; -import static org.elasticsearch.index.percolator.PercolatorFieldMapper.parseQuery; - -public final class PercolatorQueryCache extends AbstractIndexComponent - implements Closeable, LeafReader.CoreClosedListener, PercolateQuery.QueryRegistry { - - public final static Setting INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING = - Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope); - - public final static XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE; - - private final Supplier queryShardContextSupplier; - private final Cache cache; - private final boolean mapUnmappedFieldsAsString; - - public PercolatorQueryCache(IndexSettings indexSettings, Supplier queryShardContextSupplier) { - super(indexSettings); - this.queryShardContextSupplier = queryShardContextSupplier; - cache = CacheBuilder.builder().build(); - this.mapUnmappedFieldsAsString = indexSettings.getValue(INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING); - } - - @Override - public Leaf getQueries(LeafReaderContext ctx) { - QueriesLeaf percolatorQueries = cache.get(ctx.reader().getCoreCacheKey()); - if (percolatorQueries == null) { - throw new IllegalStateException("queries not loaded, queries should be have been preloaded during index warming..."); - } - return percolatorQueries; - } - - public IndexWarmer.Listener createListener(ThreadPool threadPool) { - return new IndexWarmer.Listener() { - - final Executor executor = threadPool.executor(ThreadPool.Names.WARMER); - - @Override - public TerminationHandle warmReader(IndexShard indexShard, Searcher searcher) { - final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size()); - for (final LeafReaderContext ctx : searcher.reader().leaves()) { - if (cache.get(ctx.reader().getCoreCacheKey()) != null) { - latch.countDown(); - continue; - } - executor.execute(() -> { - try { - final long start = System.nanoTime(); - QueriesLeaf queries = loadQueries(ctx, indexShard); - cache.put(ctx.reader().getCoreCacheKey(), queries); - if (indexShard.warmerService().logger().isTraceEnabled()) { - indexShard.warmerService().logger().trace( - "loading percolator queries took [{}]", - TimeValue.timeValueNanos(System.nanoTime() - start) - ); - } - } catch (Throwable t) { - indexShard.warmerService().logger().warn("failed to load percolator queries", t); - } finally { - latch.countDown(); - } - }); - } - return () -> latch.await(); - } - }; - } - - QueriesLeaf loadQueries(LeafReaderContext context, IndexShard indexShard) throws IOException { - Version indexVersionCreated = indexShard.indexSettings().getIndexVersionCreated(); - MapperService mapperService = indexShard.mapperService(); - LeafReader leafReader = context.reader(); - ShardId shardId = ShardUtils.extractShardId(leafReader); - if (shardId == null) { - throw new IllegalStateException("can't resolve shard id"); - } - if (indexSettings.getIndex().equals(shardId.getIndex()) == false) { - // percolator cache insanity - String message = "Trying to load queries for index " + shardId.getIndex() + " with cache of index " + - indexSettings.getIndex(); - throw new IllegalStateException(message); - } - - IntObjectHashMap queries = new IntObjectHashMap<>(); - boolean legacyLoading = indexVersionCreated.before(Version.V_5_0_0_alpha1); - if (legacyLoading) { - PostingsEnum postings = leafReader.postings(new Term(TypeFieldMapper.NAME, LEGACY_TYPE_NAME), PostingsEnum.NONE); - if (postings != null) { - LegacyQueryFieldVisitor visitor = new LegacyQueryFieldVisitor(); - for (int docId = postings.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = postings.nextDoc()) { - leafReader.document(docId, visitor); - queries.put(docId, parseLegacyPercolatorDocument(docId, visitor.source)); - visitor.source = null; // reset - } - } - } else { - // Each type can have one percolator field mapper, - // So for each type we check if there is a percolator field mapper - // and parse all the queries for the documents of that type. - IndexSearcher indexSearcher = new IndexSearcher(leafReader); - for (DocumentMapper documentMapper : mapperService.docMappers(false)) { - Weight queryWeight = indexSearcher.createNormalizedWeight(documentMapper.typeFilter(), false); - for (FieldMapper fieldMapper : documentMapper.mappers()) { - if (fieldMapper instanceof PercolatorFieldMapper) { - PercolatorFieldType fieldType = (PercolatorFieldType) fieldMapper.fieldType(); - BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.getQueryBuilderFieldName()); - if (binaryDocValues != null) { - // use the same leaf reader context the indexSearcher is using too: - Scorer scorer = queryWeight.scorer(leafReader.getContext()); - if (scorer != null) { - DocIdSetIterator iterator = scorer.iterator(); - for (int docId = iterator.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) { - BytesRef qbSource = binaryDocValues.get(docId); - if (qbSource.length > 0) { - queries.put(docId, parseQueryBuilder(docId, qbSource)); - } - } - } - } - break; - } - } - } - } - leafReader.addCoreClosedListener(this); - return new QueriesLeaf(shardId, queries); - } - - private Query parseQueryBuilder(int docId, BytesRef qbSource) { - XContent xContent = QUERY_BUILDER_CONTENT_TYPE.xContent(); - try (XContentParser sourceParser = xContent.createParser(qbSource.bytes, qbSource.offset, qbSource.length)) { - QueryShardContext context = queryShardContextSupplier.get(); - return parseQuery(context, mapUnmappedFieldsAsString, sourceParser); - } catch (IOException e) { - throw new PercolatorException(index(), "failed to parse query builder for document [" + docId + "]", e); - } - } - - private Query parseLegacyPercolatorDocument(int docId, BytesReference source) { - try (XContentParser sourceParser = XContentHelper.createParser(source)) { - String currentFieldName = null; - XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchException("failed to parse query [" + docId + "], not starting with OBJECT"); - } - while ((token = sourceParser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = sourceParser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if ("query".equals(currentFieldName)) { - QueryShardContext context = queryShardContextSupplier.get(); - return parseQuery(context, mapUnmappedFieldsAsString, sourceParser); - } else { - sourceParser.skipChildren(); - } - } else if (token == XContentParser.Token.START_ARRAY) { - sourceParser.skipChildren(); - } - } - } catch (Exception e) { - throw new PercolatorException(index(), "failed to parse query [" + docId + "]", e); - } - return null; - } - - public PercolatorQueryCacheStats getStats(ShardId shardId) { - int numberOfQueries = 0; - for (QueriesLeaf queries : cache.values()) { - if (shardId.equals(queries.shardId)) { - numberOfQueries += queries.queries.size(); - } - } - return new PercolatorQueryCacheStats(numberOfQueries); - } - - @Override - public void onClose(Object cacheKey) throws IOException { - cache.invalidate(cacheKey); - } - - @Override - public void close() throws IOException { - cache.invalidateAll(); - } - - final static class LegacyQueryFieldVisitor extends StoredFieldVisitor { - - private BytesArray source; - - @Override - public void binaryField(FieldInfo fieldInfo, byte[] bytes) throws IOException { - source = new BytesArray(bytes); - } - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (source != null) { - return Status.STOP; - } - if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { - return Status.YES; - } else { - return Status.NO; - } - } - - } - - final static class QueriesLeaf implements Leaf { - - final ShardId shardId; - final IntObjectHashMap queries; - - QueriesLeaf(ShardId shardId, IntObjectHashMap queries) { - this.shardId = shardId; - this.queries = queries; - } - - @Override - public Query getQuery(int docId) { - return queries.get(docId); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java deleted file mode 100644 index cbc21286e20..00000000000 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.percolator; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; - -/** - * Exposes percolator query cache statistics. - */ -public class PercolatorQueryCacheStats implements Streamable, ToXContent { - - private long numQueries; - - /** - * Noop constructor for serialization purposes. - */ - public PercolatorQueryCacheStats() { - } - - PercolatorQueryCacheStats(long numQueries) { - this.numQueries = numQueries; - } - - /** - * @return The total number of loaded percolate queries. - */ - public long getNumQueries() { - return numQueries; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.PERCOLATOR); - builder.field(Fields.QUERIES, getNumQueries()); - builder.endObject(); - return builder; - } - - public void add(PercolatorQueryCacheStats percolate) { - if (percolate == null) { - return; - } - - numQueries += percolate.getNumQueries(); - } - - static final class Fields { - static final String PERCOLATOR = "percolator"; - static final String QUERIES = "num_queries"; - } - - public static PercolatorQueryCacheStats readPercolateStats(StreamInput in) throws IOException { - PercolatorQueryCacheStats stats = new PercolatorQueryCacheStats(); - stats.readFrom(in); - return stats; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - numQueries = in.readVLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(numQueries); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java b/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java index e99687422db..41448e14ef7 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java +++ b/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java @@ -50,7 +50,7 @@ public final class PercolateQuery extends Query implements Accountable { public static class Builder { private final String docType; - private final QueryRegistry queryRegistry; + private final QueryStore queryStore; private final BytesReference documentSource; private final IndexSearcher percolatorIndexSearcher; @@ -59,15 +59,15 @@ public final class PercolateQuery extends Query implements Accountable { /** * @param docType The type of the document being percolated - * @param queryRegistry The registry holding all the percolator queries as Lucene queries. + * @param queryStore The lookup holding all the percolator queries as Lucene queries. * @param documentSource The source of the document being percolated * @param percolatorIndexSearcher The index searcher on top of the in-memory index that holds the document being percolated */ - public Builder(String docType, QueryRegistry queryRegistry, BytesReference documentSource, IndexSearcher percolatorIndexSearcher) { + public Builder(String docType, QueryStore queryStore, BytesReference documentSource, IndexSearcher percolatorIndexSearcher) { this.docType = Objects.requireNonNull(docType); this.documentSource = Objects.requireNonNull(documentSource); this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher); - this.queryRegistry = Objects.requireNonNull(queryRegistry); + this.queryStore = Objects.requireNonNull(queryStore); } /** @@ -94,7 +94,6 @@ public final class PercolateQuery extends Query implements Accountable { if (percolateTypeQuery != null && queriesMetaDataQuery != null) { throw new IllegalStateException("Either filter by deprecated percolator type or by query metadata"); } - // The query that selects which percolator queries will be evaluated by MemoryIndex: BooleanQuery.Builder builder = new BooleanQuery.Builder(); if (percolateTypeQuery != null) { @@ -103,24 +102,23 @@ public final class PercolateQuery extends Query implements Accountable { if (queriesMetaDataQuery != null) { builder.add(queriesMetaDataQuery, FILTER); } - - return new PercolateQuery(docType, queryRegistry, documentSource, builder.build(), percolatorIndexSearcher); + return new PercolateQuery(docType, queryStore, documentSource, builder.build(), percolatorIndexSearcher); } } private final String documentType; - private final QueryRegistry queryRegistry; + private final QueryStore queryStore; private final BytesReference documentSource; private final Query percolatorQueriesQuery; private final IndexSearcher percolatorIndexSearcher; - private PercolateQuery(String documentType, QueryRegistry queryRegistry, BytesReference documentSource, + private PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource, Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher) { this.documentType = documentType; this.documentSource = documentSource; this.percolatorQueriesQuery = percolatorQueriesQuery; - this.queryRegistry = queryRegistry; + this.queryStore = queryStore; this.percolatorIndexSearcher = percolatorIndexSearcher; } @@ -128,7 +126,7 @@ public final class PercolateQuery extends Query implements Accountable { public Query rewrite(IndexReader reader) throws IOException { Query rewritten = percolatorQueriesQuery.rewrite(reader); if (rewritten != percolatorQueriesQuery) { - return new PercolateQuery(documentType, queryRegistry, documentSource, rewritten, percolatorIndexSearcher); + return new PercolateQuery(documentType, queryStore, documentSource, rewritten, percolatorIndexSearcher); } else { return this; } @@ -151,7 +149,7 @@ public final class PercolateQuery extends Query implements Accountable { if (result == docId) { if (twoPhaseIterator.matches()) { if (needsScores) { - QueryRegistry.Leaf percolatorQueries = queryRegistry.getQueries(leafReaderContext); + QueryStore.Leaf percolatorQueries = queryStore.getQueries(leafReaderContext); Query query = percolatorQueries.getQuery(docId); Explanation detail = percolatorIndexSearcher.explain(query, 0); return Explanation.match(scorer.score(), "PercolateQuery", detail); @@ -181,9 +179,9 @@ public final class PercolateQuery extends Query implements Accountable { return null; } - final QueryRegistry.Leaf percolatorQueries = queryRegistry.getQueries(leafReaderContext); + final QueryStore.Leaf queries = queryStore.getQueries(leafReaderContext); if (needsScores) { - return new BaseScorer(this, approximation, percolatorQueries, percolatorIndexSearcher) { + return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) { float score; @@ -209,7 +207,7 @@ public final class PercolateQuery extends Query implements Accountable { } }; } else { - return new BaseScorer(this, approximation, percolatorQueries, percolatorIndexSearcher) { + return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) { @Override public float score() throws IOException { @@ -238,6 +236,10 @@ public final class PercolateQuery extends Query implements Accountable { return documentSource; } + public QueryStore getQueryStore() { + return queryStore; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -276,13 +278,15 @@ public final class PercolateQuery extends Query implements Accountable { return sizeInBytes; } - public interface QueryRegistry { + @FunctionalInterface + public interface QueryStore { - Leaf getQueries(LeafReaderContext ctx); + Leaf getQueries(LeafReaderContext ctx) throws IOException; + @FunctionalInterface interface Leaf { - Query getQuery(int docId); + Query getQuery(int docId) throws IOException; } @@ -291,10 +295,10 @@ public final class PercolateQuery extends Query implements Accountable { static abstract class BaseScorer extends Scorer { final Scorer approximation; - final QueryRegistry.Leaf percolatorQueries; + final QueryStore.Leaf percolatorQueries; final IndexSearcher percolatorIndexSearcher; - BaseScorer(Weight weight, Scorer approximation, QueryRegistry.Leaf percolatorQueries, IndexSearcher percolatorIndexSearcher) { + BaseScorer(Weight weight, Scorer approximation, QueryStore.Leaf percolatorQueries, IndexSearcher percolatorIndexSearcher) { super(weight); this.approximation = approximation; this.percolatorQueries = percolatorQueries; diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java index 8341459b155..59be8d7a2d0 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java @@ -21,10 +21,13 @@ package org.elasticsearch.index.query; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.BooleanClause; @@ -33,23 +36,27 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.Weight; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.FieldNameAnalyzer; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperForType; @@ -57,15 +64,16 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.percolator.PercolatorFieldMapper; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import java.io.IOException; import java.util.List; import java.util.Objects; import static org.elasticsearch.index.mapper.SourceToParse.source; +import static org.elasticsearch.index.percolator.PercolatorFieldMapper.parseQuery; public class PercolateQueryBuilder extends AbstractQueryBuilder { @@ -388,16 +396,14 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder { + LeafReader leafReader = ctx.reader(); + BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.getQueryBuilderFieldName()); + if (binaryDocValues == null) { + return docId -> null; + } + + Bits bits = leafReader.getDocsWithField(fieldType.getQueryBuilderFieldName()); + return docId -> { + if (bits.get(docId)) { + BytesRef qbSource = binaryDocValues.get(docId); + if (qbSource.length > 0) { + XContent xContent = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent(); + try (XContentParser sourceParser = xContent.createParser(qbSource.bytes, qbSource.offset, qbSource.length)) { + return parseQuery(context, mapUnmappedFieldsAsString, sourceParser); + } + } else { + return null; + } + } else { + return null; + } + }; + }; + } + + private static PercolateQuery.QueryStore createLegacyStore(QueryShardContext context, boolean mapUnmappedFieldsAsString) { + return ctx -> { + LeafReader leafReader = ctx.reader(); + return docId -> { + LegacyQueryFieldVisitor visitor = new LegacyQueryFieldVisitor(); + leafReader.document(docId, visitor); + if (visitor.source == null) { + throw new IllegalStateException("No source found for document with docid [" + docId + "]"); + } + + try (XContentParser sourceParser = XContentHelper.createParser(visitor.source)) { + String currentFieldName = null; + XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchException("failed to parse query [" + docId + "], not starting with OBJECT"); + } + while ((token = sourceParser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = sourceParser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if ("query".equals(currentFieldName)) { + return parseQuery(context, mapUnmappedFieldsAsString, sourceParser); + } else { + sourceParser.skipChildren(); + } + } else if (token == XContentParser.Token.START_ARRAY) { + sourceParser.skipChildren(); + } + } + } + return null; + }; + }; + } + + private final static class LegacyQueryFieldVisitor extends StoredFieldVisitor { + + private BytesArray source; + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] bytes) throws IOException { + source = new BytesArray(bytes); + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (source != null) { + return Status.STOP; + } + if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } else { + return Status.NO; + } + } + + } + } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 4aa72728bc8..3cdabb74d7e 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.TextFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import org.elasticsearch.index.query.support.NestedScope; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.query.IndicesQueriesRegistry; @@ -82,7 +81,6 @@ public class QueryShardContext extends QueryRewriteContext { private final Map namedQueries = new HashMap<>(); private final MapperQueryParser queryParser = new MapperQueryParser(this); private final IndicesQueriesRegistry indicesQueriesRegistry; - private final PercolatorQueryCache percolatorQueryCache; private boolean allowUnmappedFields; private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; @@ -90,7 +88,7 @@ public class QueryShardContext extends QueryRewriteContext { public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService, - final IndicesQueriesRegistry indicesQueriesRegistry, Client client, PercolatorQueryCache percolatorQueryCache, + final IndicesQueriesRegistry indicesQueriesRegistry, Client client, IndexReader reader, ClusterState clusterState) { super(indexSettings, mapperService, scriptService, indicesQueriesRegistry, client, reader, clusterState); this.indexSettings = indexSettings; @@ -100,14 +98,13 @@ public class QueryShardContext extends QueryRewriteContext { this.indexFieldDataService = indexFieldDataService; this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields(); this.indicesQueriesRegistry = indicesQueriesRegistry; - this.percolatorQueryCache = percolatorQueryCache; this.nestedScope = new NestedScope(); } public QueryShardContext(QueryShardContext source) { this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry, source.client, - source.percolatorQueryCache, source.reader, source.clusterState); + source.reader, source.clusterState); this.types = source.getTypes(); } @@ -123,10 +120,6 @@ public class QueryShardContext extends QueryRewriteContext { return mapperService.analysisService(); } - public PercolatorQueryCache getPercolatorQueryCache() { - return percolatorQueryCache; - } - public Similarity getSearchSimilarity() { return similarityService != null ? similarityService.similarity(mapperService) : null; } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index 3b9b186ddd8..6b1d24e47f1 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -280,7 +280,7 @@ public class IndicesService extends AbstractLifecycleComponent i if (indexShard.routingEntry() == null) { continue; } - IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexService.cache().getPercolatorQueryCache(), indexShard, flags), indexShard.commitStats()) }); + IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexShard, flags), indexShard.commitStats()) }); if (!statsByShard.containsKey(indexService.index())) { statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats)); } else { diff --git a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java index f4736f0332e..f205cdd8562 100644 --- a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java +++ b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java @@ -37,7 +37,6 @@ import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.percolator.PercolatorQueryCacheStats; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; @@ -102,11 +101,6 @@ public class NodeIndicesStats implements Streamable, ToXContent { return stats.getSearch(); } - @Nullable - public PercolatorQueryCacheStats getPercolate() { - return stats.getPercolatorCache(); - } - @Nullable public MergeStats getMerge() { return stats.getMerge(); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java index dbda83709ba..55ed1d8dda4 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java @@ -78,7 +78,6 @@ public class RestIndicesStatsAction extends BaseRestHandler { indicesStatsRequest.flush(metrics.contains("flush")); indicesStatsRequest.warmer(metrics.contains("warmer")); indicesStatsRequest.queryCache(metrics.contains("query_cache")); - indicesStatsRequest.percolate(metrics.contains("percolator_cache")); indicesStatsRequest.segments(metrics.contains("segments")); indicesStatsRequest.fieldData(metrics.contains("fielddata")); indicesStatsRequest.completion(metrics.contains("completion")); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index 958fa40b54b..e266efa4357 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -222,9 +222,6 @@ public class RestIndicesAction extends AbstractCatAction { table.addCell("merges.total_time", "sibling:pri;alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges"); table.addCell("pri.merges.total_time", "default:false;text-align:right;desc:time spent in merges"); - table.addCell("percolate.queries", "sibling:pri;alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries"); - table.addCell("pri.percolate.queries", "default:false;text-align:right;desc:number of registered percolation queries"); - table.addCell("refresh.total", "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes"); table.addCell("pri.refresh.total", "default:false;text-align:right;desc:total refreshes"); @@ -424,9 +421,6 @@ public class RestIndicesAction extends AbstractCatAction { table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalTime()); table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalTime()); - table.addCell(indexStats == null ? null : indexStats.getTotal().getPercolatorCache().getNumQueries()); - table.addCell(indexStats == null ? null : indexStats.getPrimaries().getPercolatorCache().getNumQueries()); - table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getTotal()); table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getTotal()); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index 15656733baf..f23f0e0a029 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -45,7 +45,6 @@ import org.elasticsearch.index.fielddata.FieldDataStats; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.percolator.PercolatorQueryCacheStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.IndexingStats; @@ -185,8 +184,6 @@ public class RestNodesAction extends AbstractCatAction { table.addCell("merges.total_size", "alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged"); table.addCell("merges.total_time", "alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges"); - table.addCell("percolate.queries", "alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries"); - table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes"); table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes"); @@ -338,9 +335,6 @@ public class RestNodesAction extends AbstractCatAction { table.addCell(mergeStats == null ? null : mergeStats.getTotalSize()); table.addCell(mergeStats == null ? null : mergeStats.getTotalTime()); - PercolatorQueryCacheStats percolatorQueryCacheStats = indicesStats == null ? null : indicesStats.getPercolate(); - table.addCell(percolatorQueryCacheStats == null ? null : percolatorQueryCacheStats.getNumQueries()); - RefreshStats refreshStats = indicesStats == null ? null : indicesStats.getRefresh(); table.addCell(refreshStats == null ? null : refreshStats.getTotal()); table.addCell(refreshStats == null ? null : refreshStats.getTotalTime()); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java index 9954bd3098b..6538b405fce 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java @@ -139,8 +139,6 @@ public class RestShardsAction extends AbstractCatAction { table.addCell("merges.total_size", "alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged"); table.addCell("merges.total_time", "alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges"); - table.addCell("percolate.queries", "alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries"); - table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes"); table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes"); @@ -278,8 +276,6 @@ public class RestShardsAction extends AbstractCatAction { table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalSize()); table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalTime()); - table.addCell(commonStats == null ? null : commonStats.getPercolatorCache().getNumQueries()); - table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotal()); table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotalTime()); diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index 96319303420..e69e4094f44 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -48,7 +48,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; @@ -216,7 +215,7 @@ public class DefaultSearchContext extends SearchContext { + "be less than [" + maxWindow + "]. This prevents allocating massive heaps for storing the results to be " + "rescored. This limit can be set by chaning the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting."); - + } } } @@ -495,11 +494,6 @@ public class DefaultSearchContext extends SearchContext { return indexService.fieldData(); } - @Override - public PercolatorQueryCache percolatorQueryCache() { - return indexService.cache().getPercolatorQueryCache(); - } - @Override public long timeoutInMillis() { return timeoutInMillis; diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 8009d0b5fe4..ac283f1f7ef 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; @@ -270,11 +269,6 @@ public abstract class FilteredSearchContext extends SearchContext { return in.fieldData(); } - @Override - public PercolatorQueryCache percolatorQueryCache() { - return in.percolatorQueryCache(); - } - @Override public long timeoutInMillis() { return in.timeoutInMillis(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 550a5f76caf..55b66c42a60 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; @@ -231,8 +230,6 @@ public abstract class SearchContext implements Releasable { public abstract IndexFieldDataService fieldData(); - public abstract PercolatorQueryCache percolatorQueryCache(); - public abstract long timeoutInMillis(); public abstract void timeoutInMillis(long timeoutInMillis); diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java index 604670f179e..8faad63c0a4 100644 --- a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java @@ -38,7 +38,6 @@ import org.junit.Before; import java.io.IOException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.percolator.PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -251,7 +250,8 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase { } private void assertQueryBuilder(BytesRef actual, QueryBuilder expected) throws IOException { - XContentParser sourceParser = QUERY_BUILDER_CONTENT_TYPE.xContent().createParser(actual.bytes, actual.offset, actual.length); + XContentParser sourceParser = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent() + .createParser(actual.bytes, actual.offset, actual.length); QueryParseContext qsc = indexService.newQueryShardContext().newParseContext(sourceParser); assertThat(qsc.parseInnerQueryBuilder(), equalTo(expected)); } diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java deleted file mode 100644 index 30986e45e4b..00000000000 --- a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java +++ /dev/null @@ -1,390 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.percolator; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.core.SimpleAnalyzer; -import org.apache.lucene.document.BinaryDocValuesField; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TieredMergePolicy; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexWarmer; -import org.elasticsearch.index.analysis.AnalysisRegistry; -import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.mapper.DocumentFieldMappers; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.internal.SourceFieldMapper; -import org.elasticsearch.index.mapper.internal.TypeFieldMapper; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.PercolateQuery; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryParser; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.WildcardQueryBuilder; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.index.warmer.ShardIndexWarmerService; -import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.IndexSettingsModule; -import org.elasticsearch.threadpool.ThreadPool; - -import java.io.IOException; -import java.util.Collections; - -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class PercolatorQueryCacheTests extends ESTestCase { - - private QueryShardContext queryShardContext; - private PercolatorQueryCache cache; - - void initialize(Object... fields) throws IOException { - Settings settings = Settings.builder() - .put("node.name", PercolatorQueryCacheTests.class.toString()) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .build(); - - IndicesQueriesRegistry indicesQueriesRegistry = new IndicesQueriesRegistry(); - QueryParser termParser = TermQueryBuilder::fromXContent; - indicesQueriesRegistry.register(termParser, TermQueryBuilder.QUERY_NAME_FIELD); - QueryParser wildcardParser = WildcardQueryBuilder::fromXContent; - indicesQueriesRegistry.register(wildcardParser, WildcardQueryBuilder.QUERY_NAME_FIELD); - QueryParser boolQueryParser = BoolQueryBuilder::fromXContent; - indicesQueriesRegistry.register(boolQueryParser, BoolQueryBuilder.QUERY_NAME_FIELD); - - Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("_index", ClusterState.UNKNOWN_UUID), indexSettings); - SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); - MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry(); - MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, - () -> queryShardContext); - mapperService.merge("type", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("type", fields).string()), - MapperService.MergeReason.MAPPING_UPDATE, false); - cache = new PercolatorQueryCache(idxSettings, () -> queryShardContext); - ClusterState state = ClusterState.builder(new ClusterName("_name")).build(); - queryShardContext = new QueryShardContext(idxSettings, null, null, mapperService, similarityService, null, - indicesQueriesRegistry, null, cache, null, state); - } - - public void testLoadQueries() throws Exception { - Directory directory = newDirectory(); - IndexWriter indexWriter = new IndexWriter( - directory, - new IndexWriterConfig(new MockAnalyzer(random())) - .setMergePolicy(NoMergePolicy.INSTANCE) - ); - - boolean legacyFormat = randomBoolean(); - Version version = legacyFormat ? Version.V_2_0_0 : Version.CURRENT; - IndexShard indexShard = mockIndexShard(version, legacyFormat); - - storeQuery("0", indexWriter, termQuery("field1", "value1"), true, legacyFormat); - storeQuery("1", indexWriter, wildcardQuery("field1", "v*"), true, legacyFormat); - storeQuery("2", indexWriter, boolQuery().must(termQuery("field1", "value1")).must(termQuery("field2", "value2")), - true, legacyFormat); - // dymmy docs should be skipped during loading: - Document doc = new Document(); - doc.add(new StringField("dummy", "value", Field.Store.YES)); - indexWriter.addDocument(doc); - storeQuery("4", indexWriter, termQuery("field2", "value2"), true, legacyFormat); - // only documents that .percolator type should be loaded: - storeQuery("5", indexWriter, termQuery("field2", "value2"), false, legacyFormat); - storeQuery("6", indexWriter, termQuery("field3", "value3"), true, legacyFormat); - indexWriter.forceMerge(1); - - // also include queries for percolator docs marked as deleted: - indexWriter.deleteDocuments(new Term("id", "6")); - indexWriter.close(); - - ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0); - IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId); - assertThat(indexReader.leaves().size(), equalTo(1)); - assertThat(indexReader.numDeletedDocs(), equalTo(1)); - assertThat(indexReader.maxDoc(), equalTo(7)); - - initialize("field1", "type=keyword", "field2", "type=keyword", "field3", "type=keyword"); - - PercolatorQueryCache.QueriesLeaf leaf = cache.loadQueries(indexReader.leaves().get(0), indexShard); - assertThat(leaf.queries.size(), equalTo(5)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("field1", "value1")))); - assertThat(leaf.getQuery(1), equalTo(new WildcardQuery(new Term("field1", "v*")))); - assertThat(leaf.getQuery(2), equalTo(new BooleanQuery.Builder() - .add(new TermQuery(new Term("field1", "value1")), BooleanClause.Occur.MUST) - .add(new TermQuery(new Term("field2", "value2")), BooleanClause.Occur.MUST) - .build() - )); - assertThat(leaf.getQuery(4), equalTo(new TermQuery(new Term("field2", "value2")))); - assertThat(leaf.getQuery(6), equalTo(new TermQuery(new Term("field3", "value3")))); - - indexReader.close(); - directory.close(); - } - - public void testGetQueries() throws Exception { - Directory directory = newDirectory(); - IndexWriter indexWriter = new IndexWriter( - directory, - new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE) - ); - - storeQuery("0", indexWriter, termQuery("a", "0"), true, false); - storeQuery("1", indexWriter, termQuery("a", "1"), true, false); - storeQuery("2", indexWriter, termQuery("a", "2"), true, false); - indexWriter.flush(); - storeQuery("3", indexWriter, termQuery("a", "3"), true, false); - storeQuery("4", indexWriter, termQuery("a", "4"), true, false); - storeQuery("5", indexWriter, termQuery("a", "5"), true, false); - indexWriter.flush(); - storeQuery("6", indexWriter, termQuery("a", "6"), true, false); - storeQuery("7", indexWriter, termQuery("a", "7"), true, false); - storeQuery("8", indexWriter, termQuery("a", "8"), true, false); - indexWriter.flush(); - indexWriter.close(); - - ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID , 0); - IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId); - assertThat(indexReader.leaves().size(), equalTo(3)); - assertThat(indexReader.maxDoc(), equalTo(9)); - - initialize("a", "type=keyword"); - - try { - cache.getQueries(indexReader.leaves().get(0)); - fail("IllegalStateException expected"); - } catch (IllegalStateException e) { - assertThat(e.getMessage(), equalTo("queries not loaded, queries should be have been preloaded during index warming...")); - } - - IndexShard indexShard = mockIndexShard(Version.CURRENT, false); - ThreadPool threadPool = mockThreadPool(); - IndexWarmer.Listener listener = cache.createListener(threadPool); - listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader))); - PercolatorQueryCacheStats stats = cache.getStats(shardId); - assertThat(stats.getNumQueries(), equalTo(9L)); - - PercolateQuery.QueryRegistry.Leaf leaf = cache.getQueries(indexReader.leaves().get(0)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0")))); - assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "1")))); - assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "2")))); - - leaf = cache.getQueries(indexReader.leaves().get(1)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "3")))); - assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "4")))); - assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "5")))); - - leaf = cache.getQueries(indexReader.leaves().get(2)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "6")))); - assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "7")))); - assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "8")))); - - indexReader.close(); - directory.close(); - } - - public void testInvalidateEntries() throws Exception { - Directory directory = newDirectory(); - IndexWriter indexWriter = new IndexWriter( - directory, - new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE) - ); - - storeQuery("0", indexWriter, termQuery("a", "0"), true, false); - indexWriter.flush(); - storeQuery("1", indexWriter, termQuery("a", "1"), true, false); - indexWriter.flush(); - storeQuery("2", indexWriter, termQuery("a", "2"), true, false); - indexWriter.flush(); - - ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0); - IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId); - assertThat(indexReader.leaves().size(), equalTo(3)); - assertThat(indexReader.maxDoc(), equalTo(3)); - - initialize("a", "type=keyword"); - - IndexShard indexShard = mockIndexShard(Version.CURRENT, false); - ThreadPool threadPool = mockThreadPool(); - IndexWarmer.Listener listener = cache.createListener(threadPool); - listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader))); - assertThat(cache.getStats(shardId).getNumQueries(), equalTo(3L)); - - PercolateQuery.QueryRegistry.Leaf leaf = cache.getQueries(indexReader.leaves().get(0)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0")))); - - leaf = cache.getQueries(indexReader.leaves().get(1)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "1")))); - - leaf = cache.getQueries(indexReader.leaves().get(2)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "2")))); - - // change merge policy, so that merges will actually happen: - indexWriter.getConfig().setMergePolicy(new TieredMergePolicy()); - indexWriter.deleteDocuments(new Term("id", "1")); - indexWriter.forceMergeDeletes(); - indexReader.close(); - indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId); - assertThat(indexReader.leaves().size(), equalTo(2)); - assertThat(indexReader.maxDoc(), equalTo(2)); - listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader))); - assertThat(cache.getStats(shardId).getNumQueries(), equalTo(2L)); - - leaf = cache.getQueries(indexReader.leaves().get(0)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0")))); - - leaf = cache.getQueries(indexReader.leaves().get(1)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "2")))); - - indexWriter.forceMerge(1); - indexReader.close(); - indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId); - assertThat(indexReader.leaves().size(), equalTo(1)); - assertThat(indexReader.maxDoc(), equalTo(2)); - listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader))); - assertThat(cache.getStats(shardId).getNumQueries(), equalTo(2L)); - - leaf = cache.getQueries(indexReader.leaves().get(0)); - assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0")))); - assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "2")))); - - indexWriter.close(); - indexReader.close(); - directory.close(); - } - - void storeQuery(String id, IndexWriter indexWriter, QueryBuilder queryBuilder, boolean typeField, boolean legacy) throws IOException { - Document doc = new Document(); - doc.add(new StringField("id", id, Field.Store.NO)); - if (typeField) { - if (legacy) { - doc.add(new StringField(TypeFieldMapper.NAME, PercolatorFieldMapper.LEGACY_TYPE_NAME, Field.Store.NO)); - } else { - doc.add(new StringField(TypeFieldMapper.NAME, "query", Field.Store.NO)); - } - } - if (legacy) { - BytesReference percolatorQuery = XContentFactory.jsonBuilder().startObject() - .field("query", queryBuilder) - .endObject().bytes(); - doc.add(new StoredField( - SourceFieldMapper.NAME, - percolatorQuery.array(), percolatorQuery.arrayOffset(), percolatorQuery.length()) - ); - } else { - BytesRef queryBuilderAsBytes = new BytesRef( - XContentFactory.contentBuilder(PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE).value(queryBuilder).bytes().toBytes() - ); - doc.add(new BinaryDocValuesField(PercolatorFieldMapper.QUERY_BUILDER_FIELD_NAME, queryBuilderAsBytes)); - } - indexWriter.addDocument(doc); - } - - IndexShard mockIndexShard(Version version, boolean legacyFormat) { - IndexShard indexShard = mock(IndexShard.class); - ShardIndexWarmerService shardIndexWarmerService = mock(ShardIndexWarmerService.class); - when(shardIndexWarmerService.logger()).thenReturn(logger); - when(indexShard.warmerService()).thenReturn(shardIndexWarmerService); - IndexSettings indexSettings = new IndexSettings( - IndexMetaData.builder("_index").settings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetaData.SETTING_VERSION_CREATED, version) - ).build(), - Settings.EMPTY - ); - when(indexShard.indexSettings()).thenReturn(indexSettings); - - PercolatorFieldMapper.PercolatorFieldType fieldType = mock(PercolatorFieldMapper.PercolatorFieldType.class); - when(fieldType.name()).thenReturn("query"); - when(fieldType.getQueryBuilderFieldName()).thenReturn(PercolatorFieldMapper.QUERY_BUILDER_FIELD_NAME); - PercolatorFieldMapper percolatorFieldMapper = mock(PercolatorFieldMapper.class); - when(percolatorFieldMapper.fieldType()).thenReturn(fieldType); - MapperService mapperService = mock(MapperService.class); - DocumentMapper documentMapper = mock(DocumentMapper.class); - if (legacyFormat) { - when(documentMapper.type()).thenReturn(PercolatorFieldMapper.LEGACY_TYPE_NAME); - when(documentMapper.typeFilter()) - .thenReturn(new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.LEGACY_TYPE_NAME))); - } else { - when(documentMapper.type()).thenReturn("query"); - when(documentMapper.typeFilter()).thenReturn(new TermQuery(new Term(TypeFieldMapper.NAME, "query"))); - } - - Analyzer analyzer = new SimpleAnalyzer(); - DocumentFieldMappers documentFieldMappers = - new DocumentFieldMappers(Collections.singleton(percolatorFieldMapper), analyzer, analyzer, analyzer); - when(documentMapper.mappers()).thenReturn(documentFieldMappers); - - when(mapperService.docMappers(false)).thenReturn(Collections.singleton(documentMapper)); - - when(indexShard.mapperService()).thenReturn(mapperService); - - return indexShard; - } - - ThreadPool mockThreadPool() { - ThreadPool threadPool = mock(ThreadPool.class); - when(threadPool.executor(anyString())).thenReturn(Runnable::run); - return threadPool; - } - -} diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java index 22bdc2c7290..b150000daf1 100644 --- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java @@ -76,7 +76,6 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import org.elasticsearch.index.query.support.QueryParsers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; @@ -190,7 +189,6 @@ public abstract class AbstractQueryTestCase> private static IndexSettings idxSettings; private static SimilarityService similarityService; private static MapperService mapperService; - private static PercolatorQueryCache percolatorQueryCache; private static BitsetFilterCache bitsetFilterCache; private static ScriptService scriptService; @@ -308,7 +306,6 @@ public abstract class AbstractQueryTestCase> } }); - percolatorQueryCache = new PercolatorQueryCache(idxSettings, () -> createShardContext()); indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class); //create some random type with some default field, those types will stick around for all of the subclasses currentTypes = new String[randomIntBetween(0, 5)]; @@ -349,7 +346,6 @@ public abstract class AbstractQueryTestCase> idxSettings = null; similarityService = null; mapperService = null; - percolatorQueryCache = null; bitsetFilterCache = null; scriptService = null; } @@ -750,7 +746,7 @@ public abstract class AbstractQueryTestCase> ClusterState state = ClusterState.builder(new ClusterName("_name")).build(); Client client = injector.getInstance(Client.class); return new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, - scriptService, indicesQueriesRegistry, client, percolatorQueryCache, null, state); + scriptService, indicesQueriesRegistry, client, null, state); } /** diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java index 690b2c03a2f..92a9369aed5 100644 --- a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.query; import com.fasterxml.jackson.core.JsonParseException; - import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ResourceNotFoundException; @@ -42,6 +41,7 @@ import org.junit.BeforeClass; import java.io.IOException; import java.util.Collections; + import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java index acb4917b421..8dde535e3cc 100644 --- a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java @@ -84,14 +84,14 @@ public class PercolateQueryTests extends ESTestCase { private Directory directory; private IndexWriter indexWriter; private Map queries; - private PercolateQuery.QueryRegistry queryRegistry; + private PercolateQuery.QueryStore queryStore; private DirectoryReader directoryReader; @Before public void init() throws Exception { directory = newDirectory(); queries = new HashMap<>(); - queryRegistry = ctx -> docId -> { + queryStore = ctx -> docId -> { try { String val = ctx.reader().document(docId).get(UidFieldMapper.NAME); return queries.get(Uid.createUid(val).id()); @@ -145,7 +145,7 @@ public class PercolateQueryTests extends ESTestCase { PercolateQuery.Builder builder = new PercolateQuery.Builder( "docType", - queryRegistry, + queryStore, new BytesArray("{}"), percolateSearcher ); @@ -219,7 +219,7 @@ public class PercolateQueryTests extends ESTestCase { PercolateQuery.Builder builder = new PercolateQuery.Builder( "docType", - queryRegistry, + queryStore, new BytesArray("{}"), percolateSearcher ); @@ -336,7 +336,7 @@ public class PercolateQueryTests extends ESTestCase { IndexSearcher percolateSearcher = memoryIndex.createSearcher(); PercolateQuery.Builder builder1 = new PercolateQuery.Builder( "docType", - queryRegistry, + queryStore, new BytesArray("{}"), percolateSearcher ); @@ -346,7 +346,7 @@ public class PercolateQueryTests extends ESTestCase { PercolateQuery.Builder builder2 = new PercolateQuery.Builder( "docType", - queryRegistry, + queryStore, new BytesArray("{}"), percolateSearcher ); diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index b6d69374661..9a4c9aece5c 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -47,7 +47,7 @@ public class QueryShardContextTests extends ESTestCase { MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); QueryShardContext context = new QueryShardContext( - indexSettings, null, null, mapperService, null, null, null, null, null, null, null + indexSettings, null, null, mapperService, null, null, null, null, null, null ); context.setAllowUnmappedFields(false); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 52c16dd74d9..f96b546855b 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -644,7 +644,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService(resolveIndex("test")); IndexShard shard = test.getShardOrNull(0); - ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), test.cache().getPercolatorQueryCache(), shard, new CommonStatsFlags()), shard.commitStats()); + ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), shard, new CommonStatsFlags()), shard.commitStats()); assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath()); assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath()); assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath()); diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 4716e7dba78..92c3260aeb0 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -670,7 +670,7 @@ public class IndexStatsIT extends ESIntegTestCase { public void testFlagOrdinalOrder() { Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh, - Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.PercolatorCache, Flag.Completion, Flag.Segments, + Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Completion, Flag.Segments, Flag.Translog, Flag.Suggest, Flag.RequestCache, Flag.Recovery}; assertThat(flags.length, equalTo(Flag.values().length)); @@ -913,9 +913,6 @@ public class IndexStatsIT extends ESIntegTestCase { case Warmer: builder.setWarmer(set); break; - case PercolatorCache: - builder.setPercolate(set); - break; case Completion: builder.setCompletion(set); break; @@ -963,8 +960,6 @@ public class IndexStatsIT extends ESIntegTestCase { return response.getStore() != null; case Warmer: return response.getWarmer() != null; - case PercolatorCache: - return response.getPercolatorCache() != null; case Completion: return response.getCompletion() != null; case Segments: diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java index 77b67227401..c489025a943 100644 --- a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java +++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java @@ -514,38 +514,6 @@ public class PercolatorIT extends ESIntegTestCase { assertThat(percolate.getMatches(), emptyArray()); } - public void testPercolateStatistics() throws Exception { - client().admin().indices().prepareCreate(INDEX_NAME) - .addMapping(TYPE_NAME, "query", "type=percolator") - .get(); - client().admin().indices().prepareCreate("test2") - .addMapping(TYPE_NAME, "query", "type=percolator") - .get(); - ensureGreen(); - - logger.info("--> register a query"); - client().prepareIndex(INDEX_NAME, TYPE_NAME, "1") - .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) - .execute().actionGet(); - client().prepareIndex("test2", TYPE_NAME, "1") - .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) - .execute().actionGet(); - refresh(); - - logger.info("--> First percolate request"); - PercolateResponse response = client().preparePercolate() - .setIndices(INDEX_NAME).setDocumentType("type") - .setSource(jsonBuilder().startObject().startObject("doc").field("field", "val").endObject().endObject()) - .execute().actionGet(); - assertMatchCount(response, 1L); - assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContaining("1")); - - NumShards numShards = getNumShards(INDEX_NAME); - - IndicesStatsResponse indicesResponse = client().admin().indices().prepareStats(INDEX_NAME).execute().actionGet(); - assertThat(indicesResponse.getTotal().getPercolatorCache().getNumQueries(), equalTo((long)numShards.dataCopies)); // number of copies - } - public void testPercolatingExistingDocs() throws Exception { client().admin().indices().prepareCreate(INDEX_NAME) .addMapping(TYPE_NAME, "query", "type=percolator") diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java index d5c6bf98835..9ebbb5b42e0 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java @@ -289,7 +289,7 @@ public class HighlightBuilderTests extends ESTestCase { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry, - null, null, null, null) { + null, null, null) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java index d0f2c0492da..5cc59046433 100644 --- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java @@ -158,7 +158,7 @@ public class QueryRescoreBuilderTests extends ESTestCase { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry, - null, null, null, null) { + null, null, null) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 21b71508bbb..e964b975bb2 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -232,7 +232,7 @@ public abstract class AbstractSortTestCase> extends EST } }); return new QueryShardContext(idxSettings, bitsetFilterCache, ifds, null, null, scriptService, - indicesQueriesRegistry, null, null, null, null) { + indicesQueriesRegistry, null, null, null) { @Override public MappedFieldType fieldMapper(String name) { return provideMappedFieldType(name); diff --git a/docs/reference/migration/migrate_5_0/percolator.asciidoc b/docs/reference/migration/migrate_5_0/percolator.asciidoc index 7a3fa97fe92..d3b1ccb97a3 100644 --- a/docs/reference/migration/migrate_5_0/percolator.asciidoc +++ b/docs/reference/migration/migrate_5_0/percolator.asciidoc @@ -46,4 +46,4 @@ the existing document. ==== Percolate Stats -Percolate stats have been replaced with `percolate` query cache stats in nodes stats and cluster stats APIs. \ No newline at end of file +The percolate stats have been removed. This is because the percolator no longer caches the percolator queries. \ No newline at end of file diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index 267647c9dbb..48ba500c98d 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -327,13 +327,10 @@ the document defined in the `percolate` query. ==== How it Works Under the Hood When indexing a document into an index that has the <> mapping configured, the query -part of the documents gets parsed into a Lucene query and is kept in memory until that percolator document is removed. -So, all the active percolator queries are kept in memory. +part of the documents gets parsed into a Lucene query and are stored into the Lucene index. A binary representation +of the query gets stored, but also the query's terms are analyzed and stored into an indexed field. At search time, the document specified in the request gets parsed into a Lucene document and is stored in a in-memory -temporary Lucene index. This in-memory index can just hold this one document and it is optimized for that. Then all the queries -that are registered to the index that the search request is targeted for, are going to be executed on this single document -in-memory index. This happens on each shard the search request needs to execute. - -By using `routing` or additional queries the amount of percolator queries that need to be executed can be reduced and thus -the time the search API needs to run can be decreased. \ No newline at end of file +temporary Lucene index. This in-memory index can just hold this one document and it is optimized for that. After this +a special query is build based on the terms in the in-memory index that select candidate percolator queries based on +their indexed query terms. These queries are then evaluated by the in-memory index if they actually match. \ No newline at end of file diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java index 139d8dbf0b8..1bdb87778f5 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java @@ -153,7 +153,7 @@ public class TemplateQueryParserTests extends ESTestCase { }); IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class); contextFactory = () -> new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, - similarityService, scriptService, indicesQueriesRegistry, proxy, null, null, null); + similarityService, scriptService, indicesQueriesRegistry, proxy, null, null); } @Override diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml index dfafd833509..97ffae1f802 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml @@ -48,7 +48,6 @@ merges.total_docs .+ \n merges.total_size .+ \n merges.total_time .+ \n - percolate.queries .+ \n refresh.total .+ \n refresh.time .+ \n search.fetch_current .+ \n diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 5cac904df35..d8ea1f60bbd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; -import org.elasticsearch.index.percolator.PercolatorQueryCache; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; @@ -73,7 +72,6 @@ public class TestSearchContext extends SearchContext { final IndexService indexService; final IndexFieldDataService indexFieldDataService; final BitsetFilterCache fixedBitSetFilterCache; - final PercolatorQueryCache percolatorQueryCache; final ThreadPool threadPool; final Map, Collector> queryCollectors = new HashMap<>(); final IndexShard indexShard; @@ -101,7 +99,6 @@ public class TestSearchContext extends SearchContext { this.indexService = indexService; this.indexFieldDataService = indexService.fieldData(); this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); - this.percolatorQueryCache = indexService.cache().getPercolatorQueryCache(); this.threadPool = threadPool; this.indexShard = indexService.getShardOrNull(0); this.scriptService = scriptService; @@ -116,7 +113,6 @@ public class TestSearchContext extends SearchContext { this.indexFieldDataService = null; this.threadPool = null; this.fixedBitSetFilterCache = null; - this.percolatorQueryCache = null; this.indexShard = null; scriptService = null; this.queryShardContext = queryShardContext; @@ -323,11 +319,6 @@ public class TestSearchContext extends SearchContext { return indexFieldDataService; } - @Override - public PercolatorQueryCache percolatorQueryCache() { - return percolatorQueryCache; - } - @Override public long timeoutInMillis() { return 0; diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java index ee567aae779..d6cd3eea5ac 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java @@ -34,7 +34,7 @@ import org.elasticsearch.test.TestSearchContext; public class MockSearchServiceTests extends ESTestCase { public void testAssertNoInFlightContext() { SearchContext s = new TestSearchContext(new QueryShardContext(new IndexSettings(IndexMetaData.PROTO, Settings.EMPTY), null, null, - null, null, null, null, null, null, null, null)) { + null, null, null, null, null, null, null)) { @Override public SearchShardTarget shardTarget() { return new SearchShardTarget("node", new Index("idx", "ignored"), 0);