From 3a52296358f28ed7dd465d525e6cb3e85f055f3b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 18 Aug 2014 17:01:12 -0700 Subject: [PATCH] Warmer (search) to support query cache allow for search based warmer to support query cache flag on the search request, and use the index level query caching flag if set. closes #7326 --- docs/reference/indices/warmers.asciidoc | 11 ++++ .../warmer/get/GetWarmersResponse.java | 19 +++++- .../warmer/put/TransportPutWarmerAction.java | 6 +- .../index/engine/internal/InternalEngine.java | 13 ++-- .../cache/query/IndicesQueryCache.java | 2 +- .../indices/warmer/IndicesWarmer.java | 40 +++--------- .../indices/warmer/InternalIndicesWarmer.java | 8 +-- .../warmer/put/RestPutWarmerAction.java | 1 + .../elasticsearch/search/SearchService.java | 55 ++++++++++++---- .../search/internal/ShardSearchRequest.java | 5 ++ .../search/warmer/IndexWarmersMetaData.java | 35 ++++++++++- .../IndicesOptionsIntegrationTests.java | 4 +- .../warmer/SimpleIndicesWarmerTests.java | 63 +++++++++++++++---- 13 files changed, 185 insertions(+), 77 deletions(-) diff --git a/docs/reference/indices/warmers.asciidoc b/docs/reference/indices/warmers.asciidoc index f89e2a2210e..c1635dde4cf 100644 --- a/docs/reference/indices/warmers.asciidoc +++ b/docs/reference/indices/warmers.asciidoc @@ -66,6 +66,12 @@ curl -XPUT localhost:9200/_template/template_1 -d ' }' -------------------------------------------------- +coming[1.4.0] + +On the same level as `types` and `source`, the `query_cache` flag is supported +to enable query caching for the warmed search request. If not specified, it will +use the index level configuration of query caching. + [float] [[warmer-adding]] === Put Warmer @@ -136,6 +142,11 @@ where Instead of `_warmer` you can also use the plural `_warmers`. +coming[1.4.0] + +The `query_cache` parameter can be used to enable query caching for +the search request. If not specified, it will use the index level configuration +of query caching. [float] diff --git a/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java b/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java index 026380ac852..c8e48cc8aa5 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java @@ -21,7 +21,9 @@ package org.elasticsearch.action.admin.indices.warmer.get; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.collect.ImmutableList; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -60,10 +62,18 @@ public class GetWarmersResponse extends ActionResponse { int valueSize = in.readVInt(); ImmutableList.Builder warmerEntryBuilder = ImmutableList.builder(); for (int j = 0; j < valueSize; j++) { + String name = in.readString(); + String[] types = in.readStringArray(); + BytesReference source = in.readBytesReference(); + Boolean queryCache = null; + if (in.getVersion().onOrAfter(Version.V_1_4_0)) { + queryCache = in.readOptionalBoolean(); + } warmerEntryBuilder.add(new IndexWarmersMetaData.Entry( - in.readString(), - in.readStringArray(), - in.readBytesReference()) + name, + types, + queryCache, + source) ); } indexMapBuilder.put(key, warmerEntryBuilder.build()); @@ -82,6 +92,9 @@ public class GetWarmersResponse extends ActionResponse { out.writeString(warmerEntry.name()); out.writeStringArray(warmerEntry.types()); out.writeBytesReference(warmerEntry.source()); + if (out.getVersion().onOrAfter(Version.V_1_4_0)) { + out.writeOptionalBoolean(warmerEntry.queryCache()); + } } } } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java b/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java index 476fde660c4..5b5806f42a6 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java @@ -127,21 +127,21 @@ public class TransportPutWarmerAction extends TransportMasterNodeOperationAction IndexWarmersMetaData warmers = indexMetaData.custom(IndexWarmersMetaData.TYPE); if (warmers == null) { logger.info("[{}] putting warmer [{}]", index, request.name()); - warmers = new IndexWarmersMetaData(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), source)); + warmers = new IndexWarmersMetaData(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), request.searchRequest().queryCache(), source)); } else { boolean found = false; List entries = new ArrayList<>(warmers.entries().size() + 1); for (IndexWarmersMetaData.Entry entry : warmers.entries()) { if (entry.name().equals(request.name())) { found = true; - entries.add(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), source)); + entries.add(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), request.searchRequest().queryCache(), source)); } else { entries.add(entry); } } if (!found) { logger.info("[{}] put warmer [{}]", index, request.name()); - entries.add(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), source)); + entries.add(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), request.searchRequest().queryCache(), source)); } else { logger.info("[{}] update warmer [{}]", index, request.name()); } diff --git a/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java b/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java index 5c7159d9759..1e6d7c124b6 100644 --- a/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java +++ b/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java @@ -1387,10 +1387,10 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin public void warm(AtomicReader reader) throws IOException { try { assert isMergedSegment(reader); - final Engine.Searcher searcher = new SimpleSearcher("warmer", new IndexSearcher(reader)); - final IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, searcher); if (warmer != null) { - warmer.warm(context); + final Engine.Searcher searcher = new SimpleSearcher("warmer", new IndexSearcher(reader)); + final IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, searcher); + warmer.warmNewReaders(context); } } catch (Throwable t) { // Don't fail a merge if the warm-up failed @@ -1576,11 +1576,10 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin } if (newSearcher != null) { - IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, - new SimpleSearcher("warmer", newSearcher)); - warmer.warm(context); + IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, new SimpleSearcher("warmer", newSearcher)); + warmer.warmNewReaders(context); } - warmer.warmTop(new IndicesWarmer.WarmerContext(shardId, searcher.getIndexReader())); + warmer.warmTopReader(new IndicesWarmer.WarmerContext(shardId, new SimpleSearcher("warmer", searcher))); } catch (Throwable e) { if (!closed) { logger.warn("failed to prepare/warm", e); diff --git a/src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java b/src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java index 7d7f94a8374..a47219e312f 100644 --- a/src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java +++ b/src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java @@ -180,7 +180,7 @@ public class IndicesQueryCache extends AbstractComponent implements RemovalListe return false; } // for now, only enable it for search type count - if (request.searchType() != SearchType.COUNT) { + if (context.searchType() != SearchType.COUNT) { return false; } IndexMetaData index = clusterService.state().getMetaData().index(request.index()); diff --git a/src/main/java/org/elasticsearch/indices/warmer/IndicesWarmer.java b/src/main/java/org/elasticsearch/indices/warmer/IndicesWarmer.java index fa314049a42..db9b1c216b5 100644 --- a/src/main/java/org/elasticsearch/indices/warmer/IndicesWarmer.java +++ b/src/main/java/org/elasticsearch/indices/warmer/IndicesWarmer.java @@ -49,31 +49,19 @@ public interface IndicesWarmer { } /** Queue tasks to warm-up the given segments and return handles that allow to wait for termination of the execution of those tasks. */ - public abstract TerminationHandle warm(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool); + public abstract TerminationHandle warmNewReaders(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool); - public TerminationHandle warmTop(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) { - return TerminationHandle.NO_WAIT; - } + public abstract TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool); } public static class WarmerContext { private final ShardId shardId; + private final Engine.Searcher searcher; - private final Engine.Searcher newSearcher; - - private final IndexReader indexReader; - - public WarmerContext(ShardId shardId, Engine.Searcher newSearcher) { + public WarmerContext(ShardId shardId, Engine.Searcher searcher) { this.shardId = shardId; - this.newSearcher = newSearcher; - this.indexReader = null; - } - - public WarmerContext(ShardId shardId, IndexReader indexReader) { - this.shardId = shardId; - this.newSearcher = null; - this.indexReader = indexReader; + this.searcher = searcher; } public ShardId shardId() { @@ -81,25 +69,17 @@ public interface IndicesWarmer { } /** Return a searcher instance that only wraps the segments to warm. */ - public Engine.Searcher newSearcher() { - return newSearcher; + public Engine.Searcher searcher() { + return searcher; } - public IndexReader indexReader() { - return indexReader; + public IndexReader reader() { + return searcher.reader(); } @Override public String toString() { - final String value; - if (newSearcher != null) { - value = newSearcher.reader().toString(); - } else if (indexReader != null) { - value = indexReader.toString(); - } else { - value = "null"; - } - return "WarmerContext: " + value; + return "WarmerContext: " + searcher.reader(); } } diff --git a/src/main/java/org/elasticsearch/indices/warmer/InternalIndicesWarmer.java b/src/main/java/org/elasticsearch/indices/warmer/InternalIndicesWarmer.java index 1bd93b4e45c..df3ecb295d3 100644 --- a/src/main/java/org/elasticsearch/indices/warmer/InternalIndicesWarmer.java +++ b/src/main/java/org/elasticsearch/indices/warmer/InternalIndicesWarmer.java @@ -67,11 +67,11 @@ public class InternalIndicesWarmer extends AbstractComponent implements IndicesW listeners.remove(listener); } - public void warm(final WarmerContext context) { + public void warmNewReaders(final WarmerContext context) { warmInternal(context, false); } - public void warmTop(WarmerContext context) { + public void warmTopReader(WarmerContext context) { warmInternal(context, true); } @@ -104,9 +104,9 @@ public class InternalIndicesWarmer extends AbstractComponent implements IndicesW // get a handle on pending tasks for (final Listener listener : listeners) { if (topReader) { - terminationHandles.add(listener.warmTop(indexShard, indexMetaData, context, threadPool)); + terminationHandles.add(listener.warmTopReader(indexShard, indexMetaData, context, threadPool)); } else { - terminationHandles.add(listener.warm(indexShard, indexMetaData, context, threadPool)); + terminationHandles.add(listener.warmNewReaders(indexShard, indexMetaData, context, threadPool)); } } // wait for termination diff --git a/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java b/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java index 000dc95a283..d3d6b31c388 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java @@ -65,6 +65,7 @@ public class RestPutWarmerAction extends BaseRestHandler { putWarmerRequest.listenerThreaded(false); SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index"))) .types(Strings.splitStringByCommaToArray(request.param("type"))) + .queryCache(request.paramAsBoolean("query_cache", null)) .source(request.content(), request.contentUnsafe()); searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); putWarmerRequest.searchRequest(searchRequest); diff --git a/src/main/java/org/elasticsearch/search/SearchService.java b/src/main/java/org/elasticsearch/search/SearchService.java index 8c08fa264ea..e977d858190 100644 --- a/src/main/java/org/elasticsearch/search/SearchService.java +++ b/src/main/java/org/elasticsearch/search/SearchService.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lease.Releasables; @@ -753,7 +752,7 @@ public class SearchService extends AbstractLifecycleComponent { static class NormsWarmer extends IndicesWarmer.Listener { @Override - public TerminationHandle warm(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { + public TerminationHandle warmNewReaders(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { final Loading defaultLoading = Loading.parse(indexMetaData.settings().get(NORMS_LOADING_KEY), Loading.LAZY); final MapperService mapperService = indexShard.mapperService(); final ObjectSet warmUp = new ObjectOpenHashSet<>(); @@ -775,7 +774,7 @@ public class SearchService extends AbstractLifecycleComponent { for (Iterator> it = warmUp.iterator(); it.hasNext(); ) { final String indexName = it.next().value; final long start = System.nanoTime(); - for (final AtomicReaderContext ctx : context.newSearcher().reader().leaves()) { + for (final AtomicReaderContext ctx : context.searcher().reader().leaves()) { final NumericDocValues values = ctx.reader().getNormValues(indexName); if (values != null) { values.get(0); @@ -800,12 +799,17 @@ public class SearchService extends AbstractLifecycleComponent { } }; } + + @Override + public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) { + return TerminationHandle.NO_WAIT; + } } static class FieldDataWarmer extends IndicesWarmer.Listener { @Override - public TerminationHandle warm(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { + public TerminationHandle warmNewReaders(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { final MapperService mapperService = indexShard.mapperService(); final Map> warmUp = new HashMap<>(); for (DocumentMapper docMapper : mapperService.docMappers(false)) { @@ -827,8 +831,8 @@ public class SearchService extends AbstractLifecycleComponent { } final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService(); final Executor executor = threadPool.executor(executor()); - final CountDownLatch latch = new CountDownLatch(context.newSearcher().reader().leaves().size() * warmUp.size()); - for (final AtomicReaderContext ctx : context.newSearcher().reader().leaves()) { + final CountDownLatch latch = new CountDownLatch(context.searcher().reader().leaves().size() * warmUp.size()); + for (final AtomicReaderContext ctx : context.searcher().reader().leaves()) { for (final FieldMapper fieldMapper : warmUp.values()) { executor.execute(new Runnable() { @@ -859,7 +863,7 @@ public class SearchService extends AbstractLifecycleComponent { } @Override - public TerminationHandle warmTop(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { + public TerminationHandle warmTopReader(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) { final MapperService mapperService = indexShard.mapperService(); final Map> warmUpGlobalOrdinals = new HashMap<>(); for (DocumentMapper docMapper : mapperService.docMappers(false)) { @@ -888,7 +892,7 @@ public class SearchService extends AbstractLifecycleComponent { try { final long start = System.nanoTime(); IndexOrdinalsFieldData ifd = indexFieldDataService.getForField(fieldMapper); - ifd.loadGlobal(context.indexReader()); + ifd.loadGlobal(context.reader()); if (indexShard.warmerService().logger().isTraceEnabled()) { indexShard.warmerService().logger().trace("warmed global ordinals for [{}], took [{}]", fieldMapper.names().name(), TimeValue.timeValueNanos(System.nanoTime() - start)); } @@ -912,7 +916,16 @@ public class SearchService extends AbstractLifecycleComponent { class SearchWarmer extends IndicesWarmer.Listener { @Override - public TerminationHandle warm(final IndexShard indexShard, final IndexMetaData indexMetaData, final IndicesWarmer.WarmerContext warmerContext, ThreadPool threadPool) { + public TerminationHandle warmNewReaders(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) { + return internalWarm(indexShard, indexMetaData, context, threadPool, false); + } + + @Override + public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) { + return internalWarm(indexShard, indexMetaData, context, threadPool, true); + } + + public TerminationHandle internalWarm(final IndexShard indexShard, final IndexMetaData indexMetaData, final IndicesWarmer.WarmerContext warmerContext, ThreadPool threadPool, final boolean top) { IndexWarmersMetaData custom = indexMetaData.custom(IndexWarmersMetaData.TYPE); if (custom == null) { return TerminationHandle.NO_WAIT; @@ -928,11 +941,27 @@ public class SearchService extends AbstractLifecycleComponent { try { long now = System.nanoTime(); ShardSearchRequest request = new ShardSearchRequest(indexShard.shardId().index().name(), indexShard.shardId().id(), indexMetaData.numberOfShards(), - SearchType.QUERY_THEN_FETCH /* we don't use COUNT so sorting will also kick in whatever warming logic*/) + SearchType.QUERY_THEN_FETCH) .source(entry.source()) - .types(entry.types()); - context = createContext(request, warmerContext.newSearcher()); - queryPhase.execute(context); + .types(entry.types()) + .queryCache(entry.queryCache()); + context = createContext(request, warmerContext.searcher()); + // if we use sort, we need to do query to sort on it and load relevant field data + // if not, we might as well use COUNT (and cache if needed) + if (context.sort() == null) { + context.searchType(SearchType.COUNT); + } + boolean canCache = indicesQueryCache.canCache(request, context); + // early terminate when we can cache, since we can only do proper caching on top level searcher + // also, if we can't cache, and its top, we don't need to execute it, since we already did when its not top + if (canCache != top) { + return; + } + if (canCache) { + indicesQueryCache.load(request, context, queryPhase); + } else { + queryPhase.execute(context); + } long took = System.nanoTime() - now; if (indexShard.warmerService().logger().isTraceEnabled()) { indexShard.warmerService().logger().trace("warmed [{}], took [{}]", entry.name(), TimeValue.timeValueNanos(took)); diff --git a/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index c788ce7c65a..2f172cdecf8 100644 --- a/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -227,6 +227,11 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque return this.queryCache; } + public ShardSearchRequest queryCache(Boolean queryCache) { + this.queryCache = queryCache; + return this; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java index 87653359b4b..7757047015b 100644 --- a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java +++ b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.warmer; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -47,11 +48,13 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { private final String name; private final String[] types; private final BytesReference source; + private final Boolean queryCache; - public Entry(String name, String[] types, BytesReference source) { + public Entry(String name, String[] types, Boolean queryCache, BytesReference source) { this.name = name; this.types = types == null ? Strings.EMPTY_ARRAY : types; this.source = source; + this.queryCache = queryCache; } public String name() { @@ -66,6 +69,11 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { public BytesReference source() { return this.source; } + + @Nullable + public Boolean queryCache() { + return this.queryCache; + } } private final ImmutableList entries; @@ -95,7 +103,17 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { public IndexWarmersMetaData readFrom(StreamInput in) throws IOException { Entry[] entries = new Entry[in.readVInt()]; for (int i = 0; i < entries.length; i++) { - entries[i] = new Entry(in.readString(), in.readStringArray(), in.readBoolean() ? in.readBytesReference() : null); + String name = in.readString(); + String[] types = in.readStringArray(); + BytesReference source = null; + if (in.readBoolean()) { + source = in.readBytesReference(); + } + Boolean queryCache = null; + if (in.getVersion().onOrAfter(Version.V_1_4_0)) { + queryCache = in.readOptionalBoolean(); + } + entries[i] = new Entry(name, types, queryCache, source); } return new IndexWarmersMetaData(entries); } @@ -112,6 +130,9 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { out.writeBoolean(true); out.writeBytesReference(entry.source()); } + if (out.getVersion().onOrAfter(Version.V_1_4_0)) { + out.writeOptionalBoolean(entry.queryCache()); + } } } @@ -142,6 +163,7 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { String name = currentFieldName; List types = new ArrayList<>(2); BytesReference source = null; + Boolean queryCache = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -160,9 +182,13 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { if ("source".equals(currentFieldName)) { source = new BytesArray(parser.binaryValue()); } + } else if (token.isValue()) { + if ("queryCache".equals(currentFieldName) || "query_cache".equals(currentFieldName)) { + queryCache = parser.booleanValue(); + } } } - entries.add(new Entry(name, types.size() == 0 ? Strings.EMPTY_ARRAY : types.toArray(new String[types.size()]), source)); + entries.add(new Entry(name, types.size() == 0 ? Strings.EMPTY_ARRAY : types.toArray(new String[types.size()]), queryCache, source)); } } return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()])); @@ -183,6 +209,9 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom { boolean binary = params.paramAsBoolean("binary", false); builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE); builder.field("types", entry.types()); + if (entry.queryCache() != null) { + builder.field("queryCache", entry.queryCache()); + } builder.field("source"); if (binary) { builder.value(entry.source()); diff --git a/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationTests.java b/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationTests.java index e560658cb23..f8d1be550a1 100644 --- a/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationTests.java +++ b/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationTests.java @@ -606,7 +606,7 @@ public class IndicesOptionsIntegrationTests extends ElasticsearchIntegrationTest @Test public void testDeleteWarmer() throws Exception { IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry( - "test1", new String[]{"typ1"}, new BytesArray("{\"query\" : { \"match_all\" : {}}}") + "test1", new String[]{"typ1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}") ); assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry))); ensureYellow(); @@ -622,7 +622,7 @@ public class IndicesOptionsIntegrationTests extends ElasticsearchIntegrationTest verify(client().admin().indices().prepareDeleteWarmer().setIndices("_all").setNames("test1"), true); IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry( - "test1", new String[]{"type1"}, new BytesArray("{\"query\" : { \"match_all\" : {}}}") + "test1", new String[]{"type1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}") ); assertAcked(prepareCreate("foo").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry))); diff --git a/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java b/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java index 084495e36e2..deb955ed51c 100644 --- a/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java +++ b/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.engine.Segment; import org.elasticsearch.index.mapper.FieldMapper.Loading; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.warmer.IndexWarmerMissingException; import org.elasticsearch.search.warmer.IndexWarmersMetaData; @@ -279,6 +280,7 @@ public class SimpleIndicesWarmerTests extends ElasticsearchIntegrationTest { CreateIndexRequestBuilder createIndex(String indexName, String type, String fieldName) { return client().admin().indices().prepareCreate(indexName).setSettings(ImmutableSettings.builder().put(SINGLE_SHARD_NO_REPLICA).put(SearchService.NORMS_LOADING_KEY, Loading.EAGER_VALUE)); } + @Override boolean isLazy() { return false; @@ -288,27 +290,30 @@ public class SimpleIndicesWarmerTests extends ElasticsearchIntegrationTest { @Override CreateIndexRequestBuilder createIndex(String indexName, String type, String fieldName) throws Exception { return client().admin().indices().prepareCreate(indexName).setSettings(ImmutableSettings.builder().put(SINGLE_SHARD_NO_REPLICA).put(SearchService.NORMS_LOADING_KEY, Loading.LAZY_VALUE)).addMapping(type, JsonXContent.contentBuilder() - .startObject() - .startObject(type) - .startObject("properties") - .startObject(fieldName) - .field("type", "string") - .startObject("norms") - .field("loading", Loading.EAGER_VALUE) + .startObject() + .startObject(type) + .startObject("properties") + .startObject(fieldName) + .field("type", "string") + .startObject("norms") + .field("loading", Loading.EAGER_VALUE) + .endObject() + .endObject() + .endObject() .endObject() .endObject() - .endObject() - .endObject() - .endObject() - ); + ); } + @Override boolean isLazy() { return false; } }; private static Settings SINGLE_SHARD_NO_REPLICA = ImmutableSettings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build(); + abstract CreateIndexRequestBuilder createIndex(String indexName, String type, String fieldName) throws Exception; + boolean isLazy() { return true; } @@ -336,4 +341,40 @@ public class SimpleIndicesWarmerTests extends ElasticsearchIntegrationTest { } } + public void testQueryCacheOnWarmer() { + createIndex("test"); + ensureGreen(); + + assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(IndicesQueryCache.INDEX_CACHE_QUERY_ENABLED, false))); + logger.info("register warmer with no query cache, validate no cache is used"); + assertAcked(client().admin().indices().preparePutWarmer("warmer_1") + .setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery())) + .get()); + + client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0l)); + + logger.info("register warmer with query cache, validate caching happened"); + assertAcked(client().admin().indices().preparePutWarmer("warmer_1") + .setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()).setQueryCache(true)) + .get()); + + // index again, to make sure it gets refreshed + client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0l)); + + client().admin().indices().prepareClearCache().setQueryCache(true).get(); // clean the cache + assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0l)); + + logger.info("enable default query caching on the index level, and test that no flag on warmer still caches"); + assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(IndicesQueryCache.INDEX_CACHE_QUERY_ENABLED, true))); + + assertAcked(client().admin().indices().preparePutWarmer("warmer_1") + .setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery())) + .get()); + + // index again, to make sure it gets refreshed + client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0l)); + } }