honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Warmer (search) to support query cache 

allow for search based warmer to support query cache flag on the search request, and use the index level query caching flag if set.
closes #7326
This commit is contained in:
Shay Banon 2014-08-18 17:01:12 -07:00
parent f7ae4d9d86
commit 3a52296358
13 changed files with 185 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/reference/indices/warmers.asciidoc
View File

@ -66,6 +66,12 @@ curl -XPUT localhost:9200/_template/template_1 -d '
}'
--------------------------------------------------
coming[1.4.0]
On the same level as `types` and `source`, the `query_cache` flag is supported
to enable query caching for the warmed search request. If not specified, it will
use the index level configuration of query caching.
[float]
[[warmer-adding]]
=== Put Warmer
@ -136,6 +142,11 @@ where
Instead of `_warmer` you can also use the plural `_warmers`.
coming[1.4.0]
The `query_cache` parameter can be used to enable query caching for
the search request. If not specified, it will use the index level configuration
of query caching.
[float]

19
src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java
View File

@ -21,7 +21,9 @@ package org.elasticsearch.action.admin.indices.warmer.get;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -60,10 +62,18 @@ public class GetWarmersResponse extends ActionResponse {
int valueSize = in.readVInt();
ImmutableList.Builder<IndexWarmersMetaData.Entry> warmerEntryBuilder = ImmutableList.builder();
for (int j = 0; j < valueSize; j++) {
String name = in.readString();
String[] types = in.readStringArray();
BytesReference source = in.readBytesReference();
Boolean queryCache = null;
if (in.getVersion().onOrAfter(Version.V_1_4_0)) {
queryCache = in.readOptionalBoolean();
}
warmerEntryBuilder.add(new IndexWarmersMetaData.Entry(
in.readString(),
in.readStringArray(),
in.readBytesReference())
name,
types,
queryCache,
source)
);
}
indexMapBuilder.put(key, warmerEntryBuilder.build());
@ -82,6 +92,9 @@ public class GetWarmersResponse extends ActionResponse {
out.writeString(warmerEntry.name());
out.writeStringArray(warmerEntry.types());
out.writeBytesReference(warmerEntry.source());
if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
out.writeOptionalBoolean(warmerEntry.queryCache());
}
}
}
}

6
src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java
View File

@ -127,21 +127,21 @@ public class TransportPutWarmerAction extends TransportMasterNodeOperationAction
IndexWarmersMetaData warmers = indexMetaData.custom(IndexWarmersMetaData.TYPE);
if (warmers == null) {
logger.info("[{}] putting warmer [{}]", index, request.name());
warmers = new IndexWarmersMetaData(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), source));
warmers = new IndexWarmersMetaData(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), request.searchRequest().queryCache(), source));
} else {
boolean found = false;
List<IndexWarmersMetaData.Entry> entries = new ArrayList<>(warmers.entries().size() + 1);
for (IndexWarmersMetaData.Entry entry : warmers.entries()) {
if (entry.name().equals(request.name())) {
found = true;
entries.add(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), source));
entries.add(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), request.searchRequest().queryCache(), source));
} else {
entries.add(entry);
}
}
if (!found) {
logger.info("[{}] put warmer [{}]", index, request.name());
entries.add(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), source));
entries.add(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), request.searchRequest().queryCache(), source));
} else {
logger.info("[{}] update warmer [{}]", index, request.name());
}

13
src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java
View File

@ -1387,10 +1387,10 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
public void warm(AtomicReader reader) throws IOException {
try {
assert isMergedSegment(reader);
final Engine.Searcher searcher = new SimpleSearcher("warmer", new IndexSearcher(reader));
final IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, searcher);
if (warmer != null) {
warmer.warm(context);
final Engine.Searcher searcher = new SimpleSearcher("warmer", new IndexSearcher(reader));
final IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, searcher);
warmer.warmNewReaders(context);
}
} catch (Throwable t) {
// Don't fail a merge if the warm-up failed
@ -1576,11 +1576,10 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
}
if (newSearcher != null) {
IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId,
new SimpleSearcher("warmer", newSearcher));
warmer.warm(context);
IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, new SimpleSearcher("warmer", newSearcher));
warmer.warmNewReaders(context);
}
warmer.warmTop(new IndicesWarmer.WarmerContext(shardId, searcher.getIndexReader()));
warmer.warmTopReader(new IndicesWarmer.WarmerContext(shardId, new SimpleSearcher("warmer", searcher)));
} catch (Throwable e) {
if (!closed) {
logger.warn("failed to prepare/warm", e);

2
src/main/java/org/elasticsearch/indices/cache/query/IndicesQueryCache.java vendored
View File

@ -180,7 +180,7 @@ public class IndicesQueryCache extends AbstractComponent implements RemovalListe
return false;
}
// for now, only enable it for search type count
if (request.searchType() != SearchType.COUNT) {
if (context.searchType() != SearchType.COUNT) {
return false;
}
IndexMetaData index = clusterService.state().getMetaData().index(request.index());

40
src/main/java/org/elasticsearch/indices/warmer/IndicesWarmer.java
View File

@ -49,31 +49,19 @@ public interface IndicesWarmer {
}
/** Queue tasks to warm-up the given segments and return handles that allow to wait for termination of the execution of those tasks. */
public abstract TerminationHandle warm(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool);
public abstract TerminationHandle warmNewReaders(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool);
public TerminationHandle warmTop(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) {
return TerminationHandle.NO_WAIT;
}
public abstract TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool);
}
public static class WarmerContext {
private final ShardId shardId;
private final Engine.Searcher searcher;
private final Engine.Searcher newSearcher;
private final IndexReader indexReader;
public WarmerContext(ShardId shardId, Engine.Searcher newSearcher) {
public WarmerContext(ShardId shardId, Engine.Searcher searcher) {
this.shardId = shardId;
this.newSearcher = newSearcher;
this.indexReader = null;
}
public WarmerContext(ShardId shardId, IndexReader indexReader) {
this.shardId = shardId;
this.newSearcher = null;
this.indexReader = indexReader;
this.searcher = searcher;
}
public ShardId shardId() {
@ -81,25 +69,17 @@ public interface IndicesWarmer {
}
/** Return a searcher instance that only wraps the segments to warm. */
public Engine.Searcher newSearcher() {
return newSearcher;
public Engine.Searcher searcher() {
return searcher;
}
public IndexReader indexReader() {
return indexReader;
public IndexReader reader() {
return searcher.reader();
}
@Override
public String toString() {
final String value;
if (newSearcher != null) {
value = newSearcher.reader().toString();
} else if (indexReader != null) {
value = indexReader.toString();
} else {
value = "null";
}
return "WarmerContext: " + value;
return "WarmerContext: " + searcher.reader();
}
}

8
src/main/java/org/elasticsearch/indices/warmer/InternalIndicesWarmer.java
View File

@ -67,11 +67,11 @@ public class InternalIndicesWarmer extends AbstractComponent implements IndicesW
listeners.remove(listener);
}
public void warm(final WarmerContext context) {
public void warmNewReaders(final WarmerContext context) {
warmInternal(context, false);
}
public void warmTop(WarmerContext context) {
public void warmTopReader(WarmerContext context) {
warmInternal(context, true);
}
@ -104,9 +104,9 @@ public class InternalIndicesWarmer extends AbstractComponent implements IndicesW
// get a handle on pending tasks
for (final Listener listener : listeners) {
if (topReader) {
terminationHandles.add(listener.warmTop(indexShard, indexMetaData, context, threadPool));
terminationHandles.add(listener.warmTopReader(indexShard, indexMetaData, context, threadPool));
} else {
terminationHandles.add(listener.warm(indexShard, indexMetaData, context, threadPool));
terminationHandles.add(listener.warmNewReaders(indexShard, indexMetaData, context, threadPool));
}
}
// wait for termination

1
src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java
View File

@ -65,6 +65,7 @@ public class RestPutWarmerAction extends BaseRestHandler {
putWarmerRequest.listenerThreaded(false);
SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")))
.types(Strings.splitStringByCommaToArray(request.param("type")))
.queryCache(request.paramAsBoolean("query_cache", null))
.source(request.content(), request.contentUnsafe());
searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
putWarmerRequest.searchRequest(searchRequest);

55
src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -36,7 +36,6 @@ import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lease.Releasables;
@ -753,7 +752,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
static class NormsWarmer extends IndicesWarmer.Listener {
@Override
public TerminationHandle warm(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) {
public TerminationHandle warmNewReaders(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) {
final Loading defaultLoading = Loading.parse(indexMetaData.settings().get(NORMS_LOADING_KEY), Loading.LAZY);
final MapperService mapperService = indexShard.mapperService();
final ObjectSet<String> warmUp = new ObjectOpenHashSet<>();
@ -775,7 +774,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
for (Iterator<ObjectCursor<String>> it = warmUp.iterator(); it.hasNext(); ) {
final String indexName = it.next().value;
final long start = System.nanoTime();
for (final AtomicReaderContext ctx : context.newSearcher().reader().leaves()) {
for (final AtomicReaderContext ctx : context.searcher().reader().leaves()) {
final NumericDocValues values = ctx.reader().getNormValues(indexName);
if (values != null) {
values.get(0);
@ -800,12 +799,17 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
}
};
}
@Override
public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) {
return TerminationHandle.NO_WAIT;
}
}
static class FieldDataWarmer extends IndicesWarmer.Listener {
@Override
public TerminationHandle warm(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) {
public TerminationHandle warmNewReaders(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, FieldMapper<?>> warmUp = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
@ -827,8 +831,8 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
}
final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
final Executor executor = threadPool.executor(executor());
final CountDownLatch latch = new CountDownLatch(context.newSearcher().reader().leaves().size() * warmUp.size());
for (final AtomicReaderContext ctx : context.newSearcher().reader().leaves()) {
final CountDownLatch latch = new CountDownLatch(context.searcher().reader().leaves().size() * warmUp.size());
for (final AtomicReaderContext ctx : context.searcher().reader().leaves()) {
for (final FieldMapper<?> fieldMapper : warmUp.values()) {
executor.execute(new Runnable() {
@ -859,7 +863,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
}
@Override
public TerminationHandle warmTop(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) {
public TerminationHandle warmTopReader(final IndexShard indexShard, IndexMetaData indexMetaData, final WarmerContext context, ThreadPool threadPool) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, FieldMapper<?>> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
@ -888,7 +892,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
try {
final long start = System.nanoTime();
IndexOrdinalsFieldData ifd = indexFieldDataService.getForField(fieldMapper);
ifd.loadGlobal(context.indexReader());
ifd.loadGlobal(context.reader());
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard.warmerService().logger().trace("warmed global ordinals for [{}], took [{}]", fieldMapper.names().name(), TimeValue.timeValueNanos(System.nanoTime() - start));
}
@ -912,7 +916,16 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
class SearchWarmer extends IndicesWarmer.Listener {
@Override
public TerminationHandle warm(final IndexShard indexShard, final IndexMetaData indexMetaData, final IndicesWarmer.WarmerContext warmerContext, ThreadPool threadPool) {
public TerminationHandle warmNewReaders(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) {
return internalWarm(indexShard, indexMetaData, context, threadPool, false);
}
@Override
public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData indexMetaData, WarmerContext context, ThreadPool threadPool) {
return internalWarm(indexShard, indexMetaData, context, threadPool, true);
}
public TerminationHandle internalWarm(final IndexShard indexShard, final IndexMetaData indexMetaData, final IndicesWarmer.WarmerContext warmerContext, ThreadPool threadPool, final boolean top) {
IndexWarmersMetaData custom = indexMetaData.custom(IndexWarmersMetaData.TYPE);
if (custom == null) {
return TerminationHandle.NO_WAIT;
@ -928,11 +941,27 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
try {
long now = System.nanoTime();
ShardSearchRequest request = new ShardSearchRequest(indexShard.shardId().index().name(), indexShard.shardId().id(), indexMetaData.numberOfShards(),
SearchType.QUERY_THEN_FETCH /* we don't use COUNT so sorting will also kick in whatever warming logic*/)
SearchType.QUERY_THEN_FETCH)
.source(entry.source())
.types(entry.types());
context = createContext(request, warmerContext.newSearcher());
queryPhase.execute(context);
.types(entry.types())
.queryCache(entry.queryCache());
context = createContext(request, warmerContext.searcher());
// if we use sort, we need to do query to sort on it and load relevant field data
// if not, we might as well use COUNT (and cache if needed)
if (context.sort() == null) {
context.searchType(SearchType.COUNT);
}
boolean canCache = indicesQueryCache.canCache(request, context);
// early terminate when we can cache, since we can only do proper caching on top level searcher
// also, if we can't cache, and its top, we don't need to execute it, since we already did when its not top
if (canCache != top) {
return;
}
if (canCache) {
indicesQueryCache.load(request, context, queryPhase);
} else {
queryPhase.execute(context);
}
long took = System.nanoTime() - now;
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard.warmerService().logger().trace("warmed [{}], took [{}]", entry.name(), TimeValue.timeValueNanos(took));

5
src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
View File

@ -227,6 +227,11 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque
return this.queryCache;
}
public ShardSearchRequest queryCache(Boolean queryCache) {
this.queryCache = queryCache;
return this;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);

35
src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.search.warmer;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
@ -47,11 +48,13 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
private final String name;
private final String[] types;
private final BytesReference source;
private final Boolean queryCache;
public Entry(String name, String[] types, BytesReference source) {
public Entry(String name, String[] types, Boolean queryCache, BytesReference source) {
this.name = name;
this.types = types == null ? Strings.EMPTY_ARRAY : types;
this.source = source;
this.queryCache = queryCache;
}
public String name() {
@ -66,6 +69,11 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
public BytesReference source() {
return this.source;
}
@Nullable
public Boolean queryCache() {
return this.queryCache;
}
}
private final ImmutableList<Entry> entries;
@ -95,7 +103,17 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
public IndexWarmersMetaData readFrom(StreamInput in) throws IOException {
Entry[] entries = new Entry[in.readVInt()];
for (int i = 0; i < entries.length; i++) {
entries[i] = new Entry(in.readString(), in.readStringArray(), in.readBoolean() ? in.readBytesReference() : null);
String name = in.readString();
String[] types = in.readStringArray();
BytesReference source = null;
if (in.readBoolean()) {
source = in.readBytesReference();
}
Boolean queryCache = null;
if (in.getVersion().onOrAfter(Version.V_1_4_0)) {
queryCache = in.readOptionalBoolean();
}
entries[i] = new Entry(name, types, queryCache, source);
}
return new IndexWarmersMetaData(entries);
}
@ -112,6 +130,9 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
out.writeBoolean(true);
out.writeBytesReference(entry.source());
}
if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
out.writeOptionalBoolean(entry.queryCache());
}
}
}
@ -142,6 +163,7 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
String name = currentFieldName;
List<String> types = new ArrayList<>(2);
BytesReference source = null;
Boolean queryCache = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
@ -160,9 +182,13 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
if ("source".equals(currentFieldName)) {
source = new BytesArray(parser.binaryValue());
}
} else if (token.isValue()) {
if ("queryCache".equals(currentFieldName) || "query_cache".equals(currentFieldName)) {
queryCache = parser.booleanValue();
}
}
}
entries.add(new Entry(name, types.size() == 0 ? Strings.EMPTY_ARRAY : types.toArray(new String[types.size()]), source));
entries.add(new Entry(name, types.size() == 0 ? Strings.EMPTY_ARRAY : types.toArray(new String[types.size()]), queryCache, source));
}
}
return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()]));
@ -183,6 +209,9 @@ public class IndexWarmersMetaData implements IndexMetaData.Custom {
boolean binary = params.paramAsBoolean("binary", false);
builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE);
builder.field("types", entry.types());
if (entry.queryCache() != null) {
builder.field("queryCache", entry.queryCache());
}
builder.field("source");
if (binary) {
builder.value(entry.source());

4
src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationTests.java
View File

@ -606,7 +606,7 @@ public class IndicesOptionsIntegrationTests extends ElasticsearchIntegrationTest
@Test
public void testDeleteWarmer() throws Exception {
IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry(
"test1", new String[]{"typ1"}, new BytesArray("{\"query\" : { \"match_all\" : {}}}")
"test1", new String[]{"typ1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}")
);
assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry)));
ensureYellow();
@ -622,7 +622,7 @@ public class IndicesOptionsIntegrationTests extends ElasticsearchIntegrationTest
verify(client().admin().indices().prepareDeleteWarmer().setIndices("_all").setNames("test1"), true);
IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry(
"test1", new String[]{"type1"}, new BytesArray("{\"query\" : { \"match_all\" : {}}}")
"test1", new String[]{"type1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}")
);
assertAcked(prepareCreate("foo").addCustom(new IndexWarmersMetaData(entry)));
assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry)));

63
src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java
View File

@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.engine.Segment;
import org.elasticsearch.index.mapper.FieldMapper.Loading;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.cache.query.IndicesQueryCache;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.search.warmer.IndexWarmerMissingException;
import org.elasticsearch.search.warmer.IndexWarmersMetaData;
@ -279,6 +280,7 @@ public class SimpleIndicesWarmerTests extends ElasticsearchIntegrationTest {
CreateIndexRequestBuilder createIndex(String indexName, String type, String fieldName) {
return client().admin().indices().prepareCreate(indexName).setSettings(ImmutableSettings.builder().put(SINGLE_SHARD_NO_REPLICA).put(SearchService.NORMS_LOADING_KEY, Loading.EAGER_VALUE));
}
@Override
boolean isLazy() {
return false;
@ -288,27 +290,30 @@ public class SimpleIndicesWarmerTests extends ElasticsearchIntegrationTest {
@Override
CreateIndexRequestBuilder createIndex(String indexName, String type, String fieldName) throws Exception {
return client().admin().indices().prepareCreate(indexName).setSettings(ImmutableSettings.builder().put(SINGLE_SHARD_NO_REPLICA).put(SearchService.NORMS_LOADING_KEY, Loading.LAZY_VALUE)).addMapping(type, JsonXContent.contentBuilder()
.startObject()
.startObject(type)
.startObject("properties")
.startObject(fieldName)
.field("type", "string")
.startObject("norms")
.field("loading", Loading.EAGER_VALUE)
.startObject()
.startObject(type)
.startObject("properties")
.startObject(fieldName)
.field("type", "string")
.startObject("norms")
.field("loading", Loading.EAGER_VALUE)
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
);
);
}
@Override
boolean isLazy() {
return false;
}
};
private static Settings SINGLE_SHARD_NO_REPLICA = ImmutableSettings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build();
abstract CreateIndexRequestBuilder createIndex(String indexName, String type, String fieldName) throws Exception;
boolean isLazy() {
return true;
}
@ -336,4 +341,40 @@ public class SimpleIndicesWarmerTests extends ElasticsearchIntegrationTest {
}
}
public void testQueryCacheOnWarmer() {
createIndex("test");
ensureGreen();
assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(IndicesQueryCache.INDEX_CACHE_QUERY_ENABLED, false)));
logger.info("register warmer with no query cache, validate no cache is used");
assertAcked(client().admin().indices().preparePutWarmer("warmer_1")
.setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()))
.get());
client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0l));
logger.info("register warmer with query cache, validate caching happened");
assertAcked(client().admin().indices().preparePutWarmer("warmer_1")
.setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()).setQueryCache(true))
.get());
// index again, to make sure it gets refreshed
client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0l));
client().admin().indices().prepareClearCache().setQueryCache(true).get(); // clean the cache
assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0l));
logger.info("enable default query caching on the index level, and test that no flag on warmer still caches");
assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(IndicesQueryCache.INDEX_CACHE_QUERY_ENABLED, true)));
assertAcked(client().admin().indices().preparePutWarmer("warmer_1")
.setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()))
.get());
// index again, to make sure it gets refreshed
client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0l));
}
}