Merge pull request #11886 from jpountz/fix/shard_filter_cache_stats

Store filter cache statistics at the shard level instead of index.

core/src/main/java/org/elasticsearch/index/cache/IndexCache.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.cache;
 
-import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -38,14 +37,12 @@ import java.io.IOException;
 public class IndexCache extends AbstractIndexComponent implements Closeable {
 
     private final QueryCache queryCache;
-    private final QueryCachingPolicy queryCachingPolicy;
     private final BitsetFilterCache bitsetFilterCache;
 
     @Inject
-    public IndexCache(Index index, @IndexSettings Settings indexSettings, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, BitsetFilterCache bitsetFilterCache) {
+    public IndexCache(Index index, @IndexSettings Settings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache) {
         super(index, indexSettings);
         this.queryCache = queryCache;
-        this.queryCachingPolicy = queryCachingPolicy;
         this.bitsetFilterCache = bitsetFilterCache;
     }
 
@@ -53,10 +50,6 @@ public class IndexCache extends AbstractIndexComponent implements Closeable {
         return queryCache;
     }
 
-    public QueryCachingPolicy queryPolicy() {
-        return queryCachingPolicy;
-    }
-
     /**
      * Return the {@link BitsetFilterCache} for this index.
      */

core/src/main/java/org/elasticsearch/index/cache/query/QueryCacheModule.java

@@ -19,8 +19,6 @@
 
 package org.elasticsearch.index.cache.query;
 
-import org.apache.lucene.search.QueryCachingPolicy;
-import org.apache.lucene.search.UsageTrackingQueryCachingPolicy;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Scopes;
 import org.elasticsearch.common.settings.Settings;
@@ -48,13 +46,5 @@ public class QueryCacheModule extends AbstractModule {
         bind(QueryCache.class)
                 .to(settings.getAsClass(QueryCacheSettings.QUERY_CACHE_TYPE, IndexQueryCache.class, "org.elasticsearch.index.cache.query.", "QueryCache"))
                 .in(Scopes.SINGLETON);
-        // the query cache is a node-level thing, however we want the most popular queries
-        // to be computed on a per-index basis, that is why we don't use the SINGLETON
-        // scope below
-        if (settings.getAsBoolean(QueryCacheSettings.QUERY_CACHE_EVERYTHING, false)) {
-            bind(QueryCachingPolicy.class).toInstance(QueryCachingPolicy.ALWAYS_CACHE);
-        } else {
-            bind(QueryCachingPolicy.class).toInstance(new UsageTrackingQueryCachingPolicy());
-        }
     }
 }

core/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -24,6 +24,8 @@ import com.google.common.base.Preconditions;
 
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.index.CheckIndex;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.UsageTrackingQueryCachingPolicy;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.ThreadInterruptedException;
@@ -59,6 +61,7 @@ import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.aliases.IndexAliasesService;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCache;
+import org.elasticsearch.index.cache.query.QueryCacheModule.QueryCacheSettings;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.cache.request.ShardRequestCache;
 import org.elasticsearch.index.codec.CodecService;
@@ -242,7 +245,15 @@ public class IndexShard extends AbstractIndexShardComponent {
         this.checkIndexOnStartup = indexSettings.get("index.shard.check_on_startup", "false");
         this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, getFromSettings(logger, indexSettings, Translog.Durabilty.REQUEST),
                 bigArrays, threadPool);
-        this.engineConfig = newEngineConfig(translogConfig);
+        final QueryCachingPolicy cachingPolicy;
+        // the query cache is a node-level thing, however we want the most popular filters
+        // to be computed on a per-shard basis
+        if (indexSettings.getAsBoolean(QueryCacheSettings.QUERY_CACHE_EVERYTHING, false)) {
+            cachingPolicy = QueryCachingPolicy.ALWAYS_CACHE;
+        } else {
+            cachingPolicy = new UsageTrackingQueryCachingPolicy();
+        }
+        this.engineConfig = newEngineConfig(translogConfig, cachingPolicy);
 
         this.indexShardOperationCounter = new IndexShardOperationCounter(logger, shardId);
 
@@ -1336,7 +1347,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         return mapperService.documentMapperWithAutoCreate(type);
     }
 
-    private final EngineConfig newEngineConfig(TranslogConfig translogConfig) {
+    private final EngineConfig newEngineConfig(TranslogConfig translogConfig, QueryCachingPolicy cachingPolicy) {
         final TranslogRecoveryPerformer translogRecoveryPerformer = new TranslogRecoveryPerformer(shardId, mapperService, queryParserService, indexAliasesService, indexCache) {
             @Override
             protected void operationProcessed() {
@@ -1346,7 +1357,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         };
         return new EngineConfig(shardId,
                 threadPool, indexingService, indexSettingsService.indexSettings(), warmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig,
-                mapperService.indexAnalyzer(), similarityService.similarity(), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), indexCache.queryPolicy(), translogConfig);
+                mapperService.indexAnalyzer(), similarityService.similarity(), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig);
     }
 
     private static class IndexShardOperationCounter extends AbstractRefCounted {

core/src/main/java/org/elasticsearch/search/SearchService.java

@@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableMap;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
@@ -373,8 +374,9 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
         contextProcessing(context);
         try {
             final IndexCache indexCache = context.indexShard().indexService().cache();
+            final QueryCachingPolicy cachingPolicy = context.indexShard().engine().config().getQueryCachingPolicy();
             context.searcher().dfSource(new CachedDfSource(context.searcher().getIndexReader(), request.dfs(), context.similarityService().similarity(),
-                    indexCache.query(), indexCache.queryPolicy()));
+                    indexCache.query(), cachingPolicy));
         } catch (Throwable e) {
             processFailure(context, e);
             cleanContext(context);
@@ -447,8 +449,9 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
         contextProcessing(context);
         try {
             final IndexCache indexCache = context.indexShard().indexService().cache();
+            final QueryCachingPolicy cachingPolicy = context.indexShard().engine().config().getQueryCachingPolicy();
             context.searcher().dfSource(new CachedDfSource(context.searcher().getIndexReader(), request.dfs(), context.similarityService().similarity(),
-                    indexCache.query(), indexCache.queryPolicy()));
+                    indexCache.query(), cachingPolicy));
         } catch (Throwable e) {
             freeContext(context.id());
             cleanContext(context);