Merge pull request elastic/elasticsearch#875 from s1monw/catchup/pull/14293

Cut over to IndexModule to register query cache

Original commit: elastic/x-pack-elasticsearch@e96e6ee6ec

marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java

@@ -16,8 +16,8 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.cache.IndexCacheModule;
 import org.elasticsearch.license.plugin.LicensePlugin;
 import org.elasticsearch.marvel.MarvelPlugin;
 import org.elasticsearch.marvel.agent.AgentService;
@@ -414,7 +414,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
                         .put("shield.audit.enabled", auditLogsEnabled)
                                 // Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the
                                 // validation in ShieldPlugin fail. Shield can only run with this query cache impl
-                        .put(IndexCacheModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE);
+                        .put(IndexModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE);
             } catch (IOException ex) {
                 throw new RuntimeException("failed to build settings for shield", ex);
             }

shield/src/main/java/org/elasticsearch/index/ProtectedServiceInstaller.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.index;
+
+import org.elasticsearch.shield.ShieldPlugin;
+import org.elasticsearch.shield.authz.accesscontrol.OptOutQueryCache;
+import org.elasticsearch.shield.authz.accesscontrol.ShieldIndexSearcherWrapper;
+
+/**
+ * This class installs package protected extension points on the {@link IndexModule}
+ */
+public class ProtectedServiceInstaller {
+
+    public static void install(IndexModule module, boolean clientMode) {
+        module.indexSearcherWrapper = ShieldIndexSearcherWrapper.class;
+        if (clientMode == false) {
+            module.registerQueryCache(ShieldPlugin.OPT_OUT_QUERY_CACHE, OptOutQueryCache::new);
+        }
+    }
+
+}

shield/src/main/java/org/elasticsearch/index/SearcherWrapperInstaller.java

@@ -1,16 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.index;
-
-import org.elasticsearch.shield.authz.accesscontrol.ShieldIndexSearcherWrapper;
-
-public class SearcherWrapperInstaller {
-
-    public static void install(IndexModule module) {
-        module.indexSearcherWrapper = ShieldIndexSearcherWrapper.class;
-    }
-
-}

shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java

@@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.http.HttpServerModule;
 import org.elasticsearch.index.IndexModule;
-import org.elasticsearch.index.cache.IndexCacheModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestModule;
 import org.elasticsearch.shield.action.ShieldActionFilter;
@@ -30,7 +29,7 @@ import org.elasticsearch.shield.authc.Realms;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.authz.AuthorizationModule;
-import org.elasticsearch.index.SearcherWrapperInstaller;
+import org.elasticsearch.index.ProtectedServiceInstaller;
 import org.elasticsearch.shield.authz.accesscontrol.OptOutQueryCache;
 import org.elasticsearch.shield.authz.store.FileRolesStore;
 import org.elasticsearch.shield.crypto.CryptoModule;
@@ -154,7 +153,7 @@ public class ShieldPlugin extends Plugin {
         if (enabled == false) {
             return;
         }
-        SearcherWrapperInstaller.install(module);
+        ProtectedServiceInstaller.install(module, clientMode);
     }
 
     public void onModule(ActionModule module) {
@@ -202,12 +201,6 @@ public class ShieldPlugin extends Plugin {
         }
     }
 
-    public void onModule(IndexCacheModule module) {
-        if (enabled && clientMode == false) {
-            module.registerQueryCache(OPT_OUT_QUERY_CACHE, OptOutQueryCache.class);
-        }
-    }
-
     private void addUserSettings(Settings.Builder settingsBuilder) {
         String authHeaderSettingName = Headers.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER;
         if (settings.get(authHeaderSettingName) != null) {
@@ -273,7 +266,7 @@ public class ShieldPlugin extends Plugin {
         unauthorized users.
      */
     private void addQueryCacheSettings(Settings.Builder settingsBuilder) {
-        settingsBuilder.put(IndexCacheModule.QUERY_CACHE_TYPE, OPT_OUT_QUERY_CACHE);
+        settingsBuilder.put(IndexModule.QUERY_CACHE_TYPE, OPT_OUT_QUERY_CACHE);
     }
 
     private static boolean isShieldMandatory(String[] existingMandatoryPlugins) {
@@ -307,12 +300,12 @@ public class ShieldPlugin extends Plugin {
             // in case this are node settings then the plugin additional settings have not been applied yet,
             // so we use 'opt_out_cache' as default. So in that case we only fail if the node settings contain
             // another cache impl than 'opt_out_cache'.
-            queryCacheImplementation = settings.get(IndexCacheModule.QUERY_CACHE_TYPE, OPT_OUT_QUERY_CACHE);
+            queryCacheImplementation = settings.get(IndexModule.QUERY_CACHE_TYPE, OPT_OUT_QUERY_CACHE);
         } else {
-            queryCacheImplementation = settings.get(IndexCacheModule.QUERY_CACHE_TYPE);
+            queryCacheImplementation = settings.get(IndexModule.QUERY_CACHE_TYPE);
         }
         if (OPT_OUT_QUERY_CACHE.equals(queryCacheImplementation) == false) {
-            throw new IllegalStateException("shield does not support a user specified query cache. remove the setting [" + IndexCacheModule.QUERY_CACHE_TYPE + "] with value [" + queryCacheImplementation + "]");
+            throw new IllegalStateException("shield does not support a user specified query cache. remove the setting [" + IndexModule.QUERY_CACHE_TYPE + "] with value [" + queryCacheImplementation + "]");
         }
     }
 }

shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/OptOutQueryCache.java

@@ -9,9 +9,7 @@ import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.AbstractIndexComponent;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.indices.cache.query.IndicesQueryCache;
@@ -25,7 +23,6 @@ public final class OptOutQueryCache extends AbstractIndexComponent implements Qu
 
     final IndicesQueryCache indicesQueryCache;
 
-    @Inject
     public OptOutQueryCache(IndexSettings indexSettings, IndicesQueryCache indicesQueryCache) {
         super(indexSettings);
         this.indicesQueryCache = indicesQueryCache;

shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java

@@ -7,7 +7,7 @@ package org.elasticsearch.integration;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.cache.IndexCacheModule;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegTestCase;
@@ -95,7 +95,7 @@ public class DocumentAndFieldLevelSecurityTests extends ShieldIntegTestCase {
 
     public void testQueryCache() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
-                        .setSettings(Settings.builder().put(IndexCacheModule.QUERY_CACHE_EVERYTHING, true))
+                        .setSettings(Settings.builder().put(IndexModule.QUERY_CACHE_EVERYTHING, true))
                         .addMapping("type1", "field1", "type=string", "field2", "type=string")
         );
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1")

shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.integration;
 
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStatsResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetResponse;
@@ -16,9 +15,8 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.cache.IndexCacheModule;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.indices.cache.request.IndicesRequestCache;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
@@ -357,7 +355,7 @@ public class FieldLevelSecurityTests extends ShieldIntegTestCase {
 
     public void testQueryCache() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
-                        .setSettings(Settings.builder().put(IndexCacheModule.QUERY_CACHE_EVERYTHING, true))
+                        .setSettings(Settings.builder().put(IndexModule.QUERY_CACHE_EVERYTHING, true))
                         .addMapping("type1", "field1", "type=string", "field2", "type=string")
         );
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2")

shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java

@@ -17,8 +17,8 @@ import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.*;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.cache.IndexCacheModule;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.shield.ShieldPlugin;
@@ -149,7 +149,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
                         // For tests we forcefully configure Shield's custom query cache because the test framework randomizes the query cache impl,
                         // but if shield is disabled then we don't need to forcefully set the query cache
                         if (useShield == false) {
-                            builder.remove(IndexCacheModule.QUERY_CACHE_TYPE);
+                            builder.remove(IndexModule.QUERY_CACHE_TYPE);
                         }
                         return builder.build();
                     }

shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java

@@ -17,12 +17,19 @@ import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.indices.IndicesWarmer;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
 import org.junit.After;
 import org.junit.Before;
 import org.mockito.Matchers;
 
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
@@ -37,18 +44,8 @@ public class DocumentSubsetReaderTests extends ESTestCase {
     @Before
     public void before() {
         directory = newDirectory();
-        bitsetFilterCache = mock(BitsetFilterCache.class);
+        IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST);
-        when(bitsetFilterCache.getBitSetProducer(Matchers.any(Query.class))).then(invocationOnMock -> {
+        bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null));
-            final Query query = (Query) invocationOnMock.getArguments()[0];
-            return (BitSetProducer) context -> {
-                IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context);
-                IndexSearcher searcher = new IndexSearcher(topLevelContext);
-                searcher.setQueryCache(null);
-                Weight weight = searcher.createNormalizedWeight(query, false);
-                DocIdSetIterator it = weight.scorer(context);
-                return BitSet.of(it, context.reader().maxDoc());
-            };
-        });
     }
 
     @After
@@ -57,6 +54,7 @@ public class DocumentSubsetReaderTests extends ESTestCase {
             directoryReader.close();
         }
         directory.close();
+        bitsetFilterCache.close();
     }
 
     public void testSearch() throws Exception {
@@ -151,8 +149,8 @@ public class DocumentSubsetReaderTests extends ESTestCase {
         IndexWriterConfig iwc = new IndexWriterConfig(null);
         IndexWriter iw = new IndexWriter(dir, iwc);
         iw.close();
-        BitsetFilterCache bitsetFilterCache = mock(BitsetFilterCache.class);
+        IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST);
-
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null));
         DirectoryReader directoryReader = DocumentSubsetReader.wrap(DirectoryReader.open(dir), bitsetFilterCache, new MatchAllDocsQuery());
         try {
             DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, new MatchAllDocsQuery());
@@ -161,6 +159,7 @@ public class DocumentSubsetReaderTests extends ESTestCase {
             assertThat(e.getMessage(), equalTo("Can't wrap [class org.elasticsearch.shield.authz.accesscontrol.DocumentSubsetReader$DocumentSubsetDirectoryReader] twice"));
         }
 
+        bitsetFilterCache.close();
         directoryReader.close();
         dir.close();
     }

shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java

@@ -32,12 +32,15 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.IndexQueryParserService;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesWarmer;
 import org.elasticsearch.shield.authz.InternalAuthorizationService;
 import org.elasticsearch.shield.license.ShieldLicenseState;
 import org.elasticsearch.test.ESTestCase;
@@ -79,28 +82,8 @@ public class ShieldIndexSearcherWrapperIntegrationTests extends ESTestCase {
         IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, null, singleton(new BytesArray("{}")));
         request.putInContext(InternalAuthorizationService.INDICES_PERMISSIONS_KEY, new IndicesAccessControl(true, singletonMap("_index", indexAccessControl)));
         IndexQueryParserService parserService = mock(IndexQueryParserService.class);
-
+        IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST);
-        BitsetFilterCache bitsetFilterCache = mock(BitsetFilterCache.class);
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null));
-        when(bitsetFilterCache.getBitSetProducer(Matchers.any(Query.class))).then(new Answer<BitSetProducer>() {
-            @Override
-            public BitSetProducer answer(InvocationOnMock invocationOnMock) throws Throwable {
-                final Query query = (Query) invocationOnMock.getArguments()[0];
-                return context -> {
-                    IndexSearcher searcher = new IndexSearcher(context);
-                    searcher.setQueryCache(null);
-                    Weight weight = searcher.createNormalizedWeight(query, false);
-                    DocIdSetIterator it = weight.scorer(context);
-                    if (it != null) {
-                        int maxDoc = context.reader().maxDoc();
-                        BitSet bitSet = randomBoolean() ? new SparseFixedBitSet(maxDoc) : new FixedBitSet(maxDoc);
-                        bitSet.or(it);
-                        return bitSet;
-                    } else {
-                        return null;
-                    }
-                };
-            }
-        });
         ShieldLicenseState licenseState = mock(ShieldLicenseState.class);
         when(licenseState.documentAndFieldLevelSecurityEnabled()).thenReturn(true);
         ShieldIndexSearcherWrapper wrapper = new ShieldIndexSearcherWrapper(
@@ -164,6 +147,7 @@ public class ShieldIndexSearcherWrapperIntegrationTests extends ESTestCase {
             assertThat(wrappedDirectoryReader.numDocs(), equalTo(expectedHitCount));
         }
 
+        bitsetFilterCache.close();
         directoryReader.close();
         directory.close();
     }

shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java

@@ -44,6 +44,7 @@ import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.indices.IndicesWarmer;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.shield.authz.InternalAuthorizationService;
@@ -246,12 +247,14 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase {
         ShardId shardId = new ShardId("_index", 0);
         EngineConfig engineConfig = new EngineConfig(shardId, null, null, Settings.EMPTY, null, null, null, null, null, null, new BM25Similarity(), null, null, null, new NoneQueryCache(IndexSettingsModule.newIndexSettings(shardId.index(), Settings.EMPTY, Collections.EMPTY_LIST)), QueryCachingPolicy.ALWAYS_CACHE, null, TimeValue.timeValueMinutes(5)); // can't mock...
 
-        BitsetFilterCache bitsetFilterCache = mock(BitsetFilterCache.class);
+        IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST);
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null));
         DirectoryReader directoryReader = DocumentSubsetReader.wrap(esIn, bitsetFilterCache, new MatchAllDocsQuery());
         IndexSearcher indexSearcher = new IndexSearcher(directoryReader);
         IndexSearcher result = shieldIndexSearcherWrapper.wrap(engineConfig, indexSearcher);
         assertThat(result, not(sameInstance(indexSearcher)));
         assertThat(result.getSimilarity(true), sameInstance(engineConfig.getSimilarity()));
+        bitsetFilterCache.close();
     }
 
     public void testIntersectScorerAndRoleBits() throws Exception {

shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java

@@ -9,7 +9,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.support.Headers;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.cache.IndexCacheModule;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.license.plugin.LicensePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.ShieldPlugin;
@@ -125,7 +125,7 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ
                 .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", configRoles()))
                 // Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the
                 // validation in ShieldPlugin fail.
-                .put(IndexCacheModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
+                .put(IndexModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
                 .put(getNodeSSLSettings());
 
         setUser(builder, nodeClientUsername(), nodeClientPassword());

watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java

@@ -18,7 +18,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.index.cache.IndexCacheModule;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.license.plugin.LicensePlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -644,7 +644,7 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase
                         .put("shield.audit.enabled", auditLogsEnabled)
                         // Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the
                         // validation in ShieldPlugin fail. Shield can only run with this query cache impl
-                        .put(IndexCacheModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
+                        .put(IndexModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
                         .build();
             } catch (IOException ex) {
                 throw new RuntimeException("failed to build settings for shield", ex);