honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request elastic/elasticsearch#875 from s1monw/catchup/pull/14293 

Cut over to IndexModule to register query cache

Original commit: elastic/x-pack-elasticsearch@e96e6ee6ec
This commit is contained in:
Simon Willnauer 2015-10-27 12:30:41 +01:00
parent f3f5381bdc 6b742a0d60
commit 90335855cb
13 changed files with 65 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java
View File

@ -16,8 +16,8 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.CountDown;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.license.plugin.LicensePlugin;
import org.elasticsearch.marvel.MarvelPlugin;
import org.elasticsearch.marvel.agent.AgentService;
@ -414,7 +414,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
.put("shield.audit.enabled", auditLogsEnabled)
// Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the
// validation in ShieldPlugin fail. Shield can only run with this query cache impl
.put(IndexCacheModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE);
.put(IndexModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE);
} catch (IOException ex) {
throw new RuntimeException("failed to build settings for shield", ex);
}

24
shield/src/main/java/org/elasticsearch/index/ProtectedServiceInstaller.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.index;
import org.elasticsearch.shield.ShieldPlugin;
import org.elasticsearch.shield.authz.accesscontrol.OptOutQueryCache;
import org.elasticsearch.shield.authz.accesscontrol.ShieldIndexSearcherWrapper;
/**
* This class installs package protected extension points on the {@link IndexModule}
*/
public class ProtectedServiceInstaller {
public static void install(IndexModule module, boolean clientMode) {
module.indexSearcherWrapper = ShieldIndexSearcherWrapper.class;
if (clientMode == false) {
module.registerQueryCache(ShieldPlugin.OPT_OUT_QUERY_CACHE, OptOutQueryCache::new);
}
}
}

16
shield/src/main/java/org/elasticsearch/index/SearcherWrapperInstaller.java
View File

@ -1,16 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.index;
import org.elasticsearch.shield.authz.accesscontrol.ShieldIndexSearcherWrapper;
public class SearcherWrapperInstaller {
public static void install(IndexModule module) {
module.indexSearcherWrapper = ShieldIndexSearcherWrapper.class;
}
}

19
shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java
View File

@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.http.HttpServerModule;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestModule;
import org.elasticsearch.shield.action.ShieldActionFilter;
@ -30,7 +29,7 @@ import org.elasticsearch.shield.authc.Realms;
import org.elasticsearch.shield.authc.support.SecuredString;
import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
import org.elasticsearch.shield.authz.AuthorizationModule;
import org.elasticsearch.index.SearcherWrapperInstaller;
import org.elasticsearch.index.ProtectedServiceInstaller;
import org.elasticsearch.shield.authz.accesscontrol.OptOutQueryCache;
import org.elasticsearch.shield.authz.store.FileRolesStore;
import org.elasticsearch.shield.crypto.CryptoModule;
@ -154,7 +153,7 @@ public class ShieldPlugin extends Plugin {
if (enabled == false) {
return;
}
SearcherWrapperInstaller.install(module);
ProtectedServiceInstaller.install(module, clientMode);
}
public void onModule(ActionModule module) {
@ -202,12 +201,6 @@ public class ShieldPlugin extends Plugin {
}
}
public void onModule(IndexCacheModule module) {
if (enabled && clientMode == false) {
module.registerQueryCache(OPT_OUT_QUERY_CACHE, OptOutQueryCache.class);
}
}
private void addUserSettings(Settings.Builder settingsBuilder) {
String authHeaderSettingName = Headers.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER;
if (settings.get(authHeaderSettingName) != null) {
@ -273,7 +266,7 @@ public class ShieldPlugin extends Plugin {
unauthorized users.
*/
private void addQueryCacheSettings(Settings.Builder settingsBuilder) {
settingsBuilder.put(IndexCacheModule.QUERY_CACHE_TYPE, OPT_OUT_QUERY_CACHE);
settingsBuilder.put(IndexModule.QUERY_CACHE_TYPE, OPT_OUT_QUERY_CACHE);
}
private static boolean isShieldMandatory(String[] existingMandatoryPlugins) {
@ -307,12 +300,12 @@ public class ShieldPlugin extends Plugin {
// in case this are node settings then the plugin additional settings have not been applied yet,
// so we use 'opt_out_cache' as default. So in that case we only fail if the node settings contain
// another cache impl than 'opt_out_cache'.
queryCacheImplementation = settings.get(IndexCacheModule.QUERY_CACHE_TYPE, OPT_OUT_QUERY_CACHE);
queryCacheImplementation = settings.get(IndexModule.QUERY_CACHE_TYPE, OPT_OUT_QUERY_CACHE);
} else {
queryCacheImplementation = settings.get(IndexCacheModule.QUERY_CACHE_TYPE);
queryCacheImplementation = settings.get(IndexModule.QUERY_CACHE_TYPE);
}
if (OPT_OUT_QUERY_CACHE.equals(queryCacheImplementation) == false) {
throw new IllegalStateException("shield does not support a user specified query cache. remove the setting [" + IndexCacheModule.QUERY_CACHE_TYPE + "] with value [" + queryCacheImplementation + "]");
throw new IllegalStateException("shield does not support a user specified query cache. remove the setting [" + IndexModule.QUERY_CACHE_TYPE + "] with value [" + queryCacheImplementation + "]");
}
}
}

3
shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/OptOutQueryCache.java
View File

@ -9,9 +9,7 @@ import org.apache.lucene.search.QueryCachingPolicy;
import org.apache.lucene.search.Weight;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.cache.query.QueryCache;
import org.elasticsearch.indices.cache.query.IndicesQueryCache;
@ -25,7 +23,6 @@ public final class OptOutQueryCache extends AbstractIndexComponent implements Qu
final IndicesQueryCache indicesQueryCache;
@Inject
public OptOutQueryCache(IndexSettings indexSettings, IndicesQueryCache indicesQueryCache) {
super(indexSettings);
this.indicesQueryCache = indicesQueryCache;

4
shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
View File

@ -7,7 +7,7 @@ package org.elasticsearch.integration;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.shield.authc.support.Hasher;
import org.elasticsearch.shield.authc.support.SecuredString;
import org.elasticsearch.test.ShieldIntegTestCase;
@ -95,7 +95,7 @@ public class DocumentAndFieldLevelSecurityTests extends ShieldIntegTestCase {
public void testQueryCache() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.setSettings(Settings.builder().put(IndexCacheModule.QUERY_CACHE_EVERYTHING, true))
.setSettings(Settings.builder().put(IndexModule.QUERY_CACHE_EVERYTHING, true))
.addMapping("type1", "field1", "type=string", "field2", "type=string")
);
client().prepareIndex("test", "type1", "1").setSource("field1", "value1")

6
shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.integration;
import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.Version;
import org.elasticsearch.action.fieldstats.FieldStatsResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetResponse;
@ -16,9 +15,8 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
import org.elasticsearch.action.termvectors.TermVectorsRequest;
import org.elasticsearch.action.termvectors.TermVectorsResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.indices.cache.request.IndicesRequestCache;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.aggregations.AggregationBuilders;
@ -357,7 +355,7 @@ public class FieldLevelSecurityTests extends ShieldIntegTestCase {
public void testQueryCache() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.setSettings(Settings.builder().put(IndexCacheModule.QUERY_CACHE_EVERYTHING, true))
.setSettings(Settings.builder().put(IndexModule.QUERY_CACHE_EVERYTHING, true))
.addMapping("type1", "field1", "type=string", "field2", "type=string")
);
client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2")

4
shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java
View File

@ -17,8 +17,8 @@ import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.*;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.shield.ShieldPlugin;
@ -149,7 +149,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
// For tests we forcefully configure Shield's custom query cache because the test framework randomizes the query cache impl,
// but if shield is disabled then we don't need to forcefully set the query cache
if (useShield == false) {
builder.remove(IndexCacheModule.QUERY_CACHE_TYPE);
builder.remove(IndexModule.QUERY_CACHE_TYPE);
}
return builder.build();
}

27
shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReaderTests.java
View File

@ -17,12 +17,19 @@ import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.junit.After;
import org.junit.Before;
import org.mockito.Matchers;
import java.util.Collections;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.mockito.Mockito.mock;
@ -37,18 +44,8 @@ public class DocumentSubsetReaderTests extends ESTestCase {
@Before
public void before() {
directory = newDirectory();
bitsetFilterCache = mock(BitsetFilterCache.class);
when(bitsetFilterCache.getBitSetProducer(Matchers.any(Query.class))).then(invocationOnMock -> {
final Query query = (Query) invocationOnMock.getArguments()[0];
return (BitSetProducer) context -> {
IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context);
IndexSearcher searcher = new IndexSearcher(topLevelContext);
searcher.setQueryCache(null);
Weight weight = searcher.createNormalizedWeight(query, false);
DocIdSetIterator it = weight.scorer(context);
return BitSet.of(it, context.reader().maxDoc());
};
});
IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST);
bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null));
}
@After
@ -57,6 +54,7 @@ public class DocumentSubsetReaderTests extends ESTestCase {
directoryReader.close();
}
directory.close();
bitsetFilterCache.close();
}
public void testSearch() throws Exception {
@ -151,8 +149,8 @@ public class DocumentSubsetReaderTests extends ESTestCase {
IndexWriterConfig iwc = new IndexWriterConfig(null);
IndexWriter iw = new IndexWriter(dir, iwc);
iw.close();
BitsetFilterCache bitsetFilterCache = mock(BitsetFilterCache.class);
IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST);
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null));
DirectoryReader directoryReader = DocumentSubsetReader.wrap(DirectoryReader.open(dir), bitsetFilterCache, new MatchAllDocsQuery());
try {
DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, new MatchAllDocsQuery());
@ -161,6 +159,7 @@ public class DocumentSubsetReaderTests extends ESTestCase {
assertThat(e.getMessage(), equalTo("Can't wrap [class org.elasticsearch.shield.authz.accesscontrol.DocumentSubsetReader$DocumentSubsetDirectoryReader] twice"));
}
bitsetFilterCache.close();
directoryReader.close();
dir.close();
}

28
shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperIntegrationTests.java
View File

@ -32,12 +32,15 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.query.ParsedQuery;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.shield.authz.InternalAuthorizationService;
import org.elasticsearch.shield.license.ShieldLicenseState;
import org.elasticsearch.test.ESTestCase;
@ -79,28 +82,8 @@ public class ShieldIndexSearcherWrapperIntegrationTests extends ESTestCase {
IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, null, singleton(new BytesArray("{}")));
request.putInContext(InternalAuthorizationService.INDICES_PERMISSIONS_KEY, new IndicesAccessControl(true, singletonMap("_index", indexAccessControl)));
IndexQueryParserService parserService = mock(IndexQueryParserService.class);
BitsetFilterCache bitsetFilterCache = mock(BitsetFilterCache.class);
when(bitsetFilterCache.getBitSetProducer(Matchers.any(Query.class))).then(new Answer<BitSetProducer>() {
@Override
public BitSetProducer answer(InvocationOnMock invocationOnMock) throws Throwable {
final Query query = (Query) invocationOnMock.getArguments()[0];
return context -> {
IndexSearcher searcher = new IndexSearcher(context);
searcher.setQueryCache(null);
Weight weight = searcher.createNormalizedWeight(query, false);
DocIdSetIterator it = weight.scorer(context);
if (it != null) {
int maxDoc = context.reader().maxDoc();
BitSet bitSet = randomBoolean() ? new SparseFixedBitSet(maxDoc) : new FixedBitSet(maxDoc);
bitSet.or(it);
return bitSet;
} else {
return null;
}
};
}
});
IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST);
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null));
ShieldLicenseState licenseState = mock(ShieldLicenseState.class);
when(licenseState.documentAndFieldLevelSecurityEnabled()).thenReturn(true);
ShieldIndexSearcherWrapper wrapper = new ShieldIndexSearcherWrapper(
@ -164,6 +147,7 @@ public class ShieldIndexSearcherWrapperIntegrationTests extends ESTestCase {
assertThat(wrappedDirectoryReader.numDocs(), equalTo(expectedHitCount));
}
bitsetFilterCache.close();
directoryReader.close();
directory.close();
}

5
shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java
View File

@ -44,6 +44,7 @@ import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.aggregations.LeafBucketCollector;
import org.elasticsearch.shield.authz.InternalAuthorizationService;
@ -246,12 +247,14 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase {
ShardId shardId = new ShardId("_index", 0);
EngineConfig engineConfig = new EngineConfig(shardId, null, null, Settings.EMPTY, null, null, null, null, null, null, new BM25Similarity(), null, null, null, new NoneQueryCache(IndexSettingsModule.newIndexSettings(shardId.index(), Settings.EMPTY, Collections.EMPTY_LIST)), QueryCachingPolicy.ALWAYS_CACHE, null, TimeValue.timeValueMinutes(5)); // can't mock...
BitsetFilterCache bitsetFilterCache = mock(BitsetFilterCache.class);
IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_index"), Settings.EMPTY, Collections.EMPTY_LIST);
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new IndicesWarmer(settings.getSettings(), null));
DirectoryReader directoryReader = DocumentSubsetReader.wrap(esIn, bitsetFilterCache, new MatchAllDocsQuery());
IndexSearcher indexSearcher = new IndexSearcher(directoryReader);
IndexSearcher result = shieldIndexSearcherWrapper.wrap(engineConfig, indexSearcher);
assertThat(result, not(sameInstance(indexSearcher)));
assertThat(result.getSimilarity(true), sameInstance(engineConfig.getSimilarity()));
bitsetFilterCache.close();
}
public void testIntersectScorerAndRoleBits() throws Exception {

4
shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java
View File

@ -9,7 +9,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.client.support.Headers;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.license.plugin.LicensePlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.shield.ShieldPlugin;
@ -125,7 +125,7 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ
.put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", configRoles()))
// Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the
// validation in ShieldPlugin fail.
.put(IndexCacheModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
.put(IndexModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
.put(getNodeSSLSettings());
setUser(builder, nodeClientUsername(), nodeClientPassword());

4
watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java
View File

@ -18,7 +18,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.Callback;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.license.plugin.LicensePlugin;
import org.elasticsearch.plugins.Plugin;
@ -644,7 +644,7 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase
.put("shield.audit.enabled", auditLogsEnabled)
// Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the
// validation in ShieldPlugin fail. Shield can only run with this query cache impl
.put(IndexCacheModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
.put(IndexModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
.build();
} catch (IOException ex) {
throw new RuntimeException("failed to build settings for shield", ex);