From 7925e2ef8475aa58e2126dde55832d77b3cf4428 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Sat, 16 Jan 2016 23:33:13 +0100 Subject: [PATCH] convert IndexModule settings --- .../common/settings/IndexScopeSettings.java | 4 ++ .../org/elasticsearch/index/IndexModule.java | 23 ++++++--- .../elasticsearch/index/shard/IndexShard.java | 2 +- .../index/store/FsDirectoryService.java | 2 +- .../index/store/StoreModule.java | 51 ------------------- .../TransportNodesListShardStoreMetaData.java | 5 +- .../elasticsearch/index/IndexModuleTests.java | 4 +- .../index/store/IndexStoreTests.java | 2 +- .../indices/stats/IndexStatsIT.java | 4 +- .../plugins/PluginsServiceTests.java | 6 +-- .../search/child/ChildQuerySearchIT.java | 4 +- .../messy/tests/ScriptQuerySearchTests.java | 4 +- .../mapper/attachments/AttachmentMapper.java | 10 ++-- .../attachments/MapperAttachmentsPlugin.java | 7 +++ .../index/mapper/size/SizeMappingTests.java | 8 +++ ...bstractAzureRepositoryServiceTestCase.java | 3 +- .../test/InternalTestCluster.java | 4 +- .../test/store/MockFSDirectoryService.java | 2 +- .../test/store/MockFSIndexStore.java | 4 +- 19 files changed, 65 insertions(+), 84 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/store/StoreModule.java diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopeSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopeSettings.java index f26a7b38723..5ccde281818 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopeSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopeSettings.java @@ -41,6 +41,7 @@ import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.gateway.PrimaryShardAllocator; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexingSlowLog; import org.elasticsearch.index.MergePolicyConfig; @@ -143,6 +144,9 @@ public final class IndexScopeSettings extends AbstractScopedSettings { PercolatorQueriesRegistry.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING, MapperService.INDEX_MAPPER_DYNAMIC_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, + IndexModule.INDEX_STORE_TYPE_SETTING, + IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING, + IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING, PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING, // this sucks but we can't really validate all the analyzers/similarity in here Setting.groupSetting("index.similarity.", false, Setting.Scope.INDEX), // this allows similarity settings to be passed diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java index 0e1331f31b2..6613df3c337 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java @@ -20,6 +20,7 @@ package org.elasticsearch.index; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; @@ -47,6 +48,7 @@ import java.util.Map; import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; /** * IndexModule represents the central extension point for index level custom implementations like: @@ -62,13 +64,22 @@ import java.util.function.Consumer; */ public final class IndexModule { - public static final String STORE_TYPE = "index.store.type"; + public static final Setting INDEX_STORE_TYPE_SETTING = new Setting<>("index.store.type", "", Function.identity(), false, Setting.Scope.INDEX); public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity"; public static final String INDEX_QUERY_CACHE = "index"; public static final String NONE_QUERY_CACHE = "none"; - public static final String QUERY_CACHE_TYPE = "index.queries.cache.type"; + public static final Setting INDEX_QUERY_CACHE_TYPE_SETTING = new Setting<>("index.queries.cache.type", INDEX_QUERY_CACHE, (s) -> { + switch (s) { + case NONE_QUERY_CACHE: + case INDEX_QUERY_CACHE: + return s; + default: + throw new IllegalArgumentException("Unknown value for [index.queries.cache.type]: " + s); + + } + }, false, Setting.Scope.INDEX); // for test purposes only - public static final String QUERY_CACHE_EVERYTHING = "index.queries.cache.everything"; + public static final Setting INDEX_QUERY_CACHE_EVERYTHING_SETTING = Setting.boolSetting("index.queries.cache.everything", false, false, Setting.Scope.INDEX); private final IndexSettings indexSettings; private final IndexStoreConfig indexStoreConfig; private final AnalysisRegistry analysisRegistry; @@ -242,9 +253,9 @@ public final class IndexModule { IndexingOperationListener... listeners) throws IOException { IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get(); IndexEventListener eventListener = freeze(); - final String storeType = indexSettings.getSettings().get(STORE_TYPE); + final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING); final IndexStore store; - if (storeType == null || isBuiltinType(storeType)) { + if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) { store = new IndexStore(indexSettings, indexStoreConfig); } else { BiFunction factory = storeTypes.get(storeType); @@ -258,7 +269,7 @@ public final class IndexModule { } indexSettings.getScopedSettings().addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, store::setMaxRate); indexSettings.getScopedSettings().addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING, store::setType); - final String queryCacheType = indexSettings.getSettings().get(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE); + final String queryCacheType = indexSettings.getValue(INDEX_QUERY_CACHE_TYPE_SETTING); final BiFunction queryCacheProvider = queryCaches.get(queryCacheType); final QueryCache queryCache = queryCacheProvider.apply(indexSettings, servicesProvider.getIndicesQueryCache()); return new IndexService(indexSettings, environment, new SimilarityService(indexSettings, similarities), shardStoreDeleter, analysisRegistry, engineFactory.get(), diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index be4a8a899d8..fbf777772d3 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -243,7 +243,7 @@ public class IndexShard extends AbstractIndexShardComponent { final QueryCachingPolicy cachingPolicy; // the query cache is a node-level thing, however we want the most popular filters // to be computed on a per-shard basis - if (settings.getAsBoolean(IndexModule.QUERY_CACHE_EVERYTHING, false)) { + if (IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.get(settings)) { cachingPolicy = QueryCachingPolicy.ALWAYS_CACHE; } else { cachingPolicy = new UsageTrackingQueryCachingPolicy(); diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java index fcc18f8b678..5ea24d1cc51 100644 --- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java +++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java @@ -112,7 +112,7 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { - final String storeType = indexSettings.getSettings().get(IndexModule.STORE_TYPE, IndexModule.Type.DEFAULT.getSettingsKey()); + final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.DEFAULT.getSettingsKey()); if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) { final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults if (open instanceof MMapDirectory && Constants.WINDOWS == false) { diff --git a/core/src/main/java/org/elasticsearch/index/store/StoreModule.java b/core/src/main/java/org/elasticsearch/index/store/StoreModule.java deleted file mode 100644 index fccd2de2e43..00000000000 --- a/core/src/main/java/org/elasticsearch/index/store/StoreModule.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.store; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.env.ShardLock; -import org.elasticsearch.index.shard.ShardPath; - -/** - * - */ -public class StoreModule extends AbstractModule { - private final ShardLock lock; - private final Store.OnClose closeCallback; - private final ShardPath path; - private final Class shardDirectory; - - - public StoreModule(Class shardDirectory, ShardLock lock, Store.OnClose closeCallback, ShardPath path) { - this.shardDirectory = shardDirectory; - this.lock = lock; - this.closeCallback = closeCallback; - this.path = path; - } - - @Override - protected void configure() { - bind(DirectoryService.class).to(shardDirectory).asEagerSingleton(); - bind(Store.class).asEagerSingleton(); - bind(ShardLock.class).toInstance(lock); - bind(Store.OnClose.class).toInstance(closeCallback); - bind(ShardPath.class).toInstance(path); - } -} diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java index 06d77822b7e..6a6b05c4ad1 100644 --- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java +++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -171,10 +172,6 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING = Setting.boolSetting("index.mapping.attachment.ignore_errors", true, false, Setting.Scope.INDEX); + public static final Setting INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING = Setting.boolSetting("index.mapping.attachment.detect_language", false, false, Setting.Scope.INDEX); + public static final Setting INDEX_ATTACHMENT_INDEXED_CHARS_SETTING = Setting.intSetting("index.mapping.attachment.indexed_chars", 100000, false, Setting.Scope.INDEX); public static final String CONTENT_TYPE = "attachment"; @@ -222,21 +226,21 @@ public class AttachmentMapper extends FieldMapper { context.path().remove(); if (defaultIndexedChars == null && context.indexSettings() != null) { - defaultIndexedChars = context.indexSettings().getAsInt("index.mapping.attachment.indexed_chars", 100000); + defaultIndexedChars = INDEX_ATTACHMENT_INDEXED_CHARS_SETTING.get(context.indexSettings()); } if (defaultIndexedChars == null) { defaultIndexedChars = 100000; } if (ignoreErrors == null && context.indexSettings() != null) { - ignoreErrors = context.indexSettings().getAsBoolean("index.mapping.attachment.ignore_errors", Boolean.TRUE); + ignoreErrors = INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING.get(context.indexSettings()); } if (ignoreErrors == null) { ignoreErrors = Boolean.TRUE; } if (langDetect == null && context.indexSettings() != null) { - langDetect = context.indexSettings().getAsBoolean("index.mapping.attachment.detect_language", Boolean.FALSE); + langDetect = INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING.get(context.indexSettings()); } if (langDetect == null) { langDetect = Boolean.FALSE; diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java index 9b640f98d16..e224607fff1 100644 --- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java @@ -19,6 +19,7 @@ package org.elasticsearch.mapper.attachments; +import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; @@ -34,6 +35,12 @@ public class MapperAttachmentsPlugin extends Plugin { return "Adds the attachment type allowing to parse difference attachment formats"; } + public void onModule(SettingsModule settingsModule) { + settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING); + settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING); + settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_INDEXED_CHARS_SETTING); + } + public void onModule(IndicesModule indicesModule) { indicesModule.registerMapper("attachment", new AttachmentMapper.TypeParser()); } diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index 956dd29402b..3e4d92beae2 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -34,9 +34,12 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -52,6 +55,11 @@ public class SizeMappingTests extends ESSingleNodeTestCase { MapperService mapperService; DocumentMapperParser parser; + @Override + protected Collection> getPlugins() { + return pluginList(InternalSettingsPlugin.class); // uses index.version.created + } + @Before public void before() { indexService = createIndex("test"); diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java index 4e1358edf39..73aa7e3921c 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java @@ -31,6 +31,7 @@ import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.test.store.MockFSDirectoryService; +import org.elasticsearch.test.store.MockFSIndexStore; import org.junit.After; import org.junit.Before; @@ -85,7 +86,7 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu @Override protected Collection> nodePlugins() { - return pluginList(AzureRepositoryPlugin.class, TestPlugin.class); + return pluginList(AzureRepositoryPlugin.class, TestPlugin.class, MockFSIndexStore.TestPlugin.class); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 628e8872fd8..2f02e46f58b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -425,11 +425,11 @@ public final class InternalTestCluster extends TestCluster { } if (random.nextBoolean()) { - builder.put(IndexModule.QUERY_CACHE_TYPE, random.nextBoolean() ? IndexModule.INDEX_QUERY_CACHE : IndexModule.NONE_QUERY_CACHE); + builder.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), random.nextBoolean() ? IndexModule.INDEX_QUERY_CACHE : IndexModule.NONE_QUERY_CACHE); } if (random.nextBoolean()) { - builder.put(IndexModule.QUERY_CACHE_EVERYTHING, random.nextBoolean()); + builder.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), random.nextBoolean()); } if (random.nextBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java index f3421b2485f..7d2e4a44640 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java @@ -182,7 +182,7 @@ public class MockFSDirectoryService extends FsDirectoryService { private FsDirectoryService randomDirectorService(IndexStore indexStore, ShardPath path) { final IndexSettings indexSettings = indexStore.getIndexSettings(); - final IndexMetaData build = IndexMetaData.builder(indexSettings.getIndexMetaData()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexModule.STORE_TYPE, RandomPicks.randomFrom(random, IndexModule.Type.values()).getSettingsKey())).build(); + final IndexMetaData build = IndexMetaData.builder(indexSettings.getIndexMetaData()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), RandomPicks.randomFrom(random, IndexModule.Type.values()).getSettingsKey())).build(); final IndexSettings newIndexSettings = new IndexSettings(build, indexSettings.getNodeSettings()); return new FsDirectoryService(newIndexSettings, indexStore, path); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java index 8e797f9c50c..80251d54951 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java +++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java @@ -57,7 +57,7 @@ public class MockFSIndexStore extends IndexStore { } @Override public Settings additionalSettings() { - return Settings.builder().put(IndexModule.STORE_TYPE, "mock").build(); + return Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mock").build(); } public void onModule(SettingsModule module) { @@ -73,7 +73,7 @@ public class MockFSIndexStore extends IndexStore { @Override public void onIndexModule(IndexModule indexModule) { Settings indexSettings = indexModule.getSettings(); - if ("mock".equals(indexSettings.get(IndexModule.STORE_TYPE))) { + if ("mock".equals(indexSettings.get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()))) { if (INDEX_CHECK_INDEX_ON_CLOSE_SETTING.get(indexSettings)) { indexModule.addIndexEventListener(new Listener()); }