convert IndexModule settings

This commit is contained in:
Simon Willnauer 2016-01-16 23:33:13 +01:00
parent dc05669fd9
commit 7925e2ef84
19 changed files with 65 additions and 84 deletions

View File

@ -41,6 +41,7 @@ import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.zen.ZenDiscovery;
import org.elasticsearch.discovery.zen.elect.ElectMasterService;
import org.elasticsearch.gateway.PrimaryShardAllocator;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexingSlowLog;
import org.elasticsearch.index.MergePolicyConfig;
@ -143,6 +144,9 @@ public final class IndexScopeSettings extends AbstractScopedSettings {
PercolatorQueriesRegistry.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
IndexModule.INDEX_STORE_TYPE_SETTING,
IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING,
IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING,
PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING,
// this sucks but we can't really validate all the analyzers/similarity in here
Setting.groupSetting("index.similarity.", false, Setting.Scope.INDEX), // this allows similarity settings to be passed

View File

@ -20,6 +20,7 @@
package org.elasticsearch.index;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
@ -47,6 +48,7 @@ import java.util.Map;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
/**
* IndexModule represents the central extension point for index level custom implementations like:
@ -62,13 +64,22 @@ import java.util.function.Consumer;
*/
public final class IndexModule {
public static final String STORE_TYPE = "index.store.type";
public static final Setting<String> INDEX_STORE_TYPE_SETTING = new Setting<>("index.store.type", "", Function.identity(), false, Setting.Scope.INDEX);
public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity";
public static final String INDEX_QUERY_CACHE = "index";
public static final String NONE_QUERY_CACHE = "none";
public static final String QUERY_CACHE_TYPE = "index.queries.cache.type";
public static final Setting<String> INDEX_QUERY_CACHE_TYPE_SETTING = new Setting<>("index.queries.cache.type", INDEX_QUERY_CACHE, (s) -> {
switch (s) {
case NONE_QUERY_CACHE:
case INDEX_QUERY_CACHE:
return s;
default:
throw new IllegalArgumentException("Unknown value for [index.queries.cache.type]: " + s);
}
}, false, Setting.Scope.INDEX);
// for test purposes only
public static final String QUERY_CACHE_EVERYTHING = "index.queries.cache.everything";
public static final Setting<Boolean> INDEX_QUERY_CACHE_EVERYTHING_SETTING = Setting.boolSetting("index.queries.cache.everything", false, false, Setting.Scope.INDEX);
private final IndexSettings indexSettings;
private final IndexStoreConfig indexStoreConfig;
private final AnalysisRegistry analysisRegistry;
@ -242,9 +253,9 @@ public final class IndexModule {
IndexingOperationListener... listeners) throws IOException {
IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get();
IndexEventListener eventListener = freeze();
final String storeType = indexSettings.getSettings().get(STORE_TYPE);
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
final IndexStore store;
if (storeType == null || isBuiltinType(storeType)) {
if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) {
store = new IndexStore(indexSettings, indexStoreConfig);
} else {
BiFunction<IndexSettings, IndexStoreConfig, IndexStore> factory = storeTypes.get(storeType);
@ -258,7 +269,7 @@ public final class IndexModule {
}
indexSettings.getScopedSettings().addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, store::setMaxRate);
indexSettings.getScopedSettings().addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING, store::setType);
final String queryCacheType = indexSettings.getSettings().get(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE);
final String queryCacheType = indexSettings.getValue(INDEX_QUERY_CACHE_TYPE_SETTING);
final BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = queryCaches.get(queryCacheType);
final QueryCache queryCache = queryCacheProvider.apply(indexSettings, servicesProvider.getIndicesQueryCache());
return new IndexService(indexSettings, environment, new SimilarityService(indexSettings, similarities), shardStoreDeleter, analysisRegistry, engineFactory.get(),

View File

@ -243,7 +243,7 @@ public class IndexShard extends AbstractIndexShardComponent {
final QueryCachingPolicy cachingPolicy;
// the query cache is a node-level thing, however we want the most popular filters
// to be computed on a per-shard basis
if (settings.getAsBoolean(IndexModule.QUERY_CACHE_EVERYTHING, false)) {
if (IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.get(settings)) {
cachingPolicy = QueryCachingPolicy.ALWAYS_CACHE;
} else {
cachingPolicy = new UsageTrackingQueryCachingPolicy();

View File

@ -112,7 +112,7 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
final String storeType = indexSettings.getSettings().get(IndexModule.STORE_TYPE, IndexModule.Type.DEFAULT.getSettingsKey());
final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.DEFAULT.getSettingsKey());
if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) {
final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults
if (open instanceof MMapDirectory && Constants.WINDOWS == false) {

View File

@ -1,51 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.env.ShardLock;
import org.elasticsearch.index.shard.ShardPath;
/**
*
*/
public class StoreModule extends AbstractModule {
private final ShardLock lock;
private final Store.OnClose closeCallback;
private final ShardPath path;
private final Class<? extends DirectoryService> shardDirectory;
public StoreModule(Class<? extends DirectoryService> shardDirectory, ShardLock lock, Store.OnClose closeCallback, ShardPath path) {
this.shardDirectory = shardDirectory;
this.lock = lock;
this.closeCallback = closeCallback;
this.path = path;
}
@Override
protected void configure() {
bind(DirectoryService.class).to(shardDirectory).asEagerSingleton();
bind(Store.class).asEagerSingleton();
bind(ShardLock.class).toInstance(lock);
bind(Store.OnClose.class).toInstance(closeCallback);
bind(ShardPath.class).toInstance(path);
}
}

View File

@ -34,6 +34,7 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -171,10 +172,6 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
if (metaData == null) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
String storeType = metaData.getSettings().get(IndexModule.STORE_TYPE, "fs");
if (!storeType.contains("fs")) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
final IndexSettings indexSettings = indexService != null ? indexService.getIndexSettings() : new IndexSettings(metaData, settings);
final ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
if (shardPath == null) {

View File

@ -148,7 +148,7 @@ public class IndexModuleTests extends ESTestCase {
public void testRegisterIndexStore() throws IOException {
final Index index = new Index("foo");
final Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("path.home", createTempDir().toString()).put(IndexModule.STORE_TYPE, "foo_store").build();
final Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("path.home", createTempDir().toString()).put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "foo_store").build();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
module.addIndexStore("foo_store", FooStore::new);
@ -291,7 +291,7 @@ public class IndexModuleTests extends ESTestCase {
public void testRegisterCustomQueryCache() throws IOException {
Settings indexSettings = Settings.settingsBuilder()
.put(IndexModule.QUERY_CACHE_TYPE, "custom")
.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), "custom")
.put("path.home", createTempDir().toString())
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment));

View File

@ -49,7 +49,7 @@ public class IndexStoreTests extends ESTestCase {
final Path tempDir = createTempDir().resolve("foo").resolve("0");
final IndexModule.Type[] values = IndexModule.Type.values();
final IndexModule.Type type = RandomPicks.randomFrom(random(), values);
Settings settings = Settings.settingsBuilder().put(IndexModule.STORE_TYPE, type.name().toLowerCase(Locale.ROOT))
Settings settings = Settings.settingsBuilder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), type.name().toLowerCase(Locale.ROOT))
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(new Index("foo"), settings);
FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0)));

View File

@ -79,8 +79,8 @@ public class IndexStatsIT extends ESIntegTestCase {
//Filter/Query cache is cleaned periodically, default is 60s, so make sure it runs often. Thread.sleep for 60s is bad
return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal))
.put(IndicesRequestCache.INDICES_CACHE_REQUEST_CLEAN_INTERVAL, "1ms")
.put(IndexModule.QUERY_CACHE_EVERYTHING, true)
.put(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE)
.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), IndexModule.INDEX_QUERY_CACHE)
.build();
}

View File

@ -42,7 +42,7 @@ public class PluginsServiceTests extends ESTestCase {
}
@Override
public Settings additionalSettings() {
return Settings.builder().put("foo.bar", "1").put(IndexModule.STORE_TYPE, IndexModule.Type.MMAPFS.getSettingsKey()).build();
return Settings.builder().put("foo.bar", "1").put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.MMAPFS.getSettingsKey()).build();
}
}
public static class AdditionalSettingsPlugin2 extends Plugin {
@ -90,12 +90,12 @@ public class PluginsServiceTests extends ESTestCase {
Settings settings = Settings.builder()
.put("path.home", createTempDir())
.put("my.setting", "test")
.put(IndexModule.STORE_TYPE, IndexModule.Type.SIMPLEFS.getSettingsKey()).build();
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.SIMPLEFS.getSettingsKey()).build();
PluginsService service = newPluginsService(settings, AdditionalSettingsPlugin1.class);
Settings newSettings = service.updatedSettings();
assertEquals("test", newSettings.get("my.setting")); // previous settings still exist
assertEquals("1", newSettings.get("foo.bar")); // added setting exists
assertEquals(IndexModule.Type.SIMPLEFS.getSettingsKey(), newSettings.get(IndexModule.STORE_TYPE)); // does not override pre existing settings
assertEquals(IndexModule.Type.SIMPLEFS.getSettingsKey(), newSettings.get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey())); // does not override pre existing settings
}
public void testAdditionalSettingsClash() {

View File

@ -96,8 +96,8 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal))
// aggressive filter caching so that we can assert on the filter cache size
.put(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE)
.put(IndexModule.QUERY_CACHE_EVERYTHING, true)
.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), IndexModule.INDEX_QUERY_CACHE)
.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
.build();
}

View File

@ -53,8 +53,8 @@ public class ScriptQuerySearchTests extends ESIntegTestCase {
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal))
// aggressive filter caching so that we can assert on the number of iterations of the script filters
.put(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE)
.put(IndexModule.QUERY_CACHE_EVERYTHING, true)
.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), IndexModule.INDEX_QUERY_CACHE)
.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
.build();
}

View File

@ -27,6 +27,7 @@ import org.elasticsearch.Version;
import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
@ -70,6 +71,9 @@ import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
public class AttachmentMapper extends FieldMapper {
private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment");
public static final Setting<Boolean> INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING = Setting.boolSetting("index.mapping.attachment.ignore_errors", true, false, Setting.Scope.INDEX);
public static final Setting<Boolean> INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING = Setting.boolSetting("index.mapping.attachment.detect_language", false, false, Setting.Scope.INDEX);
public static final Setting<Integer> INDEX_ATTACHMENT_INDEXED_CHARS_SETTING = Setting.intSetting("index.mapping.attachment.indexed_chars", 100000, false, Setting.Scope.INDEX);
public static final String CONTENT_TYPE = "attachment";
@ -222,21 +226,21 @@ public class AttachmentMapper extends FieldMapper {
context.path().remove();
if (defaultIndexedChars == null && context.indexSettings() != null) {
defaultIndexedChars = context.indexSettings().getAsInt("index.mapping.attachment.indexed_chars", 100000);
defaultIndexedChars = INDEX_ATTACHMENT_INDEXED_CHARS_SETTING.get(context.indexSettings());
}
if (defaultIndexedChars == null) {
defaultIndexedChars = 100000;
}
if (ignoreErrors == null && context.indexSettings() != null) {
ignoreErrors = context.indexSettings().getAsBoolean("index.mapping.attachment.ignore_errors", Boolean.TRUE);
ignoreErrors = INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING.get(context.indexSettings());
}
if (ignoreErrors == null) {
ignoreErrors = Boolean.TRUE;
}
if (langDetect == null && context.indexSettings() != null) {
langDetect = context.indexSettings().getAsBoolean("index.mapping.attachment.detect_language", Boolean.FALSE);
langDetect = INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING.get(context.indexSettings());
}
if (langDetect == null) {
langDetect = Boolean.FALSE;

View File

@ -19,6 +19,7 @@
package org.elasticsearch.mapper.attachments;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.plugins.Plugin;
@ -34,6 +35,12 @@ public class MapperAttachmentsPlugin extends Plugin {
return "Adds the attachment type allowing to parse difference attachment formats";
}
public void onModule(SettingsModule settingsModule) {
settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING);
settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING);
settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_INDEXED_CHARS_SETTING);
}
public void onModule(IndicesModule indicesModule) {
indicesModule.registerMapper("attachment", new AttachmentMapper.TypeParser());
}

View File

@ -34,9 +34,12 @@ import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.mapper.MapperRegistry;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.junit.Before;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@ -52,6 +55,11 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
MapperService mapperService;
DocumentMapperParser parser;
@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return pluginList(InternalSettingsPlugin.class); // uses index.version.created
}
@Before
public void before() {
indexService = createIndex("test");

View File

@ -31,6 +31,7 @@ import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.test.store.MockFSDirectoryService;
import org.elasticsearch.test.store.MockFSIndexStore;
import org.junit.After;
import org.junit.Before;
@ -85,7 +86,7 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(AzureRepositoryPlugin.class, TestPlugin.class);
return pluginList(AzureRepositoryPlugin.class, TestPlugin.class, MockFSIndexStore.TestPlugin.class);
}
@Override

View File

@ -425,11 +425,11 @@ public final class InternalTestCluster extends TestCluster {
}
if (random.nextBoolean()) {
builder.put(IndexModule.QUERY_CACHE_TYPE, random.nextBoolean() ? IndexModule.INDEX_QUERY_CACHE : IndexModule.NONE_QUERY_CACHE);
builder.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), random.nextBoolean() ? IndexModule.INDEX_QUERY_CACHE : IndexModule.NONE_QUERY_CACHE);
}
if (random.nextBoolean()) {
builder.put(IndexModule.QUERY_CACHE_EVERYTHING, random.nextBoolean());
builder.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), random.nextBoolean());
}
if (random.nextBoolean()) {

View File

@ -182,7 +182,7 @@ public class MockFSDirectoryService extends FsDirectoryService {
private FsDirectoryService randomDirectorService(IndexStore indexStore, ShardPath path) {
final IndexSettings indexSettings = indexStore.getIndexSettings();
final IndexMetaData build = IndexMetaData.builder(indexSettings.getIndexMetaData()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexModule.STORE_TYPE, RandomPicks.randomFrom(random, IndexModule.Type.values()).getSettingsKey())).build();
final IndexMetaData build = IndexMetaData.builder(indexSettings.getIndexMetaData()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), RandomPicks.randomFrom(random, IndexModule.Type.values()).getSettingsKey())).build();
final IndexSettings newIndexSettings = new IndexSettings(build, indexSettings.getNodeSettings());
return new FsDirectoryService(newIndexSettings, indexStore, path);
}

View File

@ -57,7 +57,7 @@ public class MockFSIndexStore extends IndexStore {
}
@Override
public Settings additionalSettings() {
return Settings.builder().put(IndexModule.STORE_TYPE, "mock").build();
return Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mock").build();
}
public void onModule(SettingsModule module) {
@ -73,7 +73,7 @@ public class MockFSIndexStore extends IndexStore {
@Override
public void onIndexModule(IndexModule indexModule) {
Settings indexSettings = indexModule.getSettings();
if ("mock".equals(indexSettings.get(IndexModule.STORE_TYPE))) {
if ("mock".equals(indexSettings.get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()))) {
if (INDEX_CHECK_INDEX_ON_CLOSE_SETTING.get(indexSettings)) {
indexModule.addIndexEventListener(new Listener());
}