honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert index level setting to the new setting infrastrucutre 

this is an initial commit of cutting over simple string key based settings
to a more contained scoped settings infrastructure.
This commit is contained in:
Simon Willnauer 2016-01-14 12:16:15 +01:00
parent 69b21feb3b
commit da0424c75f
30 changed files with 233 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
View File

@ -127,11 +127,6 @@ public class ClusterModule extends AbstractModule {
}
private void registerBuiltinIndexSettings() {
registerIndexDynamicSetting(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE);
registerIndexDynamicSetting(IndexStore.INDEX_STORE_THROTTLE_TYPE, Validator.EMPTY);
registerIndexDynamicSetting(MergeSchedulerConfig.MAX_THREAD_COUNT, Validator.NON_NEGATIVE_INTEGER);
registerIndexDynamicSetting(MergeSchedulerConfig.MAX_MERGE_COUNT, Validator.EMPTY);
registerIndexDynamicSetting(MergeSchedulerConfig.AUTO_THROTTLE, Validator.EMPTY);
registerIndexDynamicSetting(FilterAllocationDecider.INDEX_ROUTING_REQUIRE_GROUP + "*", Validator.EMPTY);
registerIndexDynamicSetting(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "*", Validator.EMPTY);
registerIndexDynamicSetting(FilterAllocationDecider.INDEX_ROUTING_EXCLUDE_GROUP + "*", Validator.EMPTY);

2
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
View File

@ -217,7 +217,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
try {
// We cannot instantiate real analysis server at this point because the node might not have
// been started yet. However, we don't really need real analyzers at this stage - so we can fake it
IndexSettings indexSettings = new IndexSettings(indexMetaData, this.settings, Collections.emptyList());
IndexSettings indexSettings = new IndexSettings(indexMetaData, this.settings);
SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
try (AnalysisService analysisService = new FakeAnalysisService(indexSettings)) {

45
core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
View File

@ -21,9 +21,12 @@ package org.elasticsearch.common.settings;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.util.set.Sets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -42,18 +45,27 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
private final Setting.Scope scope;
protected AbstractScopedSettings(Settings settings, Set<Setting<?>> settingsSet, Setting.Scope scope) {
super(settings);
for (Setting<?> entry : settingsSet) {
if (entry.getScope() != scope) {
throw new IllegalArgumentException("Setting must be a cluster setting but was: " + entry.getScope());
}
if (entry.hasComplexMatcher()) {
complexMatchers.put(entry.getKey(), entry);
} else {
keySettings.put(entry.getKey(), entry);
}
this(settings, Settings.EMPTY, settingsSet, scope);
}
protected AbstractScopedSettings(Settings nodeSettings, Settings scopeSettings, Set<Setting<?>> settingsSet, Setting.Scope scope) {
super(nodeSettings);
this.lastSettingsApplied = scopeSettings;
this.scope = scope;
for (Setting<?> entry : settingsSet) {
addSetting(entry);
}
}
protected final void addSetting(Setting<?> setting) {
if (setting.getScope() != scope) {
throw new IllegalArgumentException("Setting must be a " + scope + " setting but was: " + setting.getScope());
}
if (setting.hasComplexMatcher()) {
complexMatchers.putIfAbsent(setting.getKey(), setting);
} else {
keySettings.putIfAbsent(setting.getKey(), setting);
}
}
public Setting.Scope getScope() {
@ -253,4 +265,17 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
return builder.build();
}
/**
* Returns the value for the given setting.
*/
public <T> T get(Setting<T> setting) {
if (setting.getScope() != scope) {
throw new IllegalArgumentException("settings scope doesn't match the setting scope [" + this.scope + "] != [" + setting.getScope() + "]");
}
if (get(setting.getKey()) == null) {
throw new IllegalArgumentException("setting " + setting.getKey() + " has not been registered");
}
return setting.get(this.lastSettingsApplied);
}
}

2
core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
View File

@ -73,7 +73,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
}
final IndexMetaData indexMetaData = metaData.index(shard.getIndex());
final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings, Collections.emptyList());
final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
if (shard.allocatedPostIndexCreate(indexMetaData) == false) {
// when we create a fresh index

2
core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
View File

@ -130,7 +130,7 @@ public class TransportNodesListGatewayStartedShards extends TransportNodesAction
if (metaData != null) {
ShardPath shardPath = null;
try {
IndexSettings indexSettings = new IndexSettings(metaData, settings, Collections.emptyList());
IndexSettings indexSettings = new IndexSettings(metaData, settings);
shardPath = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
if (shardPath == null) {
throw new IllegalStateException(shardId + " no shard path found");

46
core/src/main/java/org/elasticsearch/index/IndexModule.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.index;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.analysis.AnalysisRegistry;
@ -35,7 +36,6 @@ import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.IndexStoreConfig;
import org.elasticsearch.indices.IndexingMemoryController;
import org.elasticsearch.indices.cache.query.IndicesQueryCache;
import org.elasticsearch.indices.mapper.MapperRegistry;
@ -57,7 +57,7 @@ import java.util.function.Consumer;
* <tt>"index.similarity.my_similarity.type : "BM25"</tt> can be used.</li>
* <li>{@link IndexStore} - Custom {@link IndexStore} instances can be registered via {@link #addIndexStore(String, BiFunction)}</li>
* <li>{@link IndexEventListener} - Custom {@link IndexEventListener} instances can be registered via {@link #addIndexEventListener(IndexEventListener)}</li>
* <li>Settings update listener - Custom settings update listener can be registered via {@link #addIndexSettingsListener(Consumer)}</li>
* <li>Settings update listener - Custom settings update listener can be registered via {@link #addSettingsUpdateConsumer(Setting, Consumer)}</li>
* </ul>
*/
public final class IndexModule {
@ -75,7 +75,6 @@ public final class IndexModule {
// pkg private so tests can mock
final SetOnce<EngineFactory> engineFactory = new SetOnce<>();
private SetOnce<IndexSearcherWrapperFactory> indexSearcherWrapper = new SetOnce<>();
private final Set<Consumer<Settings>> settingsConsumers = new HashSet<>();
private final Set<IndexEventListener> indexEventListeners = new HashSet<>();
private IndexEventListener listener;
private final Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities = new HashMap<>();
@ -92,17 +91,28 @@ public final class IndexModule {
}
/**
* Adds a settings consumer for this index
* Adds a Setting for this index.
*/
public void addIndexSettingsListener(Consumer<Settings> listener) {
if (listener == null) {
throw new IllegalArgumentException("listener must not be null");
public <T> void addSetting(Setting<T> setting) {
addSettingsUpdateConsumer(setting, null);
}
if (settingsConsumers.contains(listener)) {
throw new IllegalStateException("listener already registered");
/**
* Adds a Setting and it's consumer for this index.
*/
public <T> void addSettingsUpdateConsumer(Setting<T> setting, Consumer<T> consumer) {
if (setting == null) {
throw new IllegalArgumentException("setting must not be null");
}
settingsConsumers.add(listener);
if (indexSettings.containsSetting(setting)) {
throw new IllegalArgumentException("setting already registered: " + setting);
}
if (consumer != null) {
indexSettings.addSettingsUpdateConsumer(setting, consumer);
} else {
indexSettings.addSetting(setting);
}
}
/**
@ -245,27 +255,29 @@ public final class IndexModule {
public IndexService newIndexService(NodeEnvironment environment, IndexService.ShardStoreDeleter shardStoreDeleter, NodeServicesProvider servicesProvider, MapperRegistry mapperRegistry,
IndexingOperationListener... listeners) throws IOException {
final IndexSettings settings = indexSettings.newWithListener(settingsConsumers);
IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get();
IndexEventListener eventListener = freeze();
final String storeType = settings.getSettings().get(STORE_TYPE);
final String storeType = indexSettings.getSettings().get(STORE_TYPE);
final IndexStore store;
if (storeType == null || isBuiltinType(storeType)) {
store = new IndexStore(settings, indexStoreConfig);
store = new IndexStore(indexSettings, indexStoreConfig);
} else {
BiFunction<IndexSettings, IndexStoreConfig, IndexStore> factory = storeTypes.get(storeType);
if (factory == null) {
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
}
store = factory.apply(settings, indexStoreConfig);
store = factory.apply(indexSettings, indexStoreConfig);
if (store == null) {
throw new IllegalStateException("store must not be null");
}
}
final String queryCacheType = settings.getSettings().get(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE);
indexSettings.addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, store::setMaxRate);
indexSettings.addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING, store::setType);
final String queryCacheType = indexSettings.getSettings().get(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE);
final BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = queryCaches.get(queryCacheType);
final QueryCache queryCache = queryCacheProvider.apply(settings, servicesProvider.getIndicesQueryCache());
return new IndexService(settings, environment, new SimilarityService(settings, similarities), shardStoreDeleter, analysisRegistry, engineFactory.get(),
final QueryCache queryCache = queryCacheProvider.apply(indexSettings, servicesProvider.getIndicesQueryCache());
return new IndexService(indexSettings, environment, new SimilarityService(indexSettings, similarities), shardStoreDeleter, analysisRegistry, engineFactory.get(),
servicesProvider, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry, listeners);
}
}

5
core/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -575,11 +575,6 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
logger.warn("[{}] failed to notify shard about setting change", e, shard.shardId().id());
}
}
try {
indexStore.onRefreshSettings(settings);
} catch (Exception e) {
logger.warn("failed to refresh index store settings", e);
}
try {
slowLog.onRefreshSettings(settings); // this will be refactored soon anyway so duplication is ok here
} catch (Exception e) {

104
core/src/main/java/org/elasticsearch/index/IndexSettings.java
View File

@ -25,19 +25,21 @@ import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.AbstractScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.translog.Translog;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
@ -46,7 +48,7 @@ import java.util.function.Predicate;
* This class encapsulates all index level settings and handles settings updates.
* It's created per index and available to all index level classes and allows them to retrieve
* the latest updated settings instance. Classes that need to listen to settings updates can register
* a settings consumer at index creation via {@link IndexModule#addIndexSettingsListener(Consumer)} that will
* a settings consumer at index creation via {@link IndexModule#addSettingsUpdateConsumer(Setting, Consumer)} that will
* be called for each settings update.
*/
public final class IndexSettings {
@ -70,7 +72,6 @@ public final class IndexSettings {
public static final String INDEX_GC_DELETES_SETTING = "index.gc_deletes";
private final String uuid;
private final List<Consumer<Settings>> updateListeners;
private final Index index;
private final Version version;
private final ESLogger logger;
@ -94,9 +95,16 @@ public final class IndexSettings {
private volatile ByteSizeValue flushThresholdSize;
private final MergeSchedulerConfig mergeSchedulerConfig;
private final MergePolicyConfig mergePolicyConfig;
private final ScopedSettings scopedSettings;
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
public static Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING,
IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING,
MergeSchedulerConfig.AUTO_THROTTLE_SETTING,
MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING
)));
/**
* Returns the default search field for this index.
@ -139,10 +147,9 @@ public final class IndexSettings {
*
* @param indexMetaData the index metadata this settings object is associated with
* @param nodeSettings the nodes settings this index is allocated on.
* @param updateListeners a collection of listeners / consumers that should be notified if one or more settings are updated
*/
public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSettings, final Collection<Consumer<Settings>> updateListeners) {
this(indexMetaData, nodeSettings, updateListeners, (index) -> Regex.simpleMatch(index, indexMetaData.getIndex()));
public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSettings) {
this(indexMetaData, nodeSettings, (index) -> Regex.simpleMatch(index, indexMetaData.getIndex()));
}
/**
@ -151,13 +158,12 @@ public final class IndexSettings {
*
* @param indexMetaData the index metadata this settings object is associated with
* @param nodeSettings the nodes settings this index is allocated on.
* @param updateListeners a collection of listeners / consumers that should be notified if one or more settings are updated
* @param indexNameMatcher a matcher that can resolve an expression to the index name or index alias
*/
public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSettings, final Collection<Consumer<Settings>> updateListeners, final Predicate<String> indexNameMatcher) {
public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSettings,final Predicate<String> indexNameMatcher) {
scopedSettings = new ScopedSettings(nodeSettings, indexMetaData.getSettings(), BUILT_IN_CLUSTER_SETTINGS);
this.nodeSettings = nodeSettings;
this.settings = Settings.builder().put(nodeSettings).put(indexMetaData.getSettings()).build();
this.updateListeners = Collections.unmodifiableList( new ArrayList<>(updateListeners));
this.index = new Index(indexMetaData.getIndex());
version = Version.indexCreated(settings);
uuid = settings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE);
@ -179,22 +185,12 @@ public final class IndexSettings {
syncInterval = settings.getAsTime(INDEX_TRANSLOG_SYNC_INTERVAL, TimeValue.timeValueSeconds(5));
refreshInterval = settings.getAsTime(INDEX_REFRESH_INTERVAL, DEFAULT_REFRESH_INTERVAL);
flushThresholdSize = settings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB));
mergeSchedulerConfig = new MergeSchedulerConfig(settings);
mergeSchedulerConfig = new MergeSchedulerConfig(this);
gcDeletesInMillis = settings.getAsTime(IndexSettings.INDEX_GC_DELETES_SETTING, DEFAULT_GC_DELETES).getMillis();
this.mergePolicyConfig = new MergePolicyConfig(logger, settings);
assert indexNameMatcher.test(indexMetaData.getIndex());
}
/**
* Creates a new {@link IndexSettings} instance adding the given listeners to the settings
*/
IndexSettings newWithListener(final Collection<Consumer<Settings>> updateListeners) {
ArrayList<Consumer<Settings>> newUpdateListeners = new ArrayList<>(updateListeners);
newUpdateListeners.addAll(this.updateListeners);
return new IndexSettings(indexMetaData, nodeSettings, newUpdateListeners, indexNameMatcher);
}
/**
* Returns the settings for this index. These settings contain the node and index level settings where
* settings that are specified on both index and node level are overwritten by the index settings.
@ -325,14 +321,10 @@ public final class IndexSettings {
// nothing to update, same settings
return false;
}
scopedSettings.applySettings(newSettings);
// nocommit
final Settings mergedSettings = this.settings = Settings.builder().put(nodeSettings).put(newSettings).build();
for (final Consumer<Settings> consumer : updateListeners) {
try {
consumer.accept(mergedSettings);
} catch (Exception e) {
logger.warn("failed to refresh index settings for [{}]", e, mergedSettings);
}
}
try {
updateSettings(mergedSettings);
} catch (Exception e) {
@ -341,13 +333,6 @@ public final class IndexSettings {
return true;
}
/**
* Returns all settings update consumers
*/
List<Consumer<Settings>> getUpdateListeners() { // for testing
return updateListeners;
}
/**
* Returns the translog durability for this index.
*/
@ -384,24 +369,6 @@ public final class IndexSettings {
this.flushThresholdSize = flushThresholdSize;
}
final int maxThreadCount = settings.getAsInt(MergeSchedulerConfig.MAX_THREAD_COUNT, mergeSchedulerConfig.getMaxThreadCount());
if (maxThreadCount != mergeSchedulerConfig.getMaxThreadCount()) {
logger.info("updating [{}] from [{}] to [{}]", MergeSchedulerConfig.MAX_THREAD_COUNT, mergeSchedulerConfig.getMaxMergeCount(), maxThreadCount);
mergeSchedulerConfig.setMaxThreadCount(maxThreadCount);
}
final int maxMergeCount = settings.getAsInt(MergeSchedulerConfig.MAX_MERGE_COUNT, mergeSchedulerConfig.getMaxMergeCount());
if (maxMergeCount != mergeSchedulerConfig.getMaxMergeCount()) {
logger.info("updating [{}] from [{}] to [{}]", MergeSchedulerConfig.MAX_MERGE_COUNT, mergeSchedulerConfig.getMaxMergeCount(), maxMergeCount);
mergeSchedulerConfig.setMaxMergeCount(maxMergeCount);
}
final boolean autoThrottle = settings.getAsBoolean(MergeSchedulerConfig.AUTO_THROTTLE, mergeSchedulerConfig.isAutoThrottle());
if (autoThrottle != mergeSchedulerConfig.isAutoThrottle()) {
logger.info("updating [{}] from [{}] to [{}]", MergeSchedulerConfig.AUTO_THROTTLE, mergeSchedulerConfig.isAutoThrottle(), autoThrottle);
mergeSchedulerConfig.setAutoThrottle(autoThrottle);
}
long gcDeletesInMillis = settings.getAsTime(IndexSettings.INDEX_GC_DELETES_SETTING, TimeValue.timeValueMillis(this.gcDeletesInMillis)).getMillis();
if (gcDeletesInMillis != this.gcDeletesInMillis) {
logger.info("updating {} from [{}] to [{}]", IndexSettings.INDEX_GC_DELETES_SETTING, TimeValue.timeValueMillis(this.gcDeletesInMillis), TimeValue.timeValueMillis(gcDeletesInMillis));
@ -450,4 +417,31 @@ public final class IndexSettings {
return mergePolicyConfig.getMergePolicy();
}
boolean containsSetting(Setting<?> setting) {
return scopedSettings.get(setting.getKey()) != null;
}
public <T> T getValue(Setting<T> setting) {
return scopedSettings.get(setting);
}
private static final class ScopedSettings extends AbstractScopedSettings {
ScopedSettings(Settings settings, Settings scopeSettings, Set<Setting<?>> settingsSet) {
super(settings, scopeSettings, settingsSet, Setting.Scope.INDEX);
}
void addSettingInternal(Setting<?> settings) {
addSetting(settings);
}
}
void addSetting(Setting<?> setting) {
scopedSettings.addSettingInternal(setting);
}
<T> void addSettingsUpdateConsumer(Setting<T> setting, Consumer<T> consumer) {
scopedSettings.addSettingInternal(setting);
scopedSettings.addSettingsUpdateConsumer(setting, consumer);
}
}

18
core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.index;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.index.IndexSettings;
@ -52,18 +53,21 @@ import org.elasticsearch.index.IndexSettings;
*/
public final class MergeSchedulerConfig {
public static final String MAX_THREAD_COUNT = "index.merge.scheduler.max_thread_count";
public static final String MAX_MERGE_COUNT = "index.merge.scheduler.max_merge_count";
public static final String AUTO_THROTTLE = "index.merge.scheduler.auto_throttle";
public static final Setting<Integer> MAX_THREAD_COUNT_SETTING = new Setting<>("index.merge.scheduler.max_thread_count", (s) -> Integer.toString(Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(s) / 2))), (s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_thread_count"), true, Setting.Scope.INDEX);
public static final Setting<Integer> MAX_MERGE_COUNT_SETTING = new Setting<>("index.merge.scheduler.max_merge_count", (s) -> Integer.toString(MAX_THREAD_COUNT_SETTING.get(s) + 5), (s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_merge_count"), true, Setting.Scope.INDEX);
public static final Setting<Boolean> AUTO_THROTTLE_SETTING = Setting.boolSetting("index.merge.scheduler.auto_throttle", true, true, Setting.Scope.INDEX);
private volatile boolean autoThrottle;
private volatile int maxThreadCount;
private volatile int maxMergeCount;
MergeSchedulerConfig(Settings settings) {
maxThreadCount = settings.getAsInt(MAX_THREAD_COUNT, Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(settings) / 2)));
maxMergeCount = settings.getAsInt(MAX_MERGE_COUNT, maxThreadCount + 5);
this.autoThrottle = settings.getAsBoolean(AUTO_THROTTLE, true);
MergeSchedulerConfig(IndexSettings indexSettings) {
indexSettings.addSettingsUpdateConsumer(MAX_THREAD_COUNT_SETTING, this::setMaxThreadCount);
indexSettings.addSettingsUpdateConsumer(MAX_MERGE_COUNT_SETTING, this::setMaxMergeCount);
indexSettings.addSettingsUpdateConsumer(AUTO_THROTTLE_SETTING, this::setAutoThrottle);
maxThreadCount = indexSettings.getValue(MAX_THREAD_COUNT_SETTING);
maxMergeCount = indexSettings.getValue(MAX_MERGE_COUNT_SETTING);
this.autoThrottle = indexSettings.getValue(AUTO_THROTTLE_SETTING);
}
/**

50
core/src/main/java/org/elasticsearch/index/store/IndexStore.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.index.store;
import org.apache.lucene.store.StoreRateLimiting;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.index.AbstractIndexComponent;
@ -29,32 +30,18 @@ import org.elasticsearch.index.shard.ShardPath;
*
*/
public class IndexStore extends AbstractIndexComponent {
public static final String INDEX_STORE_THROTTLE_TYPE = "index.store.throttle.type";
public static final String INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC = "index.store.throttle.max_bytes_per_sec";
public static final Setting<StoreRateLimiting.Type> INDEX_STORE_THROTTLE_TYPE_SETTING = new Setting<>("index.store.throttle.type", "none", StoreRateLimiting.Type::fromString, true, Setting.Scope.INDEX) ;
public static final Setting<ByteSizeValue> INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting("index.store.throttle.max_bytes_per_sec", new ByteSizeValue(0), true, Setting.Scope.INDEX);
protected final IndexStoreConfig indexStoreConfig;
private volatile String rateLimitingType;
private volatile ByteSizeValue rateLimitingThrottle;
private volatile boolean nodeRateLimiting;
private final StoreRateLimiting rateLimiting = new StoreRateLimiting();
public IndexStore(IndexSettings indexSettings, IndexStoreConfig indexStoreConfig) {
super(indexSettings);
this.indexStoreConfig = indexStoreConfig;
this.rateLimitingType = indexSettings.getSettings().get(INDEX_STORE_THROTTLE_TYPE, "none");
if (rateLimitingType.equalsIgnoreCase("node")) {
nodeRateLimiting = true;
} else {
nodeRateLimiting = false;
rateLimiting.setType(rateLimitingType);
}
this.rateLimitingThrottle = indexSettings.getSettings().getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(0));
rateLimiting.setMaxRate(rateLimitingThrottle);
logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle);
rateLimiting.setType(indexSettings.getValue(INDEX_STORE_THROTTLE_TYPE_SETTING));
rateLimiting.setMaxRate(indexSettings.getValue(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING));
logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimiting.getType(), rateLimiting.getRateLimiter());
}
/**
@ -62,7 +49,7 @@ public class IndexStore extends AbstractIndexComponent {
* the node level one (defaults to the node level one).
*/
public StoreRateLimiting rateLimiting() {
return nodeRateLimiting ? indexStoreConfig.getNodeRateLimiter() : this.rateLimiting;
return rateLimiting.getType() != StoreRateLimiting.Type.NONE ? indexStoreConfig.getNodeRateLimiter() : this.rateLimiting;
}
/**
@ -72,26 +59,11 @@ public class IndexStore extends AbstractIndexComponent {
return new FsDirectoryService(indexSettings, this, path);
}
public void onRefreshSettings(Settings settings) {
String rateLimitingType = settings.get(INDEX_STORE_THROTTLE_TYPE, IndexStore.this.rateLimitingType);
if (!rateLimitingType.equals(IndexStore.this.rateLimitingType)) {
logger.info("updating index.store.throttle.type from [{}] to [{}]", IndexStore.this.rateLimitingType, rateLimitingType);
if (rateLimitingType.equalsIgnoreCase("node")) {
IndexStore.this.rateLimitingType = rateLimitingType;
IndexStore.this.nodeRateLimiting = true;
} else {
StoreRateLimiting.Type.fromString(rateLimitingType);
IndexStore.this.rateLimitingType = rateLimitingType;
IndexStore.this.nodeRateLimiting = false;
IndexStore.this.rateLimiting.setType(rateLimitingType);
}
public void setType(StoreRateLimiting.Type type) {
rateLimiting.setType(type);
}
ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndexStore.this.rateLimitingThrottle);
if (!rateLimitingThrottle.equals(IndexStore.this.rateLimitingThrottle)) {
logger.info("updating index.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndexStore.this.rateLimitingThrottle, rateLimitingThrottle, IndexStore.this.rateLimitingType);
IndexStore.this.rateLimitingThrottle = rateLimitingThrottle;
IndexStore.this.rateLimiting.setMaxRate(rateLimitingThrottle);
}
public void setMaxRate(ByteSizeValue rate) {
rateLimiting.setMaxRate(rate);
}
}

4
core/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -280,7 +280,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
}
final String indexName = indexMetaData.getIndex();
final Predicate<String> indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(indexName, indexExpression, clusterService.state());
final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, Collections.emptyList(), indexNameMatcher);
final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher);
Index index = new Index(indexMetaData.getIndex());
if (indices.containsKey(index.name())) {
throw new IndexAlreadyExistsException(index);
@ -570,7 +570,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
// play safe here and make sure that we take node level settings into account.
// we might run on nodes where we use shard FS and then in the future don't delete
// actual content.
return new IndexSettings(metaData, settings, Collections.emptyList());
return new IndexSettings(metaData, settings);
}
/**

2
core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
View File

@ -78,7 +78,7 @@ public final class AnalysisModule extends AbstractModule {
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.build();
IndexMetaData metaData = IndexMetaData.builder("_na_").settings(build).build();
NA_INDEX_SETTINGS = new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList());
NA_INDEX_SETTINGS = new IndexSettings(metaData, Settings.EMPTY);
}
private static final IndexSettings NA_INDEX_SETTINGS;
private final Environment environment;

2
core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
View File

@ -257,7 +257,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
} else {
final IndexMetaData metaData = previousState.metaData().index(index);
assert metaData != null;
indexSettings = new IndexSettings(metaData, settings, Collections.emptyList());
indexSettings = new IndexSettings(metaData, settings);
indicesService.deleteClosedIndex("closed index no longer part of the metadata", metaData, event.state());
}
try {

2
core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
View File

@ -111,7 +111,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
}
for (IndexRoutingTable indexRoutingTable : event.state().routingTable()) {
IndexSettings indexSettings = new IndexSettings(event.state().getMetaData().index(indexRoutingTable.index()), settings, Collections.emptyList());
IndexSettings indexSettings = new IndexSettings(event.state().getMetaData().index(indexRoutingTable.index()), settings);
// Note, closed indices will not have any routing information, so won't be deleted
for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
if (shardCanBeDeleted(event.state(), indexShardRoutingTable)) {

2
core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
View File

@ -175,7 +175,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
if (!storeType.contains("fs")) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
final IndexSettings indexSettings = indexService != null ? indexService.getIndexSettings() : new IndexSettings(metaData, settings, Collections.emptyList());
final IndexSettings indexSettings = indexService != null ? indexService.getIndexSettings() : new IndexSettings(metaData, settings);
final ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
if (shardPath == null) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);

23
core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.cache.recycler.PageCacheRecycler;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.env.Environment;
@ -173,13 +174,11 @@ public class IndexModuleTests extends ESTestCase {
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
Consumer<Settings> listener = (s) -> {};
module.addIndexSettingsListener(listener);
module.addIndexEventListener(eventListener);
IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry);
IndexSettings x = indexService.getIndexSettings();
assertEquals(x.getSettings().getAsMap(), indexSettings.getSettings().getAsMap());
assertEquals(x.getIndex(), index);
assertSame(x.getUpdateListeners().get(0), listener);
indexService.getIndexEventListener().beforeIndexDeleted(null);
assertTrue(atomicBoolean.get());
indexService.close("simon says", false);
@ -188,27 +187,29 @@ public class IndexModuleTests extends ESTestCase {
public void testListener() throws IOException {
IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
Consumer<Settings> listener = (s) -> {
};
module.addIndexSettingsListener(listener);
Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, true, Setting.Scope.INDEX);
Setting<Boolean> booleanSetting2 = Setting.boolSetting("foo.bar.baz", false, true, Setting.Scope.INDEX);
AtomicBoolean atomicBoolean = new AtomicBoolean(false);
module.addSetting(booleanSetting);
module.addSettingsUpdateConsumer(booleanSetting2, atomicBoolean::set);
try {
module.addIndexSettingsListener(listener);
module.addSettingsUpdateConsumer(booleanSetting, atomicBoolean::set);
fail("already added");
} catch (IllegalStateException ex) {
} catch (IllegalArgumentException ex) {
}
try {
module.addIndexSettingsListener(null);
fail("must not be null");
module.addSetting(booleanSetting2);
fail("already added");
} catch (IllegalArgumentException ex) {
}
IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry);
IndexSettings x = indexService.getIndexSettings();
assertEquals(1, x.getUpdateListeners().size());
assertSame(x.getUpdateListeners().get(0), listener);
x.containsSetting(booleanSetting);
x.containsSetting(booleanSetting2);
indexService.close("simon says", false);
}

63
core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
View File

@ -20,6 +20,8 @@ package org.elasticsearch.index;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.AbstractScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.test.ESTestCase;
@ -30,6 +32,7 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.function.Function;
public class IndexSettingsTests extends ESTestCase {
@ -38,13 +41,14 @@ public class IndexSettingsTests extends ESTestCase {
Version version = VersionUtils.getPreviousVersion();
Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build();
final AtomicInteger integer = new AtomicInteger(0);
Consumer<Settings> settingsConsumer = (s) -> integer.set(s.getAsInt("index.test.setting.int", -1));
Setting<Integer> integerSetting = Setting.intSetting("index.test.setting.int", -1, true, Setting.Scope.INDEX);
IndexMetaData metaData = newIndexMeta("index", theSettings);
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY, Collections.singleton(settingsConsumer));
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
settings.addSettingsUpdateConsumer(integerSetting, integer::set);
assertEquals(version, settings.getIndexVersionCreated());
assertEquals("0xdeadbeef", settings.getUUID());
assertEquals(1, settings.getUpdateListeners().size());
assertFalse(settings.updateIndexMetaData(metaData));
assertEquals(metaData.getSettings().getAsMap(), settings.getSettings().getAsMap());
assertEquals(0, integer.get());
@ -58,11 +62,12 @@ public class IndexSettingsTests extends ESTestCase {
.put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build();
final AtomicInteger integer = new AtomicInteger(0);
final StringBuilder builder = new StringBuilder();
Consumer<Settings> settingsConsumer = (s) -> {
integer.set(s.getAsInt("index.test.setting.int", -1));
builder.append(s.get("index.not.updated", ""));
};
IndexSettings settings = new IndexSettings(newIndexMeta("index", theSettings), Settings.EMPTY, Collections.singleton(settingsConsumer));
Setting<Integer> integerSetting = Setting.intSetting("index.test.setting.int", -1, true, Setting.Scope.INDEX);
Setting<String> notUpdated = new Setting<>("index.not.updated", "", Function.identity(), true, Setting.Scope.INDEX);
IndexSettings settings = new IndexSettings(newIndexMeta("index", theSettings), Settings.EMPTY);
settings.addSettingsUpdateConsumer(integerSetting, integer::set);
settings.addSettingsUpdateConsumer(notUpdated, builder::append);
assertEquals(0, integer.get());
assertEquals("", builder.toString());
IndexMetaData newMetaData = newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()).put("index.test.setting.int", 42).build());
@ -73,30 +78,14 @@ public class IndexSettingsTests extends ESTestCase {
integer.set(0);
assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()).put("index.not.updated", "boom").build())));
assertEquals("boom", builder.toString());
assertEquals(42, integer.get());
assertEquals("not updated - we preserve the old settings", 0, integer.get());
}
public void testListenerCanThrowException() {
Version version = VersionUtils.getPreviousVersion();
Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build();
final AtomicInteger integer = new AtomicInteger(0);
Consumer<Settings> settingsConsumer = (s) -> integer.set(s.getAsInt("index.test.setting.int", -1));
Consumer<Settings> exceptionConsumer = (s) -> {throw new RuntimeException("boom");};
List<Consumer<Settings>> list = new ArrayList<>();
list.add(settingsConsumer);
list.add(exceptionConsumer);
Collections.shuffle(list, random());
IndexSettings settings = new IndexSettings(newIndexMeta("index", theSettings), Settings.EMPTY, list);
assertEquals(0, integer.get());
assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(theSettings).put("index.test.setting.int", 42).build())));
assertEquals(42, integer.get());
}
public void testSettingsConsistency() {
Version version = VersionUtils.getPreviousVersion();
IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build());
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList());
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
assertEquals(version, settings.getIndexVersionCreated());
assertEquals("_na_", settings.getUUID());
try {
@ -107,7 +96,7 @@ public class IndexSettingsTests extends ESTestCase {
}
metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build());
settings = new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList());
settings = new IndexSettings(metaData, Settings.EMPTY);
try {
settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("index.test.setting.int", 42).build()));
fail("uuid missing/change");
@ -122,19 +111,18 @@ public class IndexSettingsTests extends ESTestCase {
final int numShards = randomIntBetween(1, 10);
final int numReplicas = randomIntBetween(0, 10);
Settings theSettings = Settings.settingsBuilder().
put("index.foo.bar", 42)
put("index.foo.bar", 0)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numReplicas)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards).build();
Settings nodeSettings = Settings.settingsBuilder().put("node.foo.bar", 43).build();
Settings nodeSettings = Settings.settingsBuilder().put("index.foo.bar", 43).build();
final AtomicInteger indexValue = new AtomicInteger(0);
final AtomicInteger nodeValue = new AtomicInteger(0);
Consumer<Settings> settingsConsumer = (s) -> {indexValue.set(s.getAsInt("index.foo.bar", -1)); nodeValue.set(s.getAsInt("node.foo.bar", -1));};
IndexSettings settings = new IndexSettings(newIndexMeta("index", theSettings), nodeSettings, Collections.singleton(settingsConsumer));
Setting<Integer> integerSetting = Setting.intSetting("index.foo.bar", -1, true, Setting.Scope.INDEX);
IndexSettings settings = new IndexSettings(newIndexMeta("index", theSettings), nodeSettings);
settings.addSettingsUpdateConsumer(integerSetting, indexValue::set);
assertEquals(numReplicas, settings.getNumberOfReplicas());
assertEquals(numShards, settings.getNumberOfShards());
assertEquals(0, indexValue.get());
assertEquals(0, nodeValue.get());
assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.settingsBuilder().
put("index.foo.bar", 42)
@ -142,9 +130,12 @@ public class IndexSettingsTests extends ESTestCase {
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards).build())));
assertEquals(42, indexValue.get());
assertEquals(43, nodeValue.get());
assertSame(nodeSettings, settings.getNodeSettings());
assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.settingsBuilder()
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numReplicas + 1)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards).build())));
assertEquals(43, indexValue.get());
}
@ -164,7 +155,7 @@ public class IndexSettingsTests extends ESTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY, "async")
.build());
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList());
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
assertEquals(Translog.Durability.ASYNC, settings.getTranslogDurability());
settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY, "request").build()));
assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability());
@ -172,7 +163,7 @@ public class IndexSettingsTests extends ESTestCase {
metaData = newIndexMeta("index", Settings.settingsBuilder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build());
settings = new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList());
settings = new IndexSettings(metaData, Settings.EMPTY);
assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability()); // test default
}

15
core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
View File

@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.settings.Validator;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
@ -29,6 +30,7 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import java.util.Collection;
import java.util.Collections;
import java.util.Set;
import java.util.function.Consumer;
import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
@ -68,8 +70,8 @@ public class SettingsListenerIT extends ESIntegTestCase {
@Override
public void onIndexModule(IndexModule module) {
if (module.getIndex().getName().equals("test")) { // only for the test index
module.addIndexSettingsListener(service);
service.accept(module.getSettings());
module.addSettingsUpdateConsumer(SettingsTestingService.VALUE, service::setValue);
service.setValue(SettingsTestingService.VALUE.get(module.getSettings()));
}
}
@ -92,13 +94,14 @@ public class SettingsListenerIT extends ESIntegTestCase {
}
}
public static class SettingsTestingService implements Consumer<Settings> {
public static class SettingsTestingService {
public volatile int value;
public static Setting<Integer> VALUE = Setting.intSetting("index.test.new.setting", -1, true, Setting.Scope.INDEX);
@Override
public void accept(Settings settings) {
value = settings.getAsInt("index.test.new.setting", -1);
public void setValue(int value) {
this.value = value;
}
}
public void testListener() {

3
core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
View File

@ -1600,8 +1600,7 @@ public class InternalEngineTests extends ESTestCase {
}
// now it should be OK.
IndexSettings indexSettings = new IndexSettings(defaultSettings.getIndexMetaData(),
Settings.builder().put(defaultSettings.getSettings()).put(EngineConfig.INDEX_FORCE_NEW_TRANSLOG, true).build(),
Collections.emptyList());
Settings.builder().put(defaultSettings.getSettings()).put(EngineConfig.INDEX_FORCE_NEW_TRANSLOG, true).build());
engine = createEngine(indexSettings, store, primaryTranslogDir, newMergePolicy());
}

2
core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
View File

@ -106,7 +106,7 @@ public class FieldDataCacheTests extends ESTestCase {
.numberOfReplicas(0)
.creationDate(System.currentTimeMillis())
.build();
return new IndexSettings(indexMetaData, settings, Collections.emptyList());
return new IndexSettings(indexMetaData, settings);
}
private class DummyAccountingFieldDataCache implements IndexFieldDataCache {

32
core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
View File

@ -142,13 +142,13 @@ public class UpdateSettingsIT extends ESIntegTestCase {
// No throttling at first, only 1 non-replicated shard, force lots of merging:
assertAcked(prepareCreate("test")
.setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "none")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "none")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT, "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT, "2")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "2")
.put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL, 0) // get stats all the time - no caching
));
ensureGreen();
@ -180,13 +180,13 @@ public class UpdateSettingsIT extends ESIntegTestCase {
.indices()
.prepareUpdateSettings("test")
.setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, "1mb"))
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), "1mb"))
.get();
// Make sure setting says it is in fact changed:
GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get();
assertThat(getSettingsResponse.getSetting("test", IndexStore.INDEX_STORE_THROTTLE_TYPE), equalTo("merge"));
assertThat(getSettingsResponse.getSetting("test", IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey()), equalTo("merge"));
// Also make sure we see throttling kicking in:
boolean done = false;
@ -220,7 +220,7 @@ public class UpdateSettingsIT extends ESIntegTestCase {
.indices()
.prepareUpdateSettings("test")
.setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "none"))
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "none"))
.get();
// Optimize does a waitForMerges, which we must do to make sure all in-flight (throttled) merges finish:
@ -312,9 +312,9 @@ public class UpdateSettingsIT extends ESIntegTestCase {
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT, "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT, "2")
.put(MergeSchedulerConfig.AUTO_THROTTLE, "true")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "2")
.put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), "true")
));
// Disable auto throttle:
@ -323,7 +323,7 @@ public class UpdateSettingsIT extends ESIntegTestCase {
.indices()
.prepareUpdateSettings("test")
.setSettings(Settings.builder()
.put(MergeSchedulerConfig.AUTO_THROTTLE, "no"))
.put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), "no"))
.get();
// Make sure we log the change:
@ -331,7 +331,7 @@ public class UpdateSettingsIT extends ESIntegTestCase {
// Make sure setting says it is in fact changed:
GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get();
assertThat(getSettingsResponse.getSetting("test", MergeSchedulerConfig.AUTO_THROTTLE), equalTo("no"));
assertThat(getSettingsResponse.getSetting("test", MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey()), equalTo("no"));
} finally {
rootLogger.removeAppender(mockAppender);
rootLogger.setLevel(savedLevel);
@ -354,8 +354,8 @@ public class UpdateSettingsIT extends ESIntegTestCase {
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT, "10000")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT, "10000")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "10000")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "10000")
));
assertFalse(mockAppender.sawUpdateMaxThreadCount);
@ -366,7 +366,7 @@ public class UpdateSettingsIT extends ESIntegTestCase {
.indices()
.prepareUpdateSettings("test")
.setSettings(Settings.builder()
.put(MergeSchedulerConfig.MAX_THREAD_COUNT, "1")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
)
.get();
@ -375,7 +375,7 @@ public class UpdateSettingsIT extends ESIntegTestCase {
// Make sure setting says it is in fact changed:
GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get();
assertThat(getSettingsResponse.getSetting("test", MergeSchedulerConfig.MAX_THREAD_COUNT), equalTo("1"));
assertThat(getSettingsResponse.getSetting("test", MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey()), equalTo("1"));
} finally {
rootLogger.removeAppender(mockAppender);

12
core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
View File

@ -277,13 +277,13 @@ public class IndexStatsIT extends ESIntegTestCase {
public void testNonThrottleStats() throws Exception {
assertAcked(prepareCreate("test")
.setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT, "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT, "10000")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "10000")
));
ensureGreen();
long termUpto = 0;
@ -309,13 +309,13 @@ public class IndexStatsIT extends ESIntegTestCase {
public void testThrottleStats() throws Exception {
assertAcked(prepareCreate("test")
.setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2")
.put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT, "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT, "1")
.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "1")
.put("index.merge.policy.type", "tiered")
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY, Translog.Durability.ASYNC.name())
));

4
core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java
View File

@ -77,9 +77,7 @@ public class PercolateDocumentParserTests extends ESTestCase {
Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
.build(),
Settings.EMPTY, Collections.emptyList()
);
.build(), Settings.EMPTY);
AnalysisService analysisService = new AnalysisService(indexSettings, Collections.<String, AnalyzerProvider>emptyMap(), Collections.<String, TokenizerFactory>emptyMap(), Collections.<String, CharFilterFactory>emptyMap(), Collections.<String, TokenFilterFactory>emptyMap());
IndicesModule indicesModule = new IndicesModule();
mapperService = new MapperService(indexSettings, analysisService, new SimilarityService(indexSettings, Collections.emptyMap()), indicesModule.getMapperRegistry(), () -> null);

4
core/src/test/java/org/elasticsearch/percolator/PercolatorServiceTests.java
View File

@ -163,9 +163,7 @@ public class PercolatorServiceTests extends ESTestCase {
Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
.build(),
Settings.EMPTY, Collections.emptyList()
);
.build(), Settings.EMPTY);
return new PercolatorQueriesRegistry(
new ShardId(index, 0),
indexSettings,

7
core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
View File

@ -766,7 +766,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
logger.info("--> update index settings to back to normal");
assertAcked(client().admin().indices().prepareUpdateSettings("test-*").setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "node")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "node")
));
// Make sure that snapshot finished - doesn't matter if it failed or succeeded
@ -888,12 +888,11 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
}
assertAcked(client().admin().indices().prepareUpdateSettings(name).setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "all")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, between(100, 50000))
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "all")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), between(100, 50000))
));
}
static {
MetaData.registerPrototype(SnapshottableMetadata.TYPE, SnapshottableMetadata.PROTO);
MetaData.registerPrototype(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.PROTO);

6
core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
View File

@ -1524,8 +1524,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
// Update settings to make sure that relocation is slow so we can start snapshot before relocation is finished
assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "all")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, 100, ByteSizeUnit.BYTES)
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "all")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), 100, ByteSizeUnit.BYTES)
));
logger.info("--> start relocations");
@ -1540,7 +1540,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
// Update settings to back to normal
assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(Settings.builder()
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "node")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "node")
));
logger.info("--> wait for snapshot to complete");

6
test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
View File

@ -465,7 +465,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
setRandomIndexNormsLoading(random, builder);
if (random.nextBoolean()) {
builder.put(MergeSchedulerConfig.AUTO_THROTTLE, false);
builder.put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), false);
}
if (random.nextBoolean()) {
@ -493,8 +493,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
case 3:
final int maxThreadCount = RandomInts.randomIntBetween(random, 1, 4);
final int maxMergeCount = RandomInts.randomIntBetween(random, maxThreadCount, maxThreadCount + 4);
builder.put(MergeSchedulerConfig.MAX_MERGE_COUNT, maxMergeCount);
builder.put(MergeSchedulerConfig.MAX_THREAD_COUNT, maxThreadCount);
builder.put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), maxMergeCount);
builder.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), maxThreadCount);
break;
}

2
test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java
View File

@ -54,6 +54,6 @@ public class IndexSettingsModule extends AbstractModule {
.put(settings)
.build();
IndexMetaData metaData = IndexMetaData.builder(index.getName()).settings(build).build();
return new IndexSettings(metaData, Settings.EMPTY, Collections.emptyList());
return new IndexSettings(metaData, Settings.EMPTY);
}
}

2
test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
View File

@ -181,7 +181,7 @@ public class MockFSDirectoryService extends FsDirectoryService {
private FsDirectoryService randomDirectorService(IndexStore indexStore, ShardPath path) {
final IndexSettings indexSettings = indexStore.getIndexSettings();
final IndexMetaData build = IndexMetaData.builder(indexSettings.getIndexMetaData()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexModule.STORE_TYPE, RandomPicks.randomFrom(random, IndexModule.Type.values()).getSettingsKey())).build();
final IndexSettings newIndexSettings = new IndexSettings(build, indexSettings.getNodeSettings(), Collections.emptyList());
final IndexSettings newIndexSettings = new IndexSettings(build, indexSettings.getNodeSettings());
return new FsDirectoryService(newIndexSettings, indexStore, path);
}