convert index.translog.durability

This commit is contained in:
Simon Willnauer 2016-01-14 14:56:08 +01:00
parent da0424c75f
commit b7b104a585
8 changed files with 20 additions and 32 deletions

View File

@ -171,7 +171,6 @@ public class ClusterModule extends AbstractModule {
registerIndexDynamicSetting(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT, Validator.NON_NEGATIVE_DOUBLE); registerIndexDynamicSetting(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT, Validator.NON_NEGATIVE_DOUBLE);
registerIndexDynamicSetting(MergePolicyConfig.INDEX_COMPOUND_FORMAT, Validator.EMPTY); registerIndexDynamicSetting(MergePolicyConfig.INDEX_COMPOUND_FORMAT, Validator.EMPTY);
registerIndexDynamicSetting(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, Validator.BYTES_SIZE); registerIndexDynamicSetting(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, Validator.BYTES_SIZE);
registerIndexDynamicSetting(IndexSettings.INDEX_TRANSLOG_DURABILITY, Validator.EMPTY);
registerIndexDynamicSetting(IndicesWarmer.INDEX_WARMER_ENABLED, Validator.EMPTY); registerIndexDynamicSetting(IndicesWarmer.INDEX_WARMER_ENABLED, Validator.EMPTY);
registerIndexDynamicSetting(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED, Validator.BOOLEAN); registerIndexDynamicSetting(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED, Validator.BOOLEAN);
registerIndexDynamicSetting(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, Validator.TIME); registerIndexDynamicSetting(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, Validator.TIME);

View File

@ -59,7 +59,7 @@ public final class IndexSettings {
public static final String QUERY_STRING_ALLOW_LEADING_WILDCARD = "indices.query.query_string.allowLeadingWildcard"; public static final String QUERY_STRING_ALLOW_LEADING_WILDCARD = "indices.query.query_string.allowLeadingWildcard";
public static final String ALLOW_UNMAPPED = "index.query.parse.allow_unmapped_fields"; public static final String ALLOW_UNMAPPED = "index.query.parse.allow_unmapped_fields";
public static final String INDEX_TRANSLOG_SYNC_INTERVAL = "index.translog.sync_interval"; public static final String INDEX_TRANSLOG_SYNC_INTERVAL = "index.translog.sync_interval";
public static final String INDEX_TRANSLOG_DURABILITY = "index.translog.durability"; public static final Setting<Translog.Durability> INDEX_TRANSLOG_DURABILITY_SETTING = new Setting<>("index.translog.durability", Translog.Durability.REQUEST.name(), (value) -> Translog.Durability.valueOf(value.toUpperCase(Locale.ROOT)), true, Setting.Scope.INDEX);
public static final String INDEX_REFRESH_INTERVAL = "index.refresh_interval"; public static final String INDEX_REFRESH_INTERVAL = "index.refresh_interval";
public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS); public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE = "index.translog.flush_threshold_size"; public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE = "index.translog.flush_threshold_size";
@ -103,7 +103,8 @@ public final class IndexSettings {
IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING,
MergeSchedulerConfig.AUTO_THROTTLE_SETTING, MergeSchedulerConfig.AUTO_THROTTLE_SETTING,
MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING,
IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING
))); )));
/** /**
@ -180,8 +181,8 @@ public final class IndexSettings {
this.parseFieldMatcher = new ParseFieldMatcher(settings); this.parseFieldMatcher = new ParseFieldMatcher(settings);
this.defaultAllowUnmappedFields = settings.getAsBoolean(ALLOW_UNMAPPED, true); this.defaultAllowUnmappedFields = settings.getAsBoolean(ALLOW_UNMAPPED, true);
this.indexNameMatcher = indexNameMatcher; this.indexNameMatcher = indexNameMatcher;
final String value = settings.get(INDEX_TRANSLOG_DURABILITY, Translog.Durability.REQUEST.name()); this.durability = scopedSettings.get(INDEX_TRANSLOG_DURABILITY_SETTING);
this.durability = getFromSettings(settings, Translog.Durability.REQUEST); scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_DURABILITY_SETTING, this::setTranslogDurability);
syncInterval = settings.getAsTime(INDEX_TRANSLOG_SYNC_INTERVAL, TimeValue.timeValueSeconds(5)); syncInterval = settings.getAsTime(INDEX_TRANSLOG_SYNC_INTERVAL, TimeValue.timeValueSeconds(5));
refreshInterval = settings.getAsTime(INDEX_REFRESH_INTERVAL, DEFAULT_REFRESH_INTERVAL); refreshInterval = settings.getAsTime(INDEX_REFRESH_INTERVAL, DEFAULT_REFRESH_INTERVAL);
flushThresholdSize = settings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB)); flushThresholdSize = settings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB));
@ -191,6 +192,10 @@ public final class IndexSettings {
assert indexNameMatcher.test(indexMetaData.getIndex()); assert indexNameMatcher.test(indexMetaData.getIndex());
} }
private void setTranslogDurability(Translog.Durability durability) {
this.durability = durability;
}
/** /**
* Returns the settings for this index. These settings contain the node and index level settings where * Returns the settings for this index. These settings contain the node and index level settings where
* settings that are specified on both index and node level are overwritten by the index settings. * settings that are specified on both index and node level are overwritten by the index settings.
@ -340,23 +345,8 @@ public final class IndexSettings {
return durability; return durability;
} }
private Translog.Durability getFromSettings(Settings settings, Translog.Durability defaultValue) {
final String value = settings.get(INDEX_TRANSLOG_DURABILITY, defaultValue.name());
try {
return Translog.Durability.valueOf(value.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException ex) {
logger.warn("Can't apply {} illegal value: {} using {} instead, use one of: {}", INDEX_TRANSLOG_DURABILITY, value, defaultValue, Arrays.toString(Translog.Durability.values()));
return defaultValue;
}
}
private void updateSettings(Settings settings) { private void updateSettings(Settings settings) {
final Translog.Durability durability = getFromSettings(settings, this.durability);
if (durability != this.durability) {
logger.info("updating durability from [{}] to [{}]", this.durability, durability);
this.durability = durability;
}
TimeValue refreshInterval = settings.getAsTime(IndexSettings.INDEX_REFRESH_INTERVAL, this.refreshInterval); TimeValue refreshInterval = settings.getAsTime(IndexSettings.INDEX_REFRESH_INTERVAL, this.refreshInterval);
if (!refreshInterval.equals(this.refreshInterval)) { if (!refreshInterval.equals(this.refreshInterval)) {
logger.info("updating refresh_interval from [{}] to [{}]", this.refreshInterval, refreshInterval); logger.info("updating refresh_interval from [{}] to [{}]", this.refreshInterval, refreshInterval);

View File

@ -314,7 +314,7 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
public void testAsyncFsyncActuallyWorks() throws Exception { public void testAsyncFsyncActuallyWorks() throws Exception {
Settings settings = Settings.builder() Settings settings = Settings.builder()
.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL, "10ms") // very often :) .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL, "10ms") // very often :)
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY, Translog.Durability.ASYNC) .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC)
.build(); .build();
IndexService indexService = createIndex("test", settings); IndexService indexService = createIndex("test", settings);
ensureGreen("test"); ensureGreen("test");

View File

@ -153,11 +153,11 @@ public class IndexSettingsTests extends ESTestCase {
public void testUpdateDurability() { public void testUpdateDurability() {
IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder() IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY, "async") .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "async")
.build()); .build());
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
assertEquals(Translog.Durability.ASYNC, settings.getTranslogDurability()); assertEquals(Translog.Durability.ASYNC, settings.getTranslogDurability());
settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY, "request").build())); settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "request").build()));
assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability()); assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability());
metaData = newIndexMeta("index", Settings.settingsBuilder() metaData = newIndexMeta("index", Settings.settingsBuilder()
@ -167,5 +167,4 @@ public class IndexSettingsTests extends ESTestCase {
assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability()); // test default assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability()); // test default
} }
} }

View File

@ -400,7 +400,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
} }
private void setDurability(IndexShard shard, Translog.Durability durability) { private void setDurability(IndexShard shard, Translog.Durability durability) {
client().admin().indices().prepareUpdateSettings(shard.shardId.getIndex()).setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY, durability.name()).build()).get(); client().admin().indices().prepareUpdateSettings(shard.shardId.getIndex()).setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), durability.name()).build()).get();
assertEquals(durability, shard.getTranslogDurability()); assertEquals(durability, shard.getTranslogDurability());
} }
@ -692,7 +692,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
} }
public void testMaybeFlush() throws Exception { public void testMaybeFlush() throws Exception {
createIndex("test", settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY, Translog.Durability.REQUEST).build()); createIndex("test", settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST).build());
ensureGreen(); ensureGreen();
IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndicesService indicesService = getInstanceFromNode(IndicesService.class);
IndexService test = indicesService.indexService("test"); IndexService test = indicesService.indexService("test");

View File

@ -317,7 +317,7 @@ public class IndexStatsIT extends ESIntegTestCase {
.put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1") .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
.put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "1") .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "1")
.put("index.merge.policy.type", "tiered") .put("index.merge.policy.type", "tiered")
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY, Translog.Durability.ASYNC.name()) .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC.name())
)); ));
ensureGreen(); ensureGreen();
long termUpto = 0; long termUpto = 0;

View File

@ -55,7 +55,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception { public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception {
logger.info("--> creating test index ..."); logger.info("--> creating test index ...");
int numberOfShards = numberOfShards(); int numberOfShards = numberOfShards();
assertAcked(prepareCreate("test", 1, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY, Translog.Durability.ASYNC))); assertAcked(prepareCreate("test", 1, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC)));
final int totalNumDocs = scaledRandomIntBetween(200, 10000); final int totalNumDocs = scaledRandomIntBetween(200, 10000);
int waitFor = totalNumDocs / 10; int waitFor = totalNumDocs / 10;
@ -108,7 +108,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
public void testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() throws Exception { public void testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() throws Exception {
logger.info("--> creating test index ..."); logger.info("--> creating test index ...");
int numberOfShards = numberOfShards(); int numberOfShards = numberOfShards();
assertAcked(prepareCreate("test", 1, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY, Translog.Durability.ASYNC))); assertAcked(prepareCreate("test", 1, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC)));
final int totalNumDocs = scaledRandomIntBetween(200, 10000); final int totalNumDocs = scaledRandomIntBetween(200, 10000);
int waitFor = totalNumDocs / 10; int waitFor = totalNumDocs / 10;
@ -159,7 +159,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception { public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception {
logger.info("--> creating test index ..."); logger.info("--> creating test index ...");
int numberOfShards = numberOfShards(); int numberOfShards = numberOfShards();
assertAcked(prepareCreate("test", 2, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY, Translog.Durability.ASYNC))); assertAcked(prepareCreate("test", 2, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC)));
final int totalNumDocs = scaledRandomIntBetween(200, 10000); final int totalNumDocs = scaledRandomIntBetween(200, 10000);
int waitFor = totalNumDocs / 10; int waitFor = totalNumDocs / 10;
@ -230,7 +230,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
final int numReplicas = 0; final int numReplicas = 0;
logger.info("--> creating test index ..."); logger.info("--> creating test index ...");
int allowNodes = 2; int allowNodes = 2;
assertAcked(prepareCreate("test", 3, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(SETTING_NUMBER_OF_REPLICAS, numReplicas).put(IndexSettings.INDEX_TRANSLOG_DURABILITY, Translog.Durability.ASYNC))); assertAcked(prepareCreate("test", 3, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(SETTING_NUMBER_OF_REPLICAS, numReplicas).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC)));
final int numDocs = scaledRandomIntBetween(200, 9999); final int numDocs = scaledRandomIntBetween(200, 9999);

View File

@ -516,7 +516,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(1, ByteSizeUnit.PB)); // just don't flush builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(1, ByteSizeUnit.PB)); // just don't flush
} }
if (random.nextBoolean()) { if (random.nextBoolean()) {
builder.put(IndexSettings.INDEX_TRANSLOG_DURABILITY, RandomPicks.randomFrom(random, Translog.Durability.values())); builder.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), RandomPicks.randomFrom(random, Translog.Durability.values()));
} }
if (random.nextBoolean()) { if (random.nextBoolean()) {