honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[TEST] Randomize translog setting per index

This commit is contained in:
Simon Willnauer 2014-04-22 14:47:47 +02:00
parent 1cf62e7782
commit cb9f7c1da5
2 changed files with 44 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/org/elasticsearch/index/translog/TranslogService.java
View File

@ -45,6 +45,18 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
*/ */
public class TranslogService extends AbstractIndexShardComponent { public class TranslogService extends AbstractIndexShardComponent {
private static final String FLUSH_THRESHOLD_OPS_KEY = "flush_threshold_ops";
private static final String FLUSH_THRESHOLD_SIZE_KEY = "flush_threshold_size";
private static final String FLUSH_THRESHOLD_PERIOD_KEY = "flush_threshold_period";
private static final String FLUSH_THRESHOLD_DISABLE_FLUSH_KEY = "disable_flush";
private static final String FLUSH_THRESHOLD_INTERVAL_KEY = "interval";
public static final String INDEX_TRANSLOG_FLUSH_INTERVAL = "index.translog." + FLUSH_THRESHOLD_INTERVAL_KEY;
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS = "index.translog." + FLUSH_THRESHOLD_OPS_KEY;
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE = "index.translog." + FLUSH_THRESHOLD_SIZE_KEY;
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_PERIOD = "index.translog." + FLUSH_THRESHOLD_PERIOD_KEY;
public static final String INDEX_TRANSLOG_DISABLE_FLUSH = "index.translog." + FLUSH_THRESHOLD_DISABLE_FLUSH_KEY;
private final ThreadPool threadPool; private final ThreadPool threadPool;
private final IndexSettingsService indexSettingsService; private final IndexSettingsService indexSettingsService;
private final IndexShard indexShard; private final IndexShard indexShard;
@ -67,11 +79,11 @@ public class TranslogService extends AbstractIndexShardComponent {
this.indexShard = indexShard; this.indexShard = indexShard;
this.translog = translog; this.translog = translog;
this.flushThresholdOperations = componentSettings.getAsInt("flush_threshold_ops", componentSettings.getAsInt("flush_threshold", Integer.MAX_VALUE)); this.flushThresholdOperations = componentSettings.getAsInt(FLUSH_THRESHOLD_OPS_KEY, componentSettings.getAsInt("flush_threshold", Integer.MAX_VALUE));
this.flushThresholdSize = componentSettings.getAsBytesSize("flush_threshold_size", new ByteSizeValue(200, ByteSizeUnit.MB)); this.flushThresholdSize = componentSettings.getAsBytesSize(FLUSH_THRESHOLD_SIZE_KEY, new ByteSizeValue(200, ByteSizeUnit.MB));
this.flushThresholdPeriod = componentSettings.getAsTime("flush_threshold_period", TimeValue.timeValueMinutes(30)); this.flushThresholdPeriod = componentSettings.getAsTime(FLUSH_THRESHOLD_PERIOD_KEY, TimeValue.timeValueMinutes(30));
this.interval = componentSettings.getAsTime("interval", timeValueMillis(5000)); this.interval = componentSettings.getAsTime(FLUSH_THRESHOLD_INTERVAL_KEY, timeValueMillis(5000));
this.disableFlush = componentSettings.getAsBoolean("disable_flush", false); this.disableFlush = componentSettings.getAsBoolean(FLUSH_THRESHOLD_DISABLE_FLUSH_KEY, false);
logger.debug("interval [{}], flush_threshold_ops [{}], flush_threshold_size [{}], flush_threshold_period [{}]", interval, flushThresholdOperations, flushThresholdSize, flushThresholdPeriod); logger.debug("interval [{}], flush_threshold_ops [{}], flush_threshold_size [{}], flush_threshold_period [{}]", interval, flushThresholdOperations, flushThresholdSize, flushThresholdPeriod);
@ -86,11 +98,7 @@ public class TranslogService extends AbstractIndexShardComponent {
this.future.cancel(true); this.future.cancel(true);
} }
public static final String INDEX_TRANSLOG_FLUSH_INTERVAL = "index.translog.interval";
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS = "index.translog.flush_threshold_ops";
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE = "index.translog.flush_threshold_size";
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_PERIOD = "index.translog.flush_threshold_period";
public static final String INDEX_TRANSLOG_DISABLE_FLUSH = "index.translog.disable_flush";
class ApplySettings implements IndexSettingsService.Listener { class ApplySettings implements IndexSettingsService.Listener {
@Override @Override

26
src/test/java/org/elasticsearch/test/ImmutableTestCluster.java
View File

@ -31,12 +31,16 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.merge.policy.*; import org.elasticsearch.index.merge.policy.*;
import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider; import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule; import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule;
import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider; import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider;
import org.elasticsearch.index.merge.scheduler.SerialMergeSchedulerProvider; import org.elasticsearch.index.merge.scheduler.SerialMergeSchedulerProvider;
import org.elasticsearch.index.translog.TranslogService;
import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndexTemplateMissingException;
import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryMissingException;
@ -223,7 +227,8 @@ public abstract class ImmutableTestCluster implements Iterable<Client> {
public void randomIndexTemplate() { public void randomIndexTemplate() {
// TODO move settings for random directory etc here into the index based randomized settings. // TODO move settings for random directory etc here into the index based randomized settings.
if (size() > 0) { if (size() > 0) {
ImmutableSettings.Builder builder = setRandomNormsLoading(setRandomMerge(random, ImmutableSettings.builder()) ImmutableSettings.Builder builder =
setRandomTranslogSettings(random, setRandomNormsLoading(setRandomMerge(random, ImmutableSettings.builder()))
.put(SETTING_INDEX_SEED, random.nextLong())) .put(SETTING_INDEX_SEED, random.nextLong()))
.put(SETTING_NUMBER_OF_SHARDS, RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_SHARDS, DEFAULT_MAX_NUM_SHARDS)) .put(SETTING_NUMBER_OF_SHARDS, RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_SHARDS, DEFAULT_MAX_NUM_SHARDS))
.put(SETTING_NUMBER_OF_REPLICAS, RandomInts.randomIntBetween(random, 0, 1)); .put(SETTING_NUMBER_OF_REPLICAS, RandomInts.randomIntBetween(random, 0, 1));
@ -242,6 +247,25 @@ public abstract class ImmutableTestCluster implements Iterable<Client> {
return builder; return builder;
} }
private static ImmutableSettings.Builder setRandomTranslogSettings(Random random, ImmutableSettings.Builder builder) {
if (random.nextBoolean()) {
builder.put(TranslogService.INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, RandomInts.randomIntBetween(random, 1, 10000));
}
if (random.nextBoolean()) {
builder.put(TranslogService.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 300), ByteSizeUnit.MB));
}
if (random.nextBoolean()) {
builder.put(TranslogService.INDEX_TRANSLOG_FLUSH_THRESHOLD_PERIOD, TimeValue.timeValueMinutes(RandomInts.randomIntBetween(random, 1, 60)));
}
if (random.nextBoolean()) {
builder.put(TranslogService.INDEX_TRANSLOG_FLUSH_INTERVAL, TimeValue.timeValueMillis(RandomInts.randomIntBetween(random, 1, 10000)));
}
if (random.nextBoolean()) {
builder.put(TranslogService.INDEX_TRANSLOG_DISABLE_FLUSH, random.nextBoolean());
}
return builder;
}
private static ImmutableSettings.Builder setRandomMerge(Random random, ImmutableSettings.Builder builder) { private static ImmutableSettings.Builder setRandomMerge(Random random, ImmutableSettings.Builder builder) {
if (random.nextBoolean()) { if (random.nextBoolean()) {
builder.put(AbstractMergePolicyProvider.INDEX_COMPOUND_FORMAT, builder.put(AbstractMergePolicyProvider.INDEX_COMPOUND_FORMAT,