honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 10:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Set randomized node/index settings in the right place

Browse Source 
Don't set node settings in the index template, and try and set less index settings in the node settings
closes #11767
This commit is contained in:
Shay Banon 2015-06-18 22:00:07 +02:00
parent 3e0bd500fc
commit 15d799361b
4 changed files with 62 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffTests.java
View File

@ -456,7 +456,7 @@ public class ClusterStateDiffTests extends ElasticsearchIntegrationTest {
public IndexMetaData randomCreate(String name) {
IndexMetaData.Builder builder = IndexMetaData.builder(name);
Settings.Builder settingsBuilder = Settings.builder();
setRandomSettings(getRandom(), settingsBuilder);
setRandomIndexSettings(getRandom(), settingsBuilder);
settingsBuilder.put(randomSettings(Settings.EMPTY)).put(IndexMetaData.SETTING_VERSION_CREATED, randomVersion(random()));
builder.settings(settingsBuilder);
builder.numberOfShards(randomIntBetween(1, 10)).numberOfReplicas(randomInt(10));

2
core/src/test/java/org/elasticsearch/test/ElasticsearchBackwardsCompatIntegrationTest.java
View File

@ -133,7 +133,7 @@ public abstract class ElasticsearchBackwardsCompatIntegrationTest extends Elasti
}
@Override
protected Settings.Builder setRandomSettings(Random random, Settings.Builder builder) {
protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builder builder) {
if (globalCompatibilityVersion().before(Version.V_1_3_2)) {
// if we test against nodes before 1.3.2 we disable all the compression due to a known bug
// see #7210

75
core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java
View File

@ -355,7 +355,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
// TODO move settings for random directory etc here into the index based randomized settings.
if (cluster().size() > 0) {
Settings.Builder randomSettingsBuilder =
setRandomSettings(getRandom(), Settings.builder())
setRandomIndexSettings(getRandom(), Settings.builder())
.put(SETTING_INDEX_SEED, getRandom().nextLong());
randomSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, numberOfShards())
@ -441,6 +441,10 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
mappings.endObject().endObject();
}
for (String setting : randomSettingsBuilder.internalMap().keySet()) {
assertThat("non index. prefix setting set on index template, its a node setting...", setting, startsWith("index."));
}
PutIndexTemplateRequestBuilder putTemplate = client().admin().indices()
.preparePutTemplate("random_index_template")
.setTemplate("*")
@ -454,42 +458,15 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
}
}
protected Settings.Builder setRandomSettings(Random random, Settings.Builder builder) {
setRandomMerge(random, builder);
setRandomTranslogSettings(random, builder);
setRandomNormsLoading(random, builder);
setRandomScriptingSettings(random, builder);
if (random.nextBoolean()) {
if (random.nextInt(10) == 0) { // do something crazy slow here
builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
} else {
builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
}
}
if (random.nextBoolean()) {
builder.put(IndicesStore.INDICES_STORE_THROTTLE_TYPE, RandomPicks.randomFrom(random, StoreRateLimiting.Type.values()));
}
protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builder builder) {
setRandomIndexMergeSettings(random, builder);
setRandomIndexTranslogSettings(random, builder);
setRandomIndexNormsLoading(random, builder);
if (random.nextBoolean()) {
builder.put(MergeSchedulerConfig.AUTO_THROTTLE, false);
}
if (random.nextBoolean()) {
if (random.nextInt(10) == 0) { // do something crazy slow here
builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
} else {
builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
}
}
if (random.nextBoolean()) {
builder.put(RecoverySettings.INDICES_RECOVERY_COMPRESS, random.nextBoolean());
}
if (random.nextBoolean()) {
builder.put(TranslogConfig.INDEX_TRANSLOG_FS_TYPE, RandomPicks.randomFrom(random, TranslogWriter.Type.values()).name());
}
if (random.nextBoolean()) {
builder.put(IndicesQueryCache.INDEX_CACHE_QUERY_ENABLED, random.nextBoolean());
}
@ -498,14 +475,6 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
builder.put("index.shard.check_on_startup", randomFrom(random, "false", "checksum", "true"));
}
if (random.nextBoolean()) {
builder.put(IndicesQueryCache.INDICES_CACHE_QUERY_CONCURRENCY_LEVEL, RandomInts.randomIntBetween(random, 1, 32));
builder.put(IndicesFieldDataCache.FIELDDATA_CACHE_CONCURRENCY_LEVEL, RandomInts.randomIntBetween(random, 1, 32));
}
if (random.nextBoolean()) {
builder.put(NettyTransport.PING_SCHEDULE, RandomInts.randomIntBetween(random, 100, 2000) + "ms");
}
if (randomBoolean()) {
// keep this low so we don't stall tests
builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, RandomInts.randomIntBetween(random, 1, 15) + "ms");
@ -514,17 +483,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
return builder;
}
private static Settings.Builder setRandomScriptingSettings(Random random, Settings.Builder builder) {
if (random.nextBoolean()) {
builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING, RandomInts.randomIntBetween(random, -100, 2000));
}
if (random.nextBoolean()) {
builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING, TimeValue.timeValueMillis(RandomInts.randomIntBetween(random, 750, 10000000)));
}
return builder;
}
private static Settings.Builder setRandomMerge(Random random, Settings.Builder builder) {
private static Settings.Builder setRandomIndexMergeSettings(Random random, Settings.Builder builder) {
if (random.nextBoolean()) {
builder.put(MergePolicyConfig.INDEX_COMPOUND_FORMAT,
random.nextBoolean() ? random.nextDouble() : random.nextBoolean());
@ -541,14 +500,14 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
return builder;
}
private static Settings.Builder setRandomNormsLoading(Random random, Settings.Builder builder) {
private static Settings.Builder setRandomIndexNormsLoading(Random random, Settings.Builder builder) {
if (random.nextBoolean()) {
builder.put(SearchService.NORMS_LOADING_KEY, RandomPicks.randomFrom(random, Arrays.asList(MappedFieldType.Loading.EAGER, MappedFieldType.Loading.LAZY)));
}
return builder;
}
private static Settings.Builder setRandomTranslogSettings(Random random, Settings.Builder builder) {
private static Settings.Builder setRandomIndexTranslogSettings(Random random, Settings.Builder builder) {
if (random.nextBoolean()) {
builder.put(TranslogService.INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, RandomInts.randomIntBetween(random, 1, 10000));
}
@ -567,6 +526,16 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
if (random.nextBoolean()) {
builder.put(TranslogConfig.INDEX_TRANSLOG_DURABILITY, RandomPicks.randomFrom(random, Translog.Durabilty.values()));
}
if (random.nextBoolean()) {
builder.put(TranslogConfig.INDEX_TRANSLOG_FS_TYPE, RandomPicks.randomFrom(random, TranslogWriter.Type.values()));
if (rarely(random)) {
builder.put(TranslogConfig.INDEX_TRANSLOG_SYNC_INTERVAL, 0); // 0 has special meaning to sync each op
} else {
builder.put(TranslogConfig.INDEX_TRANSLOG_SYNC_INTERVAL, RandomInts.randomIntBetween(random, 100, 5000), TimeUnit.MILLISECONDS);
}
}
return builder;
}

42
core/src/test/java/org/elasticsearch/test/InternalTestCluster.java
View File

@ -30,6 +30,7 @@ import com.google.common.collect.*;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import org.apache.lucene.store.StoreRateLimiting;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
@ -63,6 +64,7 @@ import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.BigArraysModule;
import org.elasticsearch.common.util.concurrent.EsExecutors;
@ -85,12 +87,15 @@ import org.elasticsearch.index.translog.TranslogWriter;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
import org.elasticsearch.indices.cache.query.IndicesQueryCache;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.indices.store.IndicesStore;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
import org.elasticsearch.node.service.NodeService;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.search.SearchServiceModule;
import org.elasticsearch.test.cache.recycler.MockBigArraysModule;
@ -444,13 +449,42 @@ public final class InternalTestCluster extends TestCluster {
}
if (random.nextBoolean()) {
builder.put(TranslogConfig.INDEX_TRANSLOG_FS_TYPE, RandomPicks.randomFrom(random, TranslogWriter.Type.values()));
if (rarely(random)) {
builder.put(TranslogConfig.INDEX_TRANSLOG_SYNC_INTERVAL, 0); // 0 has special meaning to sync each op
if (random.nextInt(10) == 0) { // do something crazy slow here
builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
} else {
builder.put(TranslogConfig.INDEX_TRANSLOG_SYNC_INTERVAL, RandomInts.randomIntBetween(random, 100, 5000), TimeUnit.MILLISECONDS);
builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
}
}
if (random.nextBoolean()) {
builder.put(IndicesStore.INDICES_STORE_THROTTLE_TYPE, RandomPicks.randomFrom(random, StoreRateLimiting.Type.values()));
}
if (random.nextBoolean()) {
if (random.nextInt(10) == 0) { // do something crazy slow here
builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
} else {
builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
}
}
if (random.nextBoolean()) {
builder.put(RecoverySettings.INDICES_RECOVERY_COMPRESS, random.nextBoolean());
}
if (random.nextBoolean()) {
builder.put(IndicesQueryCache.INDICES_CACHE_QUERY_CONCURRENCY_LEVEL, RandomInts.randomIntBetween(random, 1, 32));
builder.put(IndicesFieldDataCache.FIELDDATA_CACHE_CONCURRENCY_LEVEL, RandomInts.randomIntBetween(random, 1, 32));
}
if (random.nextBoolean()) {
builder.put(NettyTransport.PING_SCHEDULE, RandomInts.randomIntBetween(random, 100, 2000) + "ms");
}
if (random.nextBoolean()) {
builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING, RandomInts.randomIntBetween(random, -100, 2000));
}
if (random.nextBoolean()) {
builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING, TimeValue.timeValueMillis(RandomInts.randomIntBetween(random, 750, 10000000)));
}
return builder.build();
}