honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change default merge throttling to 50MB / sec 

The current setting of 20MB/sec seems to be too conservative given
the capabilities of modern hardware. Even on cloud infrastructure this
seems to be too lowish. A 50MB default should provide better out of the box
performance
This commit is contained in:
Simon Willnauer 2014-04-22 17:25:04 +02:00
parent eccbd911de
commit b4f0603169
5 changed files with 33 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/reference/index-modules/store.asciidoc
View File

@ -40,7 +40,7 @@ second. It can be set by setting `indices.store.throttle.type` to
`merge`, and setting `indices.store.throttle.max_bytes_per_sec` to
something like `5mb`. The node level settings can be changed dynamically
using the cluster update settings API. The default is set
to `20mb` with type `merge`.
to `50mb` with type `merge`.
If specific index level configuration is needed, regardless of the node
level settings, it can be set as well using the
@ -95,4 +95,4 @@ class, be sure your have plenty of virtual address space.
[[store-memory]]
=== Memory
The `memory` type stores the index in main memory.
The `memory` type stores the index in main memory.

2
docs/reference/modules/indices.asciidoc
View File

@ -59,7 +59,7 @@ The following settings can be set to manage the recovery policy:
defaults to `true`.
`indices.recovery.max_bytes_per_sec`::
defaults to `20mb`.
defaults to `50mb`.
[float]
[[throttling]]

2
src/main/java/org/elasticsearch/indices/store/IndicesStore.java
View File

@ -95,7 +95,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
// we limit with 20MB / sec by default with a default type set to merge sice 0.90.1
this.rateLimitingType = componentSettings.get("throttle.type", StoreRateLimiting.Type.MERGE.name());
rateLimiting.setType(rateLimitingType);
this.rateLimitingThrottle = componentSettings.getAsBytesSize("throttle.max_bytes_per_sec", new ByteSizeValue(20, ByteSizeUnit.MB));
this.rateLimitingThrottle = componentSettings.getAsBytesSize("throttle.max_bytes_per_sec", new ByteSizeValue(50, ByteSizeUnit.MB));
rateLimiting.setMaxRate(rateLimitingThrottle);
logger.debug("using indices.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle);

9
src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java
View File

@ -50,6 +50,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
@Test
public void testDirectoryToString() throws IOException {
cluster().wipeTemplates(); // no random settings please
createIndexWithStoreType("test", "niofs", "least_used");
String storeString = getStoreDirectory("test", 0).toString();
logger.info(storeString);
@ -58,7 +59,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
if (dataPaths.length > 1) {
assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(niofs(" + dataPaths[1].getAbsolutePath().toLowerCase(Locale.ROOT)));
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
assertThat(storeString, endsWith(", type=MERGE, rate=50.0)])"));
createIndexWithStoreType("test", "niofs", "random");
storeString = getStoreDirectory("test", 0).toString();
@ -68,7 +69,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
if (dataPaths.length > 1) {
assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(niofs(" + dataPaths[1].getAbsolutePath().toLowerCase(Locale.ROOT)));
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
assertThat(storeString, endsWith(", type=MERGE, rate=50.0)])"));
createIndexWithStoreType("test", "mmapfs", "least_used");
storeString = getStoreDirectory("test", 0).toString();
@ -78,7 +79,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
if (dataPaths.length > 1) {
assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(mmapfs(" + dataPaths[1].getAbsolutePath().toLowerCase(Locale.ROOT)));
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
assertThat(storeString, endsWith(", type=MERGE, rate=50.0)])"));
createIndexWithStoreType("test", "simplefs", "least_used");
storeString = getStoreDirectory("test", 0).toString();
@ -88,7 +89,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
if (dataPaths.length > 1) {
assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(simplefs(" + dataPaths[1].getAbsolutePath().toLowerCase(Locale.ROOT)));
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
assertThat(storeString, endsWith(", type=MERGE, rate=50.0)])"));
createIndexWithStoreType("test", "memory", "least_used");
storeString = getStoreDirectory("test", 0).toString();

27
src/test/java/org/elasticsearch/test/ImmutableTestCluster.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.test;
import com.carrotsearch.hppc.ObjectArrayList;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.store.StoreRateLimiting;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@ -43,6 +44,7 @@ import org.elasticsearch.index.merge.scheduler.SerialMergeSchedulerProvider;
import org.elasticsearch.index.translog.TranslogService;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.indices.IndexTemplateMissingException;
import org.elasticsearch.indices.store.IndicesStore;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.search.SearchService;
@ -228,10 +230,11 @@ public abstract class ImmutableTestCluster implements Iterable<Client> {
// TODO move settings for random directory etc here into the index based randomized settings.
if (size() > 0) {
ImmutableSettings.Builder builder =
setRandomTranslogSettings(random, setRandomNormsLoading(setRandomMerge(random, ImmutableSettings.builder()))
.put(SETTING_INDEX_SEED, random.nextLong()))
setRandomSettings(random, ImmutableSettings.builder())
.put(SETTING_INDEX_SEED, random.nextLong())
.put(SETTING_NUMBER_OF_SHARDS, RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_SHARDS, DEFAULT_MAX_NUM_SHARDS))
.put(SETTING_NUMBER_OF_REPLICAS, RandomInts.randomIntBetween(random, 0, 1));
client().admin().indices().preparePutTemplate("random_index_template")
.setTemplate("*")
.setOrder(0)
@ -240,7 +243,7 @@ public abstract class ImmutableTestCluster implements Iterable<Client> {
}
}
private ImmutableSettings.Builder setRandomNormsLoading(ImmutableSettings.Builder builder) {
private static ImmutableSettings.Builder setRandomNormsLoading(Random random, ImmutableSettings.Builder builder) {
if (random.nextBoolean()) {
builder.put(SearchService.NORMS_LOADING_KEY, RandomPicks.randomFrom(random, Arrays.asList(FieldMapper.Loading.EAGER, FieldMapper.Loading.LAZY)));
}
@ -266,6 +269,24 @@ public abstract class ImmutableTestCluster implements Iterable<Client> {
return builder;
}
private static ImmutableSettings.Builder setRandomSettings(Random random, ImmutableSettings.Builder builder) {
setRandomMerge(random, builder);
setRandomTranslogSettings(random, builder);
setRandomNormsLoading(random, builder);
if (random.nextBoolean()) {
if (random.nextInt(10) == 0) { // do something crazy slow here
builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
} else {
builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
}
}
if (random.nextBoolean()) {
builder.put(IndicesStore.INDICES_STORE_THROTTLE_TYPE, RandomPicks.randomFrom(random, StoreRateLimiting.Type.values()));
}
return builder;
}
private static ImmutableSettings.Builder setRandomMerge(Random random, ImmutableSettings.Builder builder) {
if (random.nextBoolean()) {
builder.put(AbstractMergePolicyProvider.INDEX_COMPOUND_FORMAT,