From 31e6acf3f22927929d4879475975076ae7aa104a Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Sat, 10 Jan 2015 16:38:56 -0500 Subject: [PATCH 1/6] first cut --- docs/reference/index-modules/store.asciidoc | 32 --- .../TrackingConcurrentMergeScheduler.java | 14 + .../lucene/store/RateLimitedFSDirectory.java | 106 ------- .../lucene/store/StoreRateLimiting.java | 94 ------- .../ClusterDynamicSettingsModule.java | 2 - .../elasticsearch/index/merge/MergeStats.java | 65 ++++- .../ConcurrentMergeSchedulerProvider.java | 47 ++-- .../scheduler/MergeSchedulerProvider.java | 3 - .../settings/IndexDynamicSettingsModule.java | 3 +- .../index/store/DirectoryService.java | 4 +- .../elasticsearch/index/store/IndexStore.java | 7 - .../org/elasticsearch/index/store/Store.java | 2 +- .../elasticsearch/index/store/StoreStats.java | 33 +-- .../index}/store/StoreUtils.java | 21 +- .../distributor/AbstractDistributor.java | 13 +- .../index/store/fs/FsDirectoryService.java | 22 +- .../store/support/AbstractIndexStore.java | 58 ---- .../indices/store/IndicesStore.java | 47 +--- .../engine/internal/InternalEngineTests.java | 5 - .../merge/policy/MergePolicySettingsTest.java | 5 - .../elasticsearch/index/store/StoreTest.java | 5 - .../store/distributor/DistributorTests.java | 5 - .../indices/settings/UpdateSettingsTests.java | 265 ++++-------------- .../indices/stats/IndexStatsTests.java | 86 +----- .../indices/store/SimpleDistributorTests.java | 50 +--- .../DedicatedClusterSnapshotRestoreTests.java | 10 - .../SharedClusterSnapshotRestoreTests.java | 11 - .../test/ElasticsearchIntegrationTest.java | 11 - .../test/store/MockFSDirectoryService.java | 16 -- 29 files changed, 220 insertions(+), 822 deletions(-) delete mode 100644 src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java delete mode 100644 src/main/java/org/apache/lucene/store/StoreRateLimiting.java rename src/main/java/org/{apache/lucene => elasticsearch/index}/store/StoreUtils.java (76%) diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc index a56c9315c56..339f1f0d218 100644 --- a/docs/reference/index-modules/store.asciidoc +++ b/docs/reference/index-modules/store.asciidoc @@ -18,38 +18,6 @@ heap space* using the "Memory" (see below) storage type. It translates to the fact that there is no need for extra large JVM heaps (with their own consequences) for storing the index in memory. - -[float] -[[store-throttling]] -=== Store Level Throttling - -The way Lucene, the IR library elasticsearch uses under the covers, -works is by creating immutable segments (up to deletes) and constantly -merging them (the merge policy settings allow to control how those -merges happen). The merge process happens in an asynchronous manner -without affecting the indexing / search speed. The problem though, -especially on systems with low IO, is that the merge process can be -expensive and affect search / index operation simply by the fact that -the box is now taxed with more IO happening. - -The store module allows to have throttling configured for merges (or -all) either on the node level, or on the index level. The node level -throttling will make sure that out of all the shards allocated on that -node, the merge process won't pass the specific setting bytes per -second. It can be set by setting `indices.store.throttle.type` to -`merge`, and setting `indices.store.throttle.max_bytes_per_sec` to -something like `5mb`. The node level settings can be changed dynamically -using the cluster update settings API. The default is set -to `20mb` with type `merge`. - -If specific index level configuration is needed, regardless of the node -level settings, it can be set as well using the -`index.store.throttle.type`, and -`index.store.throttle.max_bytes_per_sec`. The default value for the type -is `node`, meaning it will throttle based on the node level settings and -participate in the global throttling happening. Both settings can be set -using the index update settings API dynamically. - [float] [[file-system]] === File system storage types diff --git a/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java b/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java index ec2c0a003f5..d5509bab383 100644 --- a/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java +++ b/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java @@ -46,6 +46,8 @@ public class TrackingConcurrentMergeScheduler extends ConcurrentMergeScheduler { private final CounterMetric currentMerges = new CounterMetric(); private final CounterMetric currentMergesNumDocs = new CounterMetric(); private final CounterMetric currentMergesSizeInBytes = new CounterMetric(); + private final CounterMetric totalMergeStoppedTime = new CounterMetric(); + private final CounterMetric totalMergeThrottledTime = new CounterMetric(); private final Set onGoingMerges = ConcurrentCollections.newConcurrentSet(); private final Set readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges); @@ -83,6 +85,14 @@ public class TrackingConcurrentMergeScheduler extends ConcurrentMergeScheduler { return currentMergesSizeInBytes.count(); } + public long totalMergeStoppedTimeMillis() { + return totalMergeStoppedTime.count(); + } + + public long totalMergeThrottledTimeMillis() { + return totalMergeThrottledTime.count(); + } + public Set onGoingMerges() { return readOnlyOnGoingMerges; } @@ -118,6 +128,10 @@ public class TrackingConcurrentMergeScheduler extends ConcurrentMergeScheduler { totalMergesNumDocs.inc(totalNumDocs); totalMergesSizeInBytes.inc(totalSizeInBytes); totalMerges.inc(took); + + totalMergeStoppedTime.inc(merge.rateLimiter.getTotalStoppedNS()/1000000); + totalMergeThrottledTime.inc(merge.rateLimiter.getTotalPausedNS()/1000000); + String message = String.format(Locale.ROOT, "merge segment [%s] done: took [%s], [%,.1f MB], [%,d docs]", merge.info == null ? "_na_" : merge.info.info.name, diff --git a/src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java b/src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java deleted file mode 100644 index e6ec10ca34b..00000000000 --- a/src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lucene.store; - -import org.apache.lucene.store.IOContext.Context; - -import java.io.IOException; - -public final class RateLimitedFSDirectory extends FilterDirectory { - - private final StoreRateLimiting.Provider rateLimitingProvider; - - private final StoreRateLimiting.Listener rateListener; - - public RateLimitedFSDirectory(Directory wrapped, StoreRateLimiting.Provider rateLimitingProvider, - StoreRateLimiting.Listener rateListener) { - super(wrapped); - this.rateLimitingProvider = rateLimitingProvider; - this.rateListener = rateListener; - } - - @Override - public IndexOutput createOutput(String name, IOContext context) throws IOException { - final IndexOutput output = in.createOutput(name, context); - - StoreRateLimiting rateLimiting = rateLimitingProvider.rateLimiting(); - StoreRateLimiting.Type type = rateLimiting.getType(); - RateLimiter limiter = rateLimiting.getRateLimiter(); - if (type == StoreRateLimiting.Type.NONE || limiter == null) { - return output; - } - if (context.context == Context.MERGE || type == StoreRateLimiting.Type.ALL) { - // we are merging, and type is either MERGE or ALL, rate limit... - return new RateLimitedIndexOutput(new RateLimiterWrapper(limiter, rateListener), output); - } - // we shouldn't really get here... - return output; - } - - - @Override - public void close() throws IOException { - in.close(); - } - - @Override - public String toString() { - StoreRateLimiting rateLimiting = rateLimitingProvider.rateLimiting(); - StoreRateLimiting.Type type = rateLimiting.getType(); - RateLimiter limiter = rateLimiting.getRateLimiter(); - if (type == StoreRateLimiting.Type.NONE || limiter == null) { - return StoreUtils.toString(in); - } else { - return "rate_limited(" + StoreUtils.toString(in) + ", type=" + type.name() + ", rate=" + limiter.getMBPerSec() + ")"; - } - } - - // we wrap the limiter to notify our store if we limited to get statistics - static final class RateLimiterWrapper extends RateLimiter { - private final RateLimiter delegate; - private final StoreRateLimiting.Listener rateListener; - - RateLimiterWrapper(RateLimiter delegate, StoreRateLimiting.Listener rateListener) { - this.delegate = delegate; - this.rateListener = rateListener; - } - - @Override - public void setMBPerSec(double mbPerSec) { - delegate.setMBPerSec(mbPerSec); - } - - @Override - public double getMBPerSec() { - return delegate.getMBPerSec(); - } - - @Override - public long pause(long bytes) throws IOException { - long pause = delegate.pause(bytes); - rateListener.onPause(pause); - return pause; - } - - @Override - public long getMinPauseCheckBytes() { - return delegate.getMinPauseCheckBytes(); - } - } -} diff --git a/src/main/java/org/apache/lucene/store/StoreRateLimiting.java b/src/main/java/org/apache/lucene/store/StoreRateLimiting.java deleted file mode 100644 index ae021b07b09..00000000000 --- a/src/main/java/org/apache/lucene/store/StoreRateLimiting.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lucene.store; - -import org.apache.lucene.store.RateLimiter.SimpleRateLimiter; -import org.elasticsearch.ElasticsearchIllegalArgumentException; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.unit.ByteSizeValue; - -/** - */ -public class StoreRateLimiting { - - public static interface Provider { - - StoreRateLimiting rateLimiting(); - } - - public interface Listener { - - void onPause(long nanos); - } - - public static enum Type { - NONE, - MERGE, - ALL; - - public static Type fromString(String type) throws ElasticsearchIllegalArgumentException { - if ("none".equalsIgnoreCase(type)) { - return NONE; - } else if ("merge".equalsIgnoreCase(type)) { - return MERGE; - } else if ("all".equalsIgnoreCase(type)) { - return ALL; - } - throw new ElasticsearchIllegalArgumentException("rate limiting type [" + type + "] not valid, can be one of [all|merge|none]"); - } - } - - private final SimpleRateLimiter rateLimiter = new SimpleRateLimiter(0); - private volatile SimpleRateLimiter actualRateLimiter; - - private volatile Type type; - - public StoreRateLimiting() { - - } - - @Nullable - public RateLimiter getRateLimiter() { - return actualRateLimiter; - } - - public void setMaxRate(ByteSizeValue rate) { - if (rate.bytes() <= 0) { - actualRateLimiter = null; - } else if (actualRateLimiter == null) { - actualRateLimiter = rateLimiter; - actualRateLimiter.setMBPerSec(rate.mbFrac()); - } else { - assert rateLimiter == actualRateLimiter; - rateLimiter.setMBPerSec(rate.mbFrac()); - } - } - - public Type getType() { - return type; - } - - public void setType(Type type) { - this.type = type; - } - - public void setType(String type) throws ElasticsearchIllegalArgumentException { - this.type = Type.fromString(type); - } -} diff --git a/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java b/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java index cb4aca624ea..9cd7c427b41 100644 --- a/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java +++ b/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java @@ -65,8 +65,6 @@ public class ClusterDynamicSettingsModule extends AbstractModule { clusterDynamicSettings.addDynamicSetting(IndicesFilterCache.INDICES_CACHE_FILTER_SIZE); clusterDynamicSettings.addDynamicSetting(IndicesFilterCache.INDICES_CACHE_FILTER_EXPIRE, Validator.TIME); clusterDynamicSettings.addDynamicSetting(IndicesFilterCache.INDICES_CACHE_FILTER_CONCURRENCY_LEVEL, Validator.POSITIVE_INTEGER); - clusterDynamicSettings.addDynamicSetting(IndicesStore.INDICES_STORE_THROTTLE_TYPE); - clusterDynamicSettings.addDynamicSetting(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE); clusterDynamicSettings.addDynamicSetting(IndicesTTLService.INDICES_TTL_INTERVAL, Validator.TIME); clusterDynamicSettings.addDynamicSetting(MappingUpdatedAction.INDICES_MAPPING_ADDITIONAL_MAPPING_CHANGE_TIME, Validator.TIME); clusterDynamicSettings.addDynamicSetting(MetaData.SETTING_READ_ONLY); diff --git a/src/main/java/org/elasticsearch/index/merge/MergeStats.java b/src/main/java/org/elasticsearch/index/merge/MergeStats.java index ee451b1ac15..7664ab5f277 100644 --- a/src/main/java/org/elasticsearch/index/merge/MergeStats.java +++ b/src/main/java/org/elasticsearch/index/merge/MergeStats.java @@ -19,6 +19,9 @@ package org.elasticsearch.index.merge; +import java.io.IOException; + +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -28,11 +31,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; -import java.io.IOException; - -/** - * - */ public class MergeStats implements Streamable, ToXContent { private long total; @@ -43,11 +41,18 @@ public class MergeStats implements Streamable, ToXContent { private long currentNumDocs; private long currentSizeInBytes; + /** Total millis that large merges were stopped so that smaller merges would finish. */ + private long totalStoppedTimeInMillis; + + /** Total millis that we slept during writes so merge IO is throttled. */ + private long totalThrottledTimeInMillis; + public MergeStats() { } - public void add(long totalMerges, long totalMergeTime, long totalNumDocs, long totalSizeInBytes, long currentMerges, long currentNumDocs, long currentSizeInBytes) { + public void add(long totalMerges, long totalMergeTime, long totalNumDocs, long totalSizeInBytes, long currentMerges, long currentNumDocs, long currentSizeInBytes, + long stoppedTimeMillis, long throttledTimeMillis) { this.total += totalMerges; this.totalTimeInMillis += totalMergeTime; this.totalNumDocs += totalNumDocs; @@ -55,6 +60,8 @@ public class MergeStats implements Streamable, ToXContent { this.current += currentMerges; this.currentNumDocs += currentNumDocs; this.currentSizeInBytes += currentSizeInBytes; + this.totalStoppedTimeInMillis += stoppedTimeMillis; + this.totalThrottledTimeInMillis += throttledTimeMillis; } public void add(MergeStats mergeStats) { @@ -68,6 +75,8 @@ public class MergeStats implements Streamable, ToXContent { this.current += mergeStats.current; this.currentNumDocs += mergeStats.currentNumDocs; this.currentSizeInBytes += mergeStats.currentSizeInBytes; + this.totalStoppedTimeInMillis += mergeStats.totalStoppedTimeInMillis; + this.totalThrottledTimeInMillis += mergeStats.totalThrottledTimeInMillis; } /** @@ -84,6 +93,34 @@ public class MergeStats implements Streamable, ToXContent { return this.totalTimeInMillis; } + /** + * The total time large merges were stopped so smaller merges could finish. + */ + public long getTotalStoppedTimeInMillis() { + return this.totalStoppedTimeInMillis; + } + + /** + * The total time large merges were stopped so smaller merges could finish. + */ + public TimeValue getTotalStoppedTime() { + return new TimeValue(totalStoppedTimeInMillis); + } + + /** + * The total time merge IO writes were throttled. + */ + public long getTotalThrottledTimeInMillis() { + return this.totalThrottledTimeInMillis; + } + + /** + * The total time merge IO writes were throttled. + */ + public TimeValue getTotalThrottledTime() { + return new TimeValue(totalThrottledTimeInMillis); + } + /** * The total time merges have been executed. */ @@ -138,6 +175,8 @@ public class MergeStats implements Streamable, ToXContent { builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, totalTimeInMillis); builder.field(Fields.TOTAL_DOCS, totalNumDocs); builder.byteSizeField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, totalSizeInBytes); + builder.timeValueField(Fields.TOTAL_STOPPED_TIME_IN_MILLIS, Fields.TOTAL_STOPPED_TIME, totalStoppedTimeInMillis); + builder.timeValueField(Fields.TOTAL_THROTTLED_TIME_IN_MILLIS, Fields.TOTAL_THROTTLED_TIME, totalThrottledTimeInMillis); builder.endObject(); return builder; } @@ -151,6 +190,10 @@ public class MergeStats implements Streamable, ToXContent { static final XContentBuilderString TOTAL = new XContentBuilderString("total"); static final XContentBuilderString TOTAL_TIME = new XContentBuilderString("total_time"); static final XContentBuilderString TOTAL_TIME_IN_MILLIS = new XContentBuilderString("total_time_in_millis"); + static final XContentBuilderString TOTAL_STOPPED_TIME = new XContentBuilderString("total_stopped_time"); + static final XContentBuilderString TOTAL_STOPPED_TIME_IN_MILLIS = new XContentBuilderString("total_stopped_time_in_millis"); + static final XContentBuilderString TOTAL_THROTTLED_TIME = new XContentBuilderString("total_throttled_time"); + static final XContentBuilderString TOTAL_THROTTLED_TIME_IN_MILLIS = new XContentBuilderString("total_throttled_time_in_millis"); static final XContentBuilderString TOTAL_DOCS = new XContentBuilderString("total_docs"); static final XContentBuilderString TOTAL_SIZE = new XContentBuilderString("total_size"); static final XContentBuilderString TOTAL_SIZE_IN_BYTES = new XContentBuilderString("total_size_in_bytes"); @@ -165,6 +208,10 @@ public class MergeStats implements Streamable, ToXContent { current = in.readVLong(); currentNumDocs = in.readVLong(); currentSizeInBytes = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_2_0_0)) { + totalStoppedTimeInMillis = in.readVLong(); + totalThrottledTimeInMillis = in.readVLong(); + } } @Override @@ -176,5 +223,9 @@ public class MergeStats implements Streamable, ToXContent { out.writeVLong(current); out.writeVLong(currentNumDocs); out.writeVLong(currentSizeInBytes); + if (out.getVersion().onOrAfter(Version.V_2_0_0)) { + out.writeVLong(totalStoppedTimeInMillis); + out.writeVLong(totalThrottledTimeInMillis); + } } -} \ No newline at end of file +} diff --git a/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java b/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java index b181c18420d..ca8acc2f0d3 100644 --- a/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java +++ b/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java @@ -41,22 +41,18 @@ import java.io.IOException; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; -/** - * - */ public class ConcurrentMergeSchedulerProvider extends MergeSchedulerProvider { private final IndexSettingsService indexSettingsService; private final ApplySettings applySettings = new ApplySettings(); - private static final String MAX_THREAD_COUNT_KEY = "max_thread_count"; - private static final String MAX_MERGE_COUNT_KEY = "max_merge_count"; - - public static final String MAX_THREAD_COUNT = "index.merge.scheduler." + MAX_THREAD_COUNT_KEY; - public static final String MAX_MERGE_COUNT = "index.merge.scheduler." + MAX_MERGE_COUNT_KEY; + public static final String MAX_THREAD_COUNT = "index.merge.scheduler.max_thread_count"; + public static final String MAX_MERGE_COUNT = "index.merge.scheduler.max_merge_count"; + public static final String AUTO_THROTTLE = "index.merge.scheduler.auto_throttle"; private volatile int maxThreadCount; private volatile int maxMergeCount; + private volatile boolean autoThrottle; private Set schedulers = new CopyOnWriteArraySet<>(); @@ -64,10 +60,10 @@ public class ConcurrentMergeSchedulerProvider extends MergeSchedulerProvider { public ConcurrentMergeSchedulerProvider(ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool, IndexSettingsService indexSettingsService) { super(shardId, indexSettings, threadPool); this.indexSettingsService = indexSettingsService; - // TODO LUCENE MONITOR this will change in Lucene 4.0 - this.maxThreadCount = componentSettings.getAsInt(MAX_THREAD_COUNT_KEY, Math.max(1, Math.min(3, EsExecutors.boundedNumberOfProcessors(indexSettings) / 2))); - this.maxMergeCount = componentSettings.getAsInt(MAX_MERGE_COUNT_KEY, maxThreadCount + 2); - logger.debug("using [concurrent] merge scheduler with max_thread_count[{}], max_merge_count[{}]", maxThreadCount, maxMergeCount); + this.maxThreadCount = indexSettings.getAsInt(MAX_THREAD_COUNT, Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(indexSettings) / 2))); + this.maxMergeCount = indexSettings.getAsInt(MAX_MERGE_COUNT, maxThreadCount + 5); + this.autoThrottle = indexSettings.getAsBoolean(AUTO_THROTTLE, true); + logger.debug("using [concurrent] merge scheduler with max_thread_count[{}], max_merge_count[{}], auto_throttle[{}]", maxThreadCount, maxMergeCount, autoThrottle); indexSettingsService.addListener(applySettings); } @@ -75,10 +71,14 @@ public class ConcurrentMergeSchedulerProvider extends MergeSchedulerProvider { @Override public MergeScheduler newMergeScheduler() { CustomConcurrentMergeScheduler concurrentMergeScheduler = new CustomConcurrentMergeScheduler(logger, shardId, this); - // which would then stall if there are 2 merges in flight, and unstall once we are back to 1 or 0 merges // NOTE: we pass maxMergeCount+1 here so that CMS will allow one too many merges to kick off which then allows // InternalEngine.IndexThrottle to detect too-many-merges and throttle: concurrentMergeScheduler.setMaxMergesAndThreads(maxMergeCount+1, maxThreadCount); + if (autoThrottle) { + concurrentMergeScheduler.enableAutoIOThrottle(); + } else { + concurrentMergeScheduler.disableAutoIOThrottle(); + } schedulers.add(concurrentMergeScheduler); return concurrentMergeScheduler; } @@ -88,7 +88,9 @@ public class ConcurrentMergeSchedulerProvider extends MergeSchedulerProvider { MergeStats mergeStats = new MergeStats(); for (CustomConcurrentMergeScheduler scheduler : schedulers) { mergeStats.add(scheduler.totalMerges(), scheduler.totalMergeTime(), scheduler.totalMergeNumDocs(), scheduler.totalMergeSizeInBytes(), - scheduler.currentMerges(), scheduler.currentMergesNumDocs(), scheduler.currentMergesSizeInBytes()); + scheduler.currentMerges(), scheduler.currentMergesNumDocs(), scheduler.currentMergesSizeInBytes(), + scheduler.totalMergeStoppedTimeMillis(), + scheduler.totalMergeThrottledTimeMillis()); } return mergeStats; } @@ -165,7 +167,7 @@ public class ConcurrentMergeSchedulerProvider extends MergeSchedulerProvider { public void onRefreshSettings(Settings settings) { int maxThreadCount = settings.getAsInt(MAX_THREAD_COUNT, ConcurrentMergeSchedulerProvider.this.maxThreadCount); if (maxThreadCount != ConcurrentMergeSchedulerProvider.this.maxThreadCount) { - logger.info("updating [{}] from [{}] to [{}]", MAX_THREAD_COUNT_KEY, ConcurrentMergeSchedulerProvider.this.maxThreadCount, maxThreadCount); + logger.info("updating [{}] from [{}] to [{}]", MAX_THREAD_COUNT, ConcurrentMergeSchedulerProvider.this.maxThreadCount, maxThreadCount); ConcurrentMergeSchedulerProvider.this.maxThreadCount = maxThreadCount; for (CustomConcurrentMergeScheduler scheduler : schedulers) { scheduler.setMaxMergesAndThreads(ConcurrentMergeSchedulerProvider.this.maxMergeCount, maxThreadCount); @@ -174,12 +176,25 @@ public class ConcurrentMergeSchedulerProvider extends MergeSchedulerProvider { int maxMergeCount = settings.getAsInt(MAX_MERGE_COUNT, ConcurrentMergeSchedulerProvider.this.maxMergeCount); if (maxMergeCount != ConcurrentMergeSchedulerProvider.this.maxMergeCount) { - logger.info("updating [{}] from [{}] to [{}]", MAX_MERGE_COUNT_KEY, ConcurrentMergeSchedulerProvider.this.maxMergeCount, maxMergeCount); + logger.info("updating [{}] from [{}] to [{}]", MAX_MERGE_COUNT, ConcurrentMergeSchedulerProvider.this.maxMergeCount, maxMergeCount); ConcurrentMergeSchedulerProvider.this.maxMergeCount = maxMergeCount; for (CustomConcurrentMergeScheduler scheduler : schedulers) { scheduler.setMaxMergesAndThreads(maxMergeCount, ConcurrentMergeSchedulerProvider.this.maxThreadCount); } } + + boolean autoThrottle = settings.getAsBoolean(AUTO_THROTTLE, ConcurrentMergeSchedulerProvider.this.autoThrottle); + if (autoThrottle != ConcurrentMergeSchedulerProvider.this.autoThrottle) { + logger.info("updating [{}] from [{}] to [{}]", AUTO_THROTTLE, ConcurrentMergeSchedulerProvider.this.autoThrottle, autoThrottle); + ConcurrentMergeSchedulerProvider.this.autoThrottle = autoThrottle; + for (CustomConcurrentMergeScheduler scheduler : schedulers) { + if (autoThrottle) { + scheduler.enableAutoIOThrottle(); + } else { + scheduler.disableAutoIOThrottle(); + } + } + } } } } diff --git a/src/main/java/org/elasticsearch/index/merge/scheduler/MergeSchedulerProvider.java b/src/main/java/org/elasticsearch/index/merge/scheduler/MergeSchedulerProvider.java index de9a7d8bed6..11719ec866e 100644 --- a/src/main/java/org/elasticsearch/index/merge/scheduler/MergeSchedulerProvider.java +++ b/src/main/java/org/elasticsearch/index/merge/scheduler/MergeSchedulerProvider.java @@ -34,9 +34,6 @@ import java.io.Closeable; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; -/** - * - */ public abstract class MergeSchedulerProvider extends AbstractIndexShardComponent implements IndexShardComponent, Closeable { public static interface FailureListener { diff --git a/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java b/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java index 42c55b9722b..044db0c2c6b 100644 --- a/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java +++ b/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java @@ -51,10 +51,9 @@ public class IndexDynamicSettingsModule extends AbstractModule { public IndexDynamicSettingsModule() { indexDynamicSettings = new DynamicSettings(); - indexDynamicSettings.addDynamicSetting(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE); - indexDynamicSettings.addDynamicSetting(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE); indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT); indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT); + indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.AUTO_THROTTLE); indexDynamicSettings.addDynamicSetting(FilterAllocationDecider.INDEX_ROUTING_REQUIRE_GROUP + "*"); indexDynamicSettings.addDynamicSetting(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "*"); indexDynamicSettings.addDynamicSetting(FilterAllocationDecider.INDEX_ROUTING_EXCLUDE_GROUP + "*"); diff --git a/src/main/java/org/elasticsearch/index/store/DirectoryService.java b/src/main/java/org/elasticsearch/index/store/DirectoryService.java index 81d8910ed4c..2fa4e8eb455 100644 --- a/src/main/java/org/elasticsearch/index/store/DirectoryService.java +++ b/src/main/java/org/elasticsearch/index/store/DirectoryService.java @@ -39,8 +39,6 @@ public abstract class DirectoryService extends AbstractIndexShardComponent { public abstract Directory[] build() throws IOException; - public abstract long throttleTimeInNanos(); - /** * Creates a new Directory from the given distributor. * The default implementation returns a new {@link org.elasticsearch.index.store.DistributorDirectory} @@ -58,4 +56,4 @@ public abstract class DirectoryService extends AbstractIndexShardComponent { } return new DistributorDirectory(distributor); } -} \ No newline at end of file +} diff --git a/src/main/java/org/elasticsearch/index/store/IndexStore.java b/src/main/java/org/elasticsearch/index/store/IndexStore.java index d979075694b..d22c993ae18 100644 --- a/src/main/java/org/elasticsearch/index/store/IndexStore.java +++ b/src/main/java/org/elasticsearch/index/store/IndexStore.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.store; -import org.apache.lucene.store.StoreRateLimiting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.ShardId; @@ -33,12 +32,6 @@ import java.nio.file.Path; */ public interface IndexStore extends Closeable { - /** - * Returns the rate limiting, either of the index is explicitly configured, or - * the node level one (defaults to the node level one). - */ - StoreRateLimiting rateLimiting(); - /** * The shard store class that should be used for each shard. */ diff --git a/src/main/java/org/elasticsearch/index/store/Store.java b/src/main/java/org/elasticsearch/index/store/Store.java index 87df70c81f9..c5610d6f257 100644 --- a/src/main/java/org/elasticsearch/index/store/Store.java +++ b/src/main/java/org/elasticsearch/index/store/Store.java @@ -283,7 +283,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref public StoreStats stats() throws IOException { ensureOpen(); - return new StoreStats(Directories.estimateSize(directory), directoryService.throttleTimeInNanos()); + return new StoreStats(Directories.estimateSize(directory)); } public void renameFile(String from, String to) throws IOException { diff --git a/src/main/java/org/elasticsearch/index/store/StoreStats.java b/src/main/java/org/elasticsearch/index/store/StoreStats.java index 3b7f52a6895..1bdae5d4771 100644 --- a/src/main/java/org/elasticsearch/index/store/StoreStats.java +++ b/src/main/java/org/elasticsearch/index/store/StoreStats.java @@ -19,6 +19,9 @@ package org.elasticsearch.index.store; +import java.io.IOException; + +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -28,23 +31,18 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; -import java.io.IOException; - /** */ public class StoreStats implements Streamable, ToXContent { private long sizeInBytes; - private long throttleTimeInNanos; - public StoreStats() { } - public StoreStats(long sizeInBytes, long throttleTimeInNanos) { + public StoreStats(long sizeInBytes) { this.sizeInBytes = sizeInBytes; - this.throttleTimeInNanos = throttleTimeInNanos; } public void add(StoreStats stats) { @@ -52,7 +50,6 @@ public class StoreStats implements Streamable, ToXContent { return; } sizeInBytes += stats.sizeInBytes; - throttleTimeInNanos += stats.throttleTimeInNanos; } @@ -72,14 +69,6 @@ public class StoreStats implements Streamable, ToXContent { return size(); } - public TimeValue throttleTime() { - return TimeValue.timeValueNanos(throttleTimeInNanos); - } - - public TimeValue getThrottleTime() { - return throttleTime(); - } - public static StoreStats readStoreStats(StreamInput in) throws IOException { StoreStats store = new StoreStats(); store.readFrom(in); @@ -89,20 +78,25 @@ public class StoreStats implements Streamable, ToXContent { @Override public void readFrom(StreamInput in) throws IOException { sizeInBytes = in.readVLong(); - throttleTimeInNanos = in.readVLong(); + if (in.getVersion().before(Version.V_2_0_0)) { + // Ignore throttleTimeInNanos + in.readVLong(); + } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(sizeInBytes); - out.writeVLong(throttleTimeInNanos); + if (out.getVersion().before(Version.V_2_0_0)) { + // Send dummy throttleTimeInNanos + out.writeVLong(0); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.STORE); builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, sizeInBytes); - builder.timeValueField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, throttleTime()); builder.endObject(); return builder; } @@ -111,8 +105,5 @@ public class StoreStats implements Streamable, ToXContent { static final XContentBuilderString STORE = new XContentBuilderString("store"); static final XContentBuilderString SIZE = new XContentBuilderString("size"); static final XContentBuilderString SIZE_IN_BYTES = new XContentBuilderString("size_in_bytes"); - - static final XContentBuilderString THROTTLE_TIME = new XContentBuilderString("throttle_time"); - static final XContentBuilderString THROTTLE_TIME_IN_MILLIS = new XContentBuilderString("throttle_time_in_millis"); } } diff --git a/src/main/java/org/apache/lucene/store/StoreUtils.java b/src/main/java/org/elasticsearch/index/store/StoreUtils.java similarity index 76% rename from src/main/java/org/apache/lucene/store/StoreUtils.java rename to src/main/java/org/elasticsearch/index/store/StoreUtils.java index 7bf85604f34..261e363db76 100644 --- a/src/main/java/org/apache/lucene/store/StoreUtils.java +++ b/src/main/java/org/elasticsearch/index/store/StoreUtils.java @@ -16,10 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.lucene.store; -/** - */ +package org.elasticsearch.index.store; + +import java.util.Arrays; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FileSwitchDirectory; +import org.apache.lucene.store.MMapDirectory; +import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.store.SimpleFSDirectory; + public final class StoreUtils { private StoreUtils() { @@ -46,4 +53,12 @@ public final class StoreUtils { return directory.toString(); } + + public static String toString(Directory[] directories) { + String[] strings = new String[directories.length]; + for(int i=0;i ACTIVE_STATES = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED); - class ApplySettings implements NodeSettingsService.Listener { - @Override - public void onRefreshSettings(Settings settings) { - String rateLimitingType = settings.get(INDICES_STORE_THROTTLE_TYPE, IndicesStore.this.rateLimitingType); - // try and parse the type - StoreRateLimiting.Type.fromString(rateLimitingType); - if (!rateLimitingType.equals(IndicesStore.this.rateLimitingType)) { - logger.info("updating indices.store.throttle.type from [{}] to [{}]", IndicesStore.this.rateLimitingType, rateLimitingType); - IndicesStore.this.rateLimitingType = rateLimitingType; - IndicesStore.this.rateLimiting.setType(rateLimitingType); - } - - ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndicesStore.this.rateLimitingThrottle); - if (!rateLimitingThrottle.equals(IndicesStore.this.rateLimitingThrottle)) { - logger.info("updating indices.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndicesStore.this.rateLimitingThrottle, rateLimitingThrottle, IndicesStore.this.rateLimitingType); - IndicesStore.this.rateLimitingThrottle = rateLimitingThrottle; - IndicesStore.this.rateLimiting.setMaxRate(rateLimitingThrottle); - } - } - } - private final NodeEnvironment nodeEnv; private final NodeSettingsService nodeSettingsService; @@ -97,12 +72,6 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe private final ClusterService clusterService; private final TransportService transportService; - private volatile String rateLimitingType; - private volatile ByteSizeValue rateLimitingThrottle; - private final StoreRateLimiting rateLimiting = new StoreRateLimiting(); - - private final ApplySettings applySettings = new ApplySettings(); - @Inject public IndicesStore(Settings settings, NodeEnvironment nodeEnv, NodeSettingsService nodeSettingsService, IndicesService indicesService, ClusterService clusterService, TransportService transportService) { @@ -114,15 +83,6 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe this.transportService = transportService; transportService.registerHandler(ACTION_SHARD_EXISTS, new ShardActiveRequestHandler()); - // we limit with 20MB / sec by default with a default type set to merge sice 0.90.1 - this.rateLimitingType = componentSettings.get("throttle.type", StoreRateLimiting.Type.MERGE.name()); - rateLimiting.setType(rateLimitingType); - this.rateLimitingThrottle = componentSettings.getAsBytesSize("throttle.max_bytes_per_sec", new ByteSizeValue(20, ByteSizeUnit.MB)); - rateLimiting.setMaxRate(rateLimitingThrottle); - - logger.debug("using indices.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle); - - nodeSettingsService.addListener(applySettings); clusterService.addLast(this); } @@ -135,13 +95,8 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe this.transportService = null; } - public StoreRateLimiting rateLimiting() { - return this.rateLimiting; - } - @Override public void close() { - nodeSettingsService.removeListener(applySettings); clusterService.remove(this); } @@ -458,4 +413,4 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe node.writeTo(out); } } -} \ No newline at end of file +} diff --git a/src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java b/src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java index 8485a5aae7d..8aba2d37aa1 100644 --- a/src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java +++ b/src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java @@ -179,11 +179,6 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase { public Directory[] build() throws IOException { return new Directory[]{ directory }; } - - @Override - public long throttleTimeInNanos() { - return 0; - } }; return new Store(shardId, EMPTY_SETTINGS, directoryService, new LeastUsedDistributor(directoryService), new DummyShardLock(shardId)); } diff --git a/src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java b/src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java index 6caac7c3186..43306d1b315 100644 --- a/src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java +++ b/src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java @@ -178,11 +178,6 @@ public class MergePolicySettingsTest extends ElasticsearchTestCase { public Directory[] build() throws IOException { return new Directory[] { new RAMDirectory() } ; } - - @Override - public long throttleTimeInNanos() { - return 0; - } }; return new Store(shardId, settings, directoryService, new LeastUsedDistributor(directoryService), new DummyShardLock(shardId)); } diff --git a/src/test/java/org/elasticsearch/index/store/StoreTest.java b/src/test/java/org/elasticsearch/index/store/StoreTest.java index 2dd834a6047..35dc4968a12 100644 --- a/src/test/java/org/elasticsearch/index/store/StoreTest.java +++ b/src/test/java/org/elasticsearch/index/store/StoreTest.java @@ -710,11 +710,6 @@ public class StoreTest extends ElasticsearchLuceneTestCase { public Directory[] build() throws IOException { return dirs; } - - @Override - public long throttleTimeInNanos() { - return random.nextInt(1000); - } } public static void assertConsistent(Store store, Store.MetadataSnapshot metadata) throws IOException { diff --git a/src/test/java/org/elasticsearch/index/store/distributor/DistributorTests.java b/src/test/java/org/elasticsearch/index/store/distributor/DistributorTests.java index 119c7636843..5d319692681 100644 --- a/src/test/java/org/elasticsearch/index/store/distributor/DistributorTests.java +++ b/src/test/java/org/elasticsearch/index/store/distributor/DistributorTests.java @@ -150,11 +150,6 @@ public class DistributorTests extends ElasticsearchTestCase { public Directory[] build() throws IOException { return directories; } - - @Override - public long throttleTimeInNanos() { - return 0; - } } public static class FakeFsDirectory extends FSDirectory { diff --git a/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java b/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java index febb3f97091..812e6c43131 100644 --- a/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java +++ b/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java @@ -23,7 +23,6 @@ import org.apache.log4j.AppenderSkeleton; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.spi.LoggingEvent; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.apache.lucene.util.LuceneTestCase.Slow; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -126,145 +125,12 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest { } - // #6626: make sure we can update throttle settings and the changes take effect - @Test - @Slow - public void testUpdateThrottleSettings() { - - // No throttling at first, only 1 non-replicated shard, force lots of merging: - assertAcked(prepareCreate("test") - .setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "none") - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") - .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2") - .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2") - .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "1") - .put(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT, "2") - )); - ensureGreen(); - long termUpto = 0; - for(int i=0;i<100;i++) { - // Provoke slowish merging by making many unique terms: - StringBuilder sb = new StringBuilder(); - for(int j=0;j<100;j++) { - sb.append(' '); - sb.append(termUpto++); - } - client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); - if (i % 2 == 0) { - refresh(); - } - } - - // No merge IO throttling should have happened: - NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); - for(NodeStats stats : nodesStats.getNodes()) { - assertThat(stats.getIndices().getStore().getThrottleTime().getMillis(), equalTo(0l)); - } - - logger.info("test: set low merge throttling"); - - // Now updates settings to turn on merge throttling lowish rate - client() - .admin() - .indices() - .prepareUpdateSettings("test") - .setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, "1mb")) - .get(); - - // Make sure setting says it is in fact changed: - GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get(); - assertThat(getSettingsResponse.getSetting("test", AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE), equalTo("merge")); - - // Also make sure we see throttling kicking in: - boolean done = false; - while (done == false) { - // Provoke slowish merging by making many unique terms: - for(int i=0;i<5;i++) { - StringBuilder sb = new StringBuilder(); - for(int j=0;j<100;j++) { - sb.append(' '); - sb.append(termUpto++); - sb.append(" some random text that keeps repeating over and over again hambone"); - } - client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); - } - refresh(); - nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); - for(NodeStats stats : nodesStats.getNodes()) { - long throttleMillis = stats.getIndices().getStore().getThrottleTime().getMillis(); - if (throttleMillis > 0) { - done = true; - break; - } - } - } - - logger.info("test: disable merge throttling"); - - // Now updates settings to disable merge throttling - client() - .admin() - .indices() - .prepareUpdateSettings("test") - .setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "none")) - .get(); - - // Optimize does a waitForMerges, which we must do to make sure all in-flight (throttled) merges finish: - logger.info("test: optimize"); - client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); - logger.info("test: optimize done"); - - // Record current throttling so far - long sumThrottleTime = 0; - nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); - for(NodeStats stats : nodesStats.getNodes()) { - sumThrottleTime += stats.getIndices().getStore().getThrottleTime().getMillis(); - } - - // Make sure no further throttling happens: - for(int i=0;i<100;i++) { - // Provoke slowish merging by making many unique terms: - StringBuilder sb = new StringBuilder(); - for(int j=0;j<100;j++) { - sb.append(' '); - sb.append(termUpto++); - } - client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); - if (i % 2 == 0) { - refresh(); - } - } - logger.info("test: done indexing after disabling throttling"); - - long newSumThrottleTime = 0; - nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); - for(NodeStats stats : nodesStats.getNodes()) { - newSumThrottleTime += stats.getIndices().getStore().getThrottleTime().getMillis(); - } - - // No additional merge IO throttling should have happened: - assertEquals(sumThrottleTime, newSumThrottleTime); - - // Optimize & flush and wait; else we sometimes get a "Delete Index failed - not acked" - // when ElasticsearchIntegrationTest.after tries to remove indices created by the test: - - // Wait for merges to finish - client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); - flush(); - - logger.info("test: test done"); - } - private static class MockAppender extends AppenderSkeleton { public boolean sawIndexWriterMessage; public boolean sawFlushDeletes; public boolean sawMergeThreadPaused; - public boolean sawUpdateSetting; + public boolean sawUpdateMaxThreadCount; + public boolean sawUpdateAutoThrottle; @Override protected void append(LoggingEvent event) { @@ -274,8 +140,11 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest { sawFlushDeletes |= message.contains("IW: apply all deletes during flush"); sawMergeThreadPaused |= message.contains("CMS: pause thread"); } - if (event.getLevel() == Level.INFO && message.contains("updating [max_thread_count] from [10000] to [1]")) { - sawUpdateSetting = true; + if (event.getLevel() == Level.INFO && message.contains("updating [index.merge.scheduler.max_thread_count] from [10000] to [1]")) { + sawUpdateMaxThreadCount = true; + } + if (event.getLevel() == Level.INFO && message.contains("updating [index.merge.scheduler.auto_throttle] from [true] to [false]")) { + sawUpdateAutoThrottle = true; } } @@ -289,10 +158,49 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest { } } + @Test + public void testUpdateAutoThrottleSettings() { + + MockAppender mockAppender = new MockAppender(); + Logger rootLogger = Logger.getRootLogger(); + Level savedLevel = rootLogger.getLevel(); + rootLogger.addAppender(mockAppender); + rootLogger.setLevel(Level.TRACE); + + try { + // No throttling at first, only 1 non-replicated shard, force lots of merging: + assertAcked(prepareCreate("test") + .setSettings(ImmutableSettings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2") + .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "1") + .put(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT, "2") + )); + + // Disable auto throttle: + client() + .admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(ImmutableSettings.builder() + .put(ConcurrentMergeSchedulerProvider.AUTO_THROTTLE, "no")) + .get(); + + assertTrue(mockAppender.sawUpdateAutoThrottle); + + // Make sure setting says it is in fact changed: + GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get(); + assertThat(getSettingsResponse.getSetting("test", ConcurrentMergeSchedulerProvider.AUTO_THROTTLE), equalTo("no")); + } finally { + rootLogger.removeAppender(mockAppender); + rootLogger.setLevel(savedLevel); + } + } + // #6882: make sure we can change index.merge.scheduler.max_thread_count live @Test - @Slow - @AwaitsFix(bugUrl="Super slow because of LUCENE-6119. Muted until we clean up merge throttling.") public void testUpdateMergeMaxThreadCount() { MockAppender mockAppender = new MockAppender(); @@ -303,11 +211,8 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest { try { - // Tons of merge threads allowed, only 1 non-replicated shard, force lots of merging, throttle so they fall behind: assertAcked(prepareCreate("test") .setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, "1mb") .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2") @@ -316,79 +221,33 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest { .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "10000") .put(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT, "10000") )); - ensureGreen(); - long termUpto = 0; - for(int i=0;i<100;i++) { - // Provoke slowish merging by making many unique terms: - StringBuilder sb = new StringBuilder(); - for(int j=0;j<100;j++) { - sb.append(' '); - sb.append(termUpto++); - } - client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); - if (i % 2 == 0) { - refresh(); - } - } - assertTrue(mockAppender.sawFlushDeletes); - assertFalse(mockAppender.sawMergeThreadPaused); - mockAppender.sawFlushDeletes = false; - mockAppender.sawMergeThreadPaused = false; + assertFalse(mockAppender.sawUpdateMaxThreadCount); - assertFalse(mockAppender.sawUpdateSetting); - - // Now make a live change to reduce allowed merge threads, and waaay over-throttle merging so they fall behind: + // Now make a live change to reduce allowed merge threads: client() .admin() .indices() .prepareUpdateSettings("test") .setSettings(ImmutableSettings.builder() .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "1") - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, "10kb") ) .get(); - try { + // Make sure we log the change: + assertTrue(mockAppender.sawUpdateMaxThreadCount); - // Make sure we log the change: - assertTrue(mockAppender.sawUpdateSetting); - - int i = 0; - while (true) { - // Provoke slowish merging by making many unique terms: - StringBuilder sb = new StringBuilder(); - for(int j=0;j<100;j++) { - sb.append(' '); - sb.append(termUpto++); - } - client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); - if (i % 2 == 0) { - refresh(); - } - // This time we should see some merges were in fact paused: - if (mockAppender.sawMergeThreadPaused) { - break; - } - i++; - } - } finally { - // Make merges fast again & finish merges before we try to close; else we sometimes get a "Delete Index failed - not acked" - // when ElasticsearchIntegrationTest.after tries to remove indices created by the test: - client() - .admin() - .indices() - .prepareUpdateSettings("test") - .setSettings(ImmutableSettings.builder() - .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "3") - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, "20mb") - ) - .get(); - - // Wait for merges to finish - client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); - } + client() + .admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(ImmutableSettings.builder() + .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "3") + ) + .get(); + // Wait for merges to finish + client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); } finally { rootLogger.removeAppender(mockAppender); diff --git a/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java b/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java index 0adebd64bb3..5f4a7800c7d 100644 --- a/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java +++ b/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java @@ -301,90 +301,6 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest { assertThat(client().admin().indices().prepareStats("idx").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0l)); } - - @Test - public void nonThrottleStats() throws Exception { - assertAcked(prepareCreate("test") - .setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") - .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2") - .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2") - .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "1") - .put(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT, "10000") - )); - ensureGreen(); - long termUpto = 0; - IndicesStatsResponse stats; - // Provoke slowish merging by making many unique terms: - for(int i=0; i<100; i++) { - StringBuilder sb = new StringBuilder(); - for(int j=0; j<100; j++) { - sb.append(' '); - sb.append(termUpto++); - sb.append(" some random text that keeps repeating over and over again hambone"); - } - client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); - } - refresh(); - stats = client().admin().indices().prepareStats().execute().actionGet(); - //nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); - - stats = client().admin().indices().prepareStats().execute().actionGet(); - assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0l)); - } - - @Test - public void throttleStats() throws Exception { - assertAcked(prepareCreate("test") - .setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") - .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2") - .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2") - .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "1") - .put(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT, "1") - .put("index.merge.policy.type", "tiered") - - )); - ensureGreen(); - long termUpto = 0; - IndicesStatsResponse stats; - // make sure we see throttling kicking in: - boolean done = false; - long start = System.currentTimeMillis(); - while (!done) { - for(int i=0; i<100; i++) { - // Provoke slowish merging by making many unique terms: - StringBuilder sb = new StringBuilder(); - for(int j=0; j<100; j++) { - sb.append(' '); - sb.append(termUpto++); - } - client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); - if (i % 2 == 0) { - refresh(); - } - } - refresh(); - stats = client().admin().indices().prepareStats().execute().actionGet(); - //nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); - done = stats.getPrimaries().getIndexing().getTotal().getThrottleTimeInMillis() > 0; - if (System.currentTimeMillis() - start > 300*1000) { //Wait 5 minutes for throttling to kick in - fail("index throttling didn't kick in after 5 minutes of intense merging"); - } - } - - // Optimize & flush and wait; else we sometimes get a "Delete Index failed - not acked" - // when ElasticsearchIntegrationTest.after tries to remove indices created by the test: - logger.info("test: now optimize"); - client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); - flush(); - logger.info("test: test done"); - } - @Test public void simpleStats() throws Exception { createIndex("test1", "test2"); @@ -524,6 +440,8 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest { assertThat(stats.getTotal().getMerge(), notNullValue()); assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0l)); + assertThat(stats.getTotal().getMerge().getTotalStoppedTime(), notNullValue()); + assertThat(stats.getTotal().getMerge().getTotalThrottledTime(), notNullValue()); } @Test diff --git a/src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java b/src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java index 9fe7a037850..adab2a2df18 100644 --- a/src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java +++ b/src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java @@ -54,63 +54,48 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest { String storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); Path[] dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } - assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); createIndexWithStoreType("test", IndexStoreModule.Type.NIOFS, "random"); storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(random[rate_limited(niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(random[niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } - assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); createIndexWithStoreType("test", IndexStoreModule.Type.MMAPFS, "least_used"); storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(mmapfs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[mmapfs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(mmapfs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), mmapfs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } - assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); createIndexWithStoreType("test", IndexStoreModule.Type.SIMPLEFS, "least_used"); storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(simplefs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[simplefs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(simplefs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), simplefs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } - assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); createIndexWithStoreType("test", IndexStoreModule.Type.DEFAULT, "least_used"); storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(default(mmapfs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[default(mmapfs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); assertThat(storeString.toLowerCase(Locale.ROOT), containsString("),niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(default(mmapfs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), default(mmapfs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } - assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); - - createIndexWithoutRateLimitingStoreType("test", IndexStoreModule.Type.NIOFS, "least_used"); - storeString = getStoreDirectory("test", 0).toString(); - logger.info(storeString); - dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); - if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); - } - assertThat(storeString, endsWith(")])")); } private void createIndexWithStoreType(String index, IndexStoreModule.Type storeType, String distributor) { @@ -126,21 +111,6 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest { assertThat(client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet().isTimedOut(), equalTo(false)); } - private void createIndexWithoutRateLimitingStoreType(String index, IndexStoreModule.Type storeType, String distributor) { - cluster().wipeIndices(index); - client().admin().indices().prepareCreate(index) - .setSettings(settingsBuilder() - .put("index.store.distributor", distributor) - .put("index.store.type", storeType) - .put("index.store.throttle.type", "none") - .put("index.number_of_replicas", 0) - .put("index.number_of_shards", 1) - ) - .execute().actionGet(); - assertThat(client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet().isTimedOut(), equalTo(false)); - } - - private Path[] dataPaths() { Set nodes = internalCluster().nodesInclude("test"); assertThat(nodes.isEmpty(), equalTo(false)); diff --git a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java index fad835d5e24..7c3a00ed0e2 100644 --- a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java @@ -633,11 +633,6 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests asyncIndexThreads[i].join(); } - logger.info("--> update index settings to back to normal"); - assertAcked(client().admin().indices().prepareUpdateSettings("test-*").setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "node") - )); - // Make sure that snapshot finished - doesn't matter if it failed or succeeded try { CreateSnapshotResponse snapshotResponse = snapshotResponseFuture.get(); @@ -679,11 +674,6 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests for (int i = 0; i < between(10, 500); i++) { index(name, "doc", Integer.toString(i), "foo", "bar" + i); } - - assertAcked(client().admin().indices().prepareUpdateSettings(name).setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "all") - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, between(100, 50000)) - )); } public static abstract class TestCustomMetaData implements MetaData.Custom { diff --git a/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java index 08bc0d3c1fc..81d14aea5da 100644 --- a/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java @@ -1361,12 +1361,6 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests { refresh(); assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); - // Update settings to make sure that relocation is slow so we can start snapshot before relocation is finished - assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "all") - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, 100) - )); - logger.info("--> start relocations"); allowNodes("test-idx", internalCluster().numDataNodes()); @@ -1377,11 +1371,6 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests { logger.info("--> snapshot"); client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get(); - // Update settings to back to normal - assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(ImmutableSettings.builder() - .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "node") - )); - logger.info("--> wait for snapshot to complete"); SnapshotInfo snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600)); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java index d37dfbadd7e..6f4f0e94a4b 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java @@ -27,7 +27,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Predicate; import com.google.common.collect.Lists; -import org.apache.lucene.store.StoreRateLimiting; import org.apache.lucene.util.AbstractRandomizedTest; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; @@ -430,16 +429,6 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase setRandomTranslogSettings(random, builder); setRandomNormsLoading(random, builder); setRandomScriptingSettings(random, builder); - if (random.nextBoolean()) { - if (random.nextInt(10) == 0) { // do something crazy slow here - builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB)); - } else { - builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB)); - } - } - if (random.nextBoolean()) { - builder.put(IndicesStore.INDICES_STORE_THROTTLE_TYPE, RandomPicks.randomFrom(random, StoreRateLimiting.Type.values())); - } if (random.nextBoolean()) { builder.put(StoreModule.DISTIBUTOR_KEY, random.nextBoolean() ? StoreModule.LEAST_USED_DISTRIBUTOR : StoreModule.RANDOM_WEIGHT_DISTRIBUTOR); diff --git a/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java index bb2a262c278..fccd693866b 100644 --- a/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java +++ b/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java @@ -24,7 +24,6 @@ import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.StoreRateLimiting; import org.apache.lucene.util.AbstractRandomizedTest; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; @@ -151,21 +150,6 @@ public class MockFSDirectoryService extends FsDirectoryService { } } - @Override - public void onPause(long nanos) { - delegateService.onPause(nanos); - } - - @Override - public StoreRateLimiting rateLimiting() { - return delegateService.rateLimiting(); - } - - @Override - public long throttleTimeInNanos() { - return delegateService.throttleTimeInNanos(); - } - @Override public Directory newFromDistributor(Distributor distributor) throws IOException { return helper.wrap(super.newFromDistributor(distributor)); From 1aad275c55dfdd2ff7431246a2024213bc8c75cd Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Sun, 11 Jan 2015 05:52:43 -0500 Subject: [PATCH 2/6] expose current CMS throttle in merge stats; fix tests, docs; also log per-merge stop/throttle/rate --- docs/reference/cluster/nodes-stats.asciidoc | 2 +- docs/reference/cluster/stats.asciidoc | 2 -- .../cluster/update-settings.asciidoc | 9 --------- docs/reference/index-modules/merge.asciidoc | 19 ++++++++++++------- .../indices/update-settings.asciidoc | 3 +++ docs/reference/modules/indices.asciidoc | 14 -------------- .../TrackingConcurrentMergeScheduler.java | 15 +++++++++++---- .../elasticsearch/index/merge/MergeStats.java | 15 ++++++++++++++- .../ConcurrentMergeSchedulerProvider.java | 4 +++- .../indices/settings/UpdateSettingsTests.java | 15 ++++----------- .../indices/stats/IndexStatsTests.java | 1 + 11 files changed, 49 insertions(+), 50 deletions(-) diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index dbb63c7e9fd..81d027f8779 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -25,7 +25,7 @@ of `indices`, `os`, `process`, `jvm`, `network`, `transport`, `http`, [horizontal] `indices`:: Indices stats about size, document count, indexing and - deletion times, search times, field cache size , merges and flushes + deletion times, search times, field cache size, merges and flushes `fs`:: File system information, data path, free disk space, read/write diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 512c5a4da37..3ce6b92ed03 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -48,8 +48,6 @@ Will return, for example: "store": { "size": "5.6kb", "size_in_bytes": 5770, - "throttle_time": "0s", - "throttle_time_in_millis": 0 }, "fielddata": { "memory_size": "0b", diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc index 9962cf2bb23..aded0fda24b 100644 --- a/docs/reference/cluster/update-settings.asciidoc +++ b/docs/reference/cluster/update-settings.asciidoc @@ -183,15 +183,6 @@ due to forced awareness or allocation filtering. `indices.recovery.max_bytes_per_sec`:: See <> -[float] -==== Store level throttling - -`indices.store.throttle.type`:: - See <> - -`indices.store.throttle.max_bytes_per_sec`:: - See <> - [float] [[logger]] === Logger diff --git a/docs/reference/index-modules/merge.asciidoc b/docs/reference/index-modules/merge.asciidoc index aaf42a57429..4d276f00037 100644 --- a/docs/reference/index-modules/merge.asciidoc +++ b/docs/reference/index-modules/merge.asciidoc @@ -7,12 +7,6 @@ where the index data is stored, and are immutable up to delete markers. Segments are, periodically, merged into larger segments to keep the index size at bay and expunge deletes. -The more segments one has in the Lucene index means slower searches and -more memory used. Segment merging is used to reduce the number of segments, -however merges can be expensive to perform, especially on low IO environments. -Merges can be throttled using <>. - - [float] [[policy]] === Policy @@ -194,10 +188,21 @@ scheduler supports this setting: `index.merge.scheduler.max_thread_count`:: The maximum number of threads that may be merging at once. Defaults to -`Math.max(1, Math.min(3, Runtime.getRuntime().availableProcessors() / 2))` +`Math.max(1, Math.min(4, Runtime.getRuntime().availableProcessors() / 2))` which works well for a good solid-state-disk (SSD). If your index is on spinning platter drives instead, decrease this to 1. +`index.merge.scheduler.auto_throttle`:: + +If this is true (the default), then the merge scheduler will +rate-limit IO (writes) for merges to an adaptive value depending on +how many merges are requested over time. An application with a low +indexing rate that unluckily suddenly requires a large merge will see +that merge aggressively throttled, while an application doing heavy +indexing will see the throttle move higher to allow merges to keep up +with ongoing indexing. This is a dynamic setting (you can <<../indices/update-settings,change it +at any time on a running index>>). + [float] ==== SerialMergeScheduler diff --git a/docs/reference/indices/update-settings.asciidoc b/docs/reference/indices/update-settings.asciidoc index 036de02c7a4..626a4e00870 100644 --- a/docs/reference/indices/update-settings.asciidoc +++ b/docs/reference/indices/update-settings.asciidoc @@ -90,6 +90,9 @@ settings API: All the settings for the merge policy currently configured. A different merge policy can't be set. +`index.merge.scheduler.*`:: + All the settings for the merge scheduler. + `index.routing.allocation.include.*`:: A node matching any rule will be allowed to host shards from the index. diff --git a/docs/reference/modules/indices.asciidoc b/docs/reference/modules/indices.asciidoc index 16c22a9dba7..774178e7b80 100644 --- a/docs/reference/modules/indices.asciidoc +++ b/docs/reference/modules/indices.asciidoc @@ -60,17 +60,3 @@ The following settings can be set to manage the recovery policy: `indices.recovery.max_bytes_per_sec`:: defaults to `20mb`. - -[float] -[[throttling]] -=== Store level throttling - -The following settings can be set to control the store throttling: - -[horizontal] -`indices.store.throttle.type`:: - could be `merge` (default), `none` or `all`. See <>. - -`indices.store.throttle.max_bytes_per_sec`:: - defaults to `20mb`. - diff --git a/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java b/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java index d5509bab383..94d86087f9c 100644 --- a/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java +++ b/src/main/java/org/apache/lucene/index/TrackingConcurrentMergeScheduler.java @@ -129,15 +129,22 @@ public class TrackingConcurrentMergeScheduler extends ConcurrentMergeScheduler { totalMergesSizeInBytes.inc(totalSizeInBytes); totalMerges.inc(took); - totalMergeStoppedTime.inc(merge.rateLimiter.getTotalStoppedNS()/1000000); - totalMergeThrottledTime.inc(merge.rateLimiter.getTotalPausedNS()/1000000); + long stoppedMS = merge.rateLimiter.getTotalStoppedNS()/1000000; + long throttledMS = merge.rateLimiter.getTotalPausedNS()/1000000; + + totalMergeStoppedTime.inc(stoppedMS); + totalMergeThrottledTime.inc(throttledMS); String message = String.format(Locale.ROOT, - "merge segment [%s] done: took [%s], [%,.1f MB], [%,d docs]", + "merge segment [%s] done: took [%s], [%,.1f MB], [%,d docs], [%s stopped], [%s throttled], [%,.1f MB written], [%,.1f MB/sec throttle]", merge.info == null ? "_na_" : merge.info.info.name, TimeValue.timeValueMillis(took), totalSizeInBytes/1024f/1024f, - totalNumDocs); + totalNumDocs, + TimeValue.timeValueMillis(stoppedMS), + TimeValue.timeValueMillis(throttledMS), + merge.rateLimiter.getTotalBytesWritten()/1024f/1024f, + merge.rateLimiter.getMBPerSec()); if (took > 20000) { // if more than 20 seconds, DEBUG log it logger.debug(message); diff --git a/src/main/java/org/elasticsearch/index/merge/MergeStats.java b/src/main/java/org/elasticsearch/index/merge/MergeStats.java index 7664ab5f277..ca53ac7aa21 100644 --- a/src/main/java/org/elasticsearch/index/merge/MergeStats.java +++ b/src/main/java/org/elasticsearch/index/merge/MergeStats.java @@ -47,12 +47,14 @@ public class MergeStats implements Streamable, ToXContent { /** Total millis that we slept during writes so merge IO is throttled. */ private long totalThrottledTimeInMillis; + private long totalBytesPerSecAutoThrottle; + public MergeStats() { } public void add(long totalMerges, long totalMergeTime, long totalNumDocs, long totalSizeInBytes, long currentMerges, long currentNumDocs, long currentSizeInBytes, - long stoppedTimeMillis, long throttledTimeMillis) { + long stoppedTimeMillis, long throttledTimeMillis, double mbPerSecAutoThrottle) { this.total += totalMerges; this.totalTimeInMillis += totalMergeTime; this.totalNumDocs += totalNumDocs; @@ -62,6 +64,7 @@ public class MergeStats implements Streamable, ToXContent { this.currentSizeInBytes += currentSizeInBytes; this.totalStoppedTimeInMillis += stoppedTimeMillis; this.totalThrottledTimeInMillis += throttledTimeMillis; + this.totalBytesPerSecAutoThrottle += (long) (mbPerSecAutoThrottle * 1024 * 1024); } public void add(MergeStats mergeStats) { @@ -77,6 +80,7 @@ public class MergeStats implements Streamable, ToXContent { this.currentSizeInBytes += mergeStats.currentSizeInBytes; this.totalStoppedTimeInMillis += mergeStats.totalStoppedTimeInMillis; this.totalThrottledTimeInMillis += mergeStats.totalThrottledTimeInMillis; + this.totalBytesPerSecAutoThrottle += mergeStats.totalBytesPerSecAutoThrottle; } /** @@ -140,6 +144,10 @@ public class MergeStats implements Streamable, ToXContent { return new ByteSizeValue(totalSizeInBytes); } + public long getTotalBytesPerSecAutoThrottle() { + return totalBytesPerSecAutoThrottle; + } + /** * The current number of merges executing. */ @@ -177,6 +185,7 @@ public class MergeStats implements Streamable, ToXContent { builder.byteSizeField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, totalSizeInBytes); builder.timeValueField(Fields.TOTAL_STOPPED_TIME_IN_MILLIS, Fields.TOTAL_STOPPED_TIME, totalStoppedTimeInMillis); builder.timeValueField(Fields.TOTAL_THROTTLED_TIME_IN_MILLIS, Fields.TOTAL_THROTTLED_TIME, totalThrottledTimeInMillis); + builder.byteSizeField(Fields.TOTAL_THROTTLE_BYTES_PER_SEC_IN_BYTES, Fields.TOTAL_THROTTLE_BYTES_PER_SEC, totalBytesPerSecAutoThrottle); builder.endObject(); return builder; } @@ -197,6 +206,8 @@ public class MergeStats implements Streamable, ToXContent { static final XContentBuilderString TOTAL_DOCS = new XContentBuilderString("total_docs"); static final XContentBuilderString TOTAL_SIZE = new XContentBuilderString("total_size"); static final XContentBuilderString TOTAL_SIZE_IN_BYTES = new XContentBuilderString("total_size_in_bytes"); + static final XContentBuilderString TOTAL_THROTTLE_BYTES_PER_SEC_IN_BYTES = new XContentBuilderString("total_auto_throttle_in_bytes"); + static final XContentBuilderString TOTAL_THROTTLE_BYTES_PER_SEC = new XContentBuilderString("total_auto_throttle"); } @Override @@ -211,6 +222,7 @@ public class MergeStats implements Streamable, ToXContent { if (in.getVersion().onOrAfter(Version.V_2_0_0)) { totalStoppedTimeInMillis = in.readVLong(); totalThrottledTimeInMillis = in.readVLong(); + totalBytesPerSecAutoThrottle = in.readVLong(); } } @@ -226,6 +238,7 @@ public class MergeStats implements Streamable, ToXContent { if (out.getVersion().onOrAfter(Version.V_2_0_0)) { out.writeVLong(totalStoppedTimeInMillis); out.writeVLong(totalThrottledTimeInMillis); + out.writeVLong(totalBytesPerSecAutoThrottle); } } } diff --git a/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java b/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java index ca8acc2f0d3..c4bdc179c69 100644 --- a/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java +++ b/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java @@ -86,11 +86,13 @@ public class ConcurrentMergeSchedulerProvider extends MergeSchedulerProvider { @Override public MergeStats stats() { MergeStats mergeStats = new MergeStats(); + // TODO: why would there be more than one CMS for a single shard...? for (CustomConcurrentMergeScheduler scheduler : schedulers) { mergeStats.add(scheduler.totalMerges(), scheduler.totalMergeTime(), scheduler.totalMergeNumDocs(), scheduler.totalMergeSizeInBytes(), scheduler.currentMerges(), scheduler.currentMergesNumDocs(), scheduler.currentMergesSizeInBytes(), scheduler.totalMergeStoppedTimeMillis(), - scheduler.totalMergeThrottledTimeMillis()); + scheduler.totalMergeThrottledTimeMillis(), + autoThrottle ? scheduler.getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY); } return mergeStats; } diff --git a/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java b/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java index 812e6c43131..d03577fb07d 100644 --- a/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java +++ b/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java @@ -188,6 +188,7 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest { .put(ConcurrentMergeSchedulerProvider.AUTO_THROTTLE, "no")) .get(); + // Make sure we log the change: assertTrue(mockAppender.sawUpdateAutoThrottle); // Make sure setting says it is in fact changed: @@ -237,17 +238,9 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest { // Make sure we log the change: assertTrue(mockAppender.sawUpdateMaxThreadCount); - client() - .admin() - .indices() - .prepareUpdateSettings("test") - .setSettings(ImmutableSettings.builder() - .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "3") - ) - .get(); - - // Wait for merges to finish - client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); + // Make sure setting says it is in fact changed: + GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get(); + assertThat(getSettingsResponse.getSetting("test", ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT), equalTo("1")); } finally { rootLogger.removeAppender(mockAppender); diff --git a/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java b/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java index 5f4a7800c7d..0d7621ac783 100644 --- a/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java +++ b/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java @@ -442,6 +442,7 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest { assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0l)); assertThat(stats.getTotal().getMerge().getTotalStoppedTime(), notNullValue()); assertThat(stats.getTotal().getMerge().getTotalThrottledTime(), notNullValue()); + assertThat(stats.getTotal().getMerge().getTotalBytesPerSecAutoThrottle(), greaterThan(0l)); } @Test From ff3e2cb569ee78ce7a602ce09e2799c9c4619630 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Tue, 13 Jan 2015 14:21:21 -0500 Subject: [PATCH 3/6] remove version checks --- .../elasticsearch/index/merge/MergeStats.java | 18 ++++++++---------- .../elasticsearch/index/store/StoreStats.java | 8 -------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/merge/MergeStats.java b/src/main/java/org/elasticsearch/index/merge/MergeStats.java index ca53ac7aa21..a5cf9cc62fd 100644 --- a/src/main/java/org/elasticsearch/index/merge/MergeStats.java +++ b/src/main/java/org/elasticsearch/index/merge/MergeStats.java @@ -219,11 +219,10 @@ public class MergeStats implements Streamable, ToXContent { current = in.readVLong(); currentNumDocs = in.readVLong(); currentSizeInBytes = in.readVLong(); - if (in.getVersion().onOrAfter(Version.V_2_0_0)) { - totalStoppedTimeInMillis = in.readVLong(); - totalThrottledTimeInMillis = in.readVLong(); - totalBytesPerSecAutoThrottle = in.readVLong(); - } + // Added in 2.0: + totalStoppedTimeInMillis = in.readVLong(); + totalThrottledTimeInMillis = in.readVLong(); + totalBytesPerSecAutoThrottle = in.readVLong(); } @Override @@ -235,10 +234,9 @@ public class MergeStats implements Streamable, ToXContent { out.writeVLong(current); out.writeVLong(currentNumDocs); out.writeVLong(currentSizeInBytes); - if (out.getVersion().onOrAfter(Version.V_2_0_0)) { - out.writeVLong(totalStoppedTimeInMillis); - out.writeVLong(totalThrottledTimeInMillis); - out.writeVLong(totalBytesPerSecAutoThrottle); - } + // Added in 2.0: + out.writeVLong(totalStoppedTimeInMillis); + out.writeVLong(totalThrottledTimeInMillis); + out.writeVLong(totalBytesPerSecAutoThrottle); } } diff --git a/src/main/java/org/elasticsearch/index/store/StoreStats.java b/src/main/java/org/elasticsearch/index/store/StoreStats.java index 1bdae5d4771..2db10ff0f80 100644 --- a/src/main/java/org/elasticsearch/index/store/StoreStats.java +++ b/src/main/java/org/elasticsearch/index/store/StoreStats.java @@ -78,19 +78,11 @@ public class StoreStats implements Streamable, ToXContent { @Override public void readFrom(StreamInput in) throws IOException { sizeInBytes = in.readVLong(); - if (in.getVersion().before(Version.V_2_0_0)) { - // Ignore throttleTimeInNanos - in.readVLong(); - } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(sizeInBytes); - if (out.getVersion().before(Version.V_2_0_0)) { - // Send dummy throttleTimeInNanos - out.writeVLong(0); - } } @Override From 87d8fbff285b5469e22537fdb3a983abe23de61f Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 14 Jan 2015 05:26:37 -0500 Subject: [PATCH 4/6] put back fixed throttling, but off by default --- .../lucene/store/RateLimitedFSDirectory.java | 106 ++++++++++++++++++ .../lucene/store/StoreRateLimiting.java | 94 ++++++++++++++++ .../lucene}/store/StoreUtils.java | 0 3 files changed, 200 insertions(+) create mode 100644 src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java create mode 100644 src/main/java/org/apache/lucene/store/StoreRateLimiting.java rename src/main/java/org/{elasticsearch/index => apache/lucene}/store/StoreUtils.java (100%) diff --git a/src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java b/src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java new file mode 100644 index 00000000000..e6ec10ca34b --- /dev/null +++ b/src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java @@ -0,0 +1,106 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.store; + +import org.apache.lucene.store.IOContext.Context; + +import java.io.IOException; + +public final class RateLimitedFSDirectory extends FilterDirectory { + + private final StoreRateLimiting.Provider rateLimitingProvider; + + private final StoreRateLimiting.Listener rateListener; + + public RateLimitedFSDirectory(Directory wrapped, StoreRateLimiting.Provider rateLimitingProvider, + StoreRateLimiting.Listener rateListener) { + super(wrapped); + this.rateLimitingProvider = rateLimitingProvider; + this.rateListener = rateListener; + } + + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + final IndexOutput output = in.createOutput(name, context); + + StoreRateLimiting rateLimiting = rateLimitingProvider.rateLimiting(); + StoreRateLimiting.Type type = rateLimiting.getType(); + RateLimiter limiter = rateLimiting.getRateLimiter(); + if (type == StoreRateLimiting.Type.NONE || limiter == null) { + return output; + } + if (context.context == Context.MERGE || type == StoreRateLimiting.Type.ALL) { + // we are merging, and type is either MERGE or ALL, rate limit... + return new RateLimitedIndexOutput(new RateLimiterWrapper(limiter, rateListener), output); + } + // we shouldn't really get here... + return output; + } + + + @Override + public void close() throws IOException { + in.close(); + } + + @Override + public String toString() { + StoreRateLimiting rateLimiting = rateLimitingProvider.rateLimiting(); + StoreRateLimiting.Type type = rateLimiting.getType(); + RateLimiter limiter = rateLimiting.getRateLimiter(); + if (type == StoreRateLimiting.Type.NONE || limiter == null) { + return StoreUtils.toString(in); + } else { + return "rate_limited(" + StoreUtils.toString(in) + ", type=" + type.name() + ", rate=" + limiter.getMBPerSec() + ")"; + } + } + + // we wrap the limiter to notify our store if we limited to get statistics + static final class RateLimiterWrapper extends RateLimiter { + private final RateLimiter delegate; + private final StoreRateLimiting.Listener rateListener; + + RateLimiterWrapper(RateLimiter delegate, StoreRateLimiting.Listener rateListener) { + this.delegate = delegate; + this.rateListener = rateListener; + } + + @Override + public void setMBPerSec(double mbPerSec) { + delegate.setMBPerSec(mbPerSec); + } + + @Override + public double getMBPerSec() { + return delegate.getMBPerSec(); + } + + @Override + public long pause(long bytes) throws IOException { + long pause = delegate.pause(bytes); + rateListener.onPause(pause); + return pause; + } + + @Override + public long getMinPauseCheckBytes() { + return delegate.getMinPauseCheckBytes(); + } + } +} diff --git a/src/main/java/org/apache/lucene/store/StoreRateLimiting.java b/src/main/java/org/apache/lucene/store/StoreRateLimiting.java new file mode 100644 index 00000000000..ae021b07b09 --- /dev/null +++ b/src/main/java/org/apache/lucene/store/StoreRateLimiting.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.store; + +import org.apache.lucene.store.RateLimiter.SimpleRateLimiter; +import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.unit.ByteSizeValue; + +/** + */ +public class StoreRateLimiting { + + public static interface Provider { + + StoreRateLimiting rateLimiting(); + } + + public interface Listener { + + void onPause(long nanos); + } + + public static enum Type { + NONE, + MERGE, + ALL; + + public static Type fromString(String type) throws ElasticsearchIllegalArgumentException { + if ("none".equalsIgnoreCase(type)) { + return NONE; + } else if ("merge".equalsIgnoreCase(type)) { + return MERGE; + } else if ("all".equalsIgnoreCase(type)) { + return ALL; + } + throw new ElasticsearchIllegalArgumentException("rate limiting type [" + type + "] not valid, can be one of [all|merge|none]"); + } + } + + private final SimpleRateLimiter rateLimiter = new SimpleRateLimiter(0); + private volatile SimpleRateLimiter actualRateLimiter; + + private volatile Type type; + + public StoreRateLimiting() { + + } + + @Nullable + public RateLimiter getRateLimiter() { + return actualRateLimiter; + } + + public void setMaxRate(ByteSizeValue rate) { + if (rate.bytes() <= 0) { + actualRateLimiter = null; + } else if (actualRateLimiter == null) { + actualRateLimiter = rateLimiter; + actualRateLimiter.setMBPerSec(rate.mbFrac()); + } else { + assert rateLimiter == actualRateLimiter; + rateLimiter.setMBPerSec(rate.mbFrac()); + } + } + + public Type getType() { + return type; + } + + public void setType(Type type) { + this.type = type; + } + + public void setType(String type) throws ElasticsearchIllegalArgumentException { + this.type = Type.fromString(type); + } +} diff --git a/src/main/java/org/elasticsearch/index/store/StoreUtils.java b/src/main/java/org/apache/lucene/store/StoreUtils.java similarity index 100% rename from src/main/java/org/elasticsearch/index/store/StoreUtils.java rename to src/main/java/org/apache/lucene/store/StoreUtils.java From 107099affa0f6e8bbacad140bb71bd1be62e1532 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 14 Jan 2015 05:35:09 -0500 Subject: [PATCH 5/6] put back fixed throttling, but off by default --- docs/reference/cluster/stats.asciidoc | 2 + .../cluster/update-settings.asciidoc | 9 ++ docs/reference/index-modules/merge.asciidoc | 6 + docs/reference/index-modules/store.asciidoc | 32 +++++ docs/reference/modules/indices.asciidoc | 14 ++ .../org/apache/lucene/store/StoreUtils.java | 11 +- .../ClusterDynamicSettingsModule.java | 2 + .../elasticsearch/index/merge/MergeStats.java | 20 ++- .../ConcurrentMergeSchedulerProvider.java | 3 + .../scheduler/MergeSchedulerProvider.java | 3 + .../settings/IndexDynamicSettingsModule.java | 2 + .../index/store/DirectoryService.java | 4 +- .../elasticsearch/index/store/IndexStore.java | 7 + .../org/elasticsearch/index/store/Store.java | 2 +- .../elasticsearch/index/store/StoreStats.java | 25 +++- .../distributor/AbstractDistributor.java | 12 +- .../index/store/fs/FsDirectoryService.java | 22 ++- .../store/support/AbstractIndexStore.java | 58 ++++++++ .../indices/store/IndicesStore.java | 45 ++++++ .../engine/internal/InternalEngineTests.java | 5 + .../merge/policy/MergePolicySettingsTest.java | 5 + .../elasticsearch/index/store/StoreTest.java | 5 + .../store/distributor/DistributorTests.java | 5 + .../indices/settings/UpdateSettingsTests.java | 134 ++++++++++++++++++ .../indices/stats/IndexStatsTests.java | 87 +++++++++++- .../indices/store/SimpleDistributorTests.java | 52 +++++-- .../DedicatedClusterSnapshotRestoreTests.java | 10 ++ .../SharedClusterSnapshotRestoreTests.java | 11 ++ .../test/ElasticsearchIntegrationTest.java | 15 +- .../test/store/MockFSDirectoryService.java | 16 +++ 30 files changed, 584 insertions(+), 40 deletions(-) diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 3ce6b92ed03..512c5a4da37 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -48,6 +48,8 @@ Will return, for example: "store": { "size": "5.6kb", "size_in_bytes": 5770, + "throttle_time": "0s", + "throttle_time_in_millis": 0 }, "fielddata": { "memory_size": "0b", diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc index aded0fda24b..9962cf2bb23 100644 --- a/docs/reference/cluster/update-settings.asciidoc +++ b/docs/reference/cluster/update-settings.asciidoc @@ -183,6 +183,15 @@ due to forced awareness or allocation filtering. `indices.recovery.max_bytes_per_sec`:: See <> +[float] +==== Store level throttling + +`indices.store.throttle.type`:: + See <> + +`indices.store.throttle.max_bytes_per_sec`:: + See <> + [float] [[logger]] === Logger diff --git a/docs/reference/index-modules/merge.asciidoc b/docs/reference/index-modules/merge.asciidoc index 4d276f00037..6fd4d65793a 100644 --- a/docs/reference/index-modules/merge.asciidoc +++ b/docs/reference/index-modules/merge.asciidoc @@ -7,6 +7,12 @@ where the index data is stored, and are immutable up to delete markers. Segments are, periodically, merged into larger segments to keep the index size at bay and expunge deletes. +The more segments one has in the Lucene index means slower searches and +more memory used. Segment merging is used to reduce the number of segments, +however merges can be expensive to perform, especially on low IO environments. +Merges can be throttled using <>. + + [float] [[policy]] === Policy diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc index 339f1f0d218..f7fdb86ad9a 100644 --- a/docs/reference/index-modules/store.asciidoc +++ b/docs/reference/index-modules/store.asciidoc @@ -18,6 +18,38 @@ heap space* using the "Memory" (see below) storage type. It translates to the fact that there is no need for extra large JVM heaps (with their own consequences) for storing the index in memory. + +[float] +[[store-throttling]] +=== Store Level Throttling + +The way Lucene, the IR library elasticsearch uses under the covers, +works is by creating immutable segments (up to deletes) and constantly +merging them (the merge policy settings allow to control how those +merges happen). The merge process happens in an asynchronous manner +without affecting the indexing / search speed. The problem though, +especially on systems with low IO, is that the merge process can be +expensive and affect search / index operation simply by the fact that +the box is now taxed with more IO happening. + +The store module allows to have throttling configured for merges (or +all) either on the node level, or on the index level. The node level +throttling will make sure that out of all the shards allocated on that +node, the merge process won't pass the specific setting bytes per +second. It can be set by setting `indices.store.throttle.type` to +`merge`, and setting `indices.store.throttle.max_bytes_per_sec` to +something like `5mb`. The node level settings can be changed dynamically +using the cluster update settings API. The default is disabled (set to `none`), +in favor of . + +If specific index level configuration is needed, regardless of the node +level settings, it can be set as well using the +`index.store.throttle.type`, and +`index.store.throttle.max_bytes_per_sec`. The default value for the type +is `node`, meaning it will throttle based on the node level settings and +participate in the global throttling happening. Both settings can be set +using the index update settings API dynamically. + [float] [[file-system]] === File system storage types diff --git a/docs/reference/modules/indices.asciidoc b/docs/reference/modules/indices.asciidoc index 774178e7b80..16c22a9dba7 100644 --- a/docs/reference/modules/indices.asciidoc +++ b/docs/reference/modules/indices.asciidoc @@ -60,3 +60,17 @@ The following settings can be set to manage the recovery policy: `indices.recovery.max_bytes_per_sec`:: defaults to `20mb`. + +[float] +[[throttling]] +=== Store level throttling + +The following settings can be set to control the store throttling: + +[horizontal] +`indices.store.throttle.type`:: + could be `merge` (default), `none` or `all`. See <>. + +`indices.store.throttle.max_bytes_per_sec`:: + defaults to `20mb`. + diff --git a/src/main/java/org/apache/lucene/store/StoreUtils.java b/src/main/java/org/apache/lucene/store/StoreUtils.java index 261e363db76..b7de08b1ec1 100644 --- a/src/main/java/org/apache/lucene/store/StoreUtils.java +++ b/src/main/java/org/apache/lucene/store/StoreUtils.java @@ -16,17 +16,12 @@ * specific language governing permissions and limitations * under the License. */ - -package org.elasticsearch.index.store; +package org.apache.lucene.store; import java.util.Arrays; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FileSwitchDirectory; -import org.apache.lucene.store.MMapDirectory; -import org.apache.lucene.store.NIOFSDirectory; -import org.apache.lucene.store.SimpleFSDirectory; - +/** + */ public final class StoreUtils { private StoreUtils() { diff --git a/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java b/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java index 9cd7c427b41..cb4aca624ea 100644 --- a/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java +++ b/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java @@ -65,6 +65,8 @@ public class ClusterDynamicSettingsModule extends AbstractModule { clusterDynamicSettings.addDynamicSetting(IndicesFilterCache.INDICES_CACHE_FILTER_SIZE); clusterDynamicSettings.addDynamicSetting(IndicesFilterCache.INDICES_CACHE_FILTER_EXPIRE, Validator.TIME); clusterDynamicSettings.addDynamicSetting(IndicesFilterCache.INDICES_CACHE_FILTER_CONCURRENCY_LEVEL, Validator.POSITIVE_INTEGER); + clusterDynamicSettings.addDynamicSetting(IndicesStore.INDICES_STORE_THROTTLE_TYPE); + clusterDynamicSettings.addDynamicSetting(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE); clusterDynamicSettings.addDynamicSetting(IndicesTTLService.INDICES_TTL_INTERVAL, Validator.TIME); clusterDynamicSettings.addDynamicSetting(MappingUpdatedAction.INDICES_MAPPING_ADDITIONAL_MAPPING_CHANGE_TIME, Validator.TIME); clusterDynamicSettings.addDynamicSetting(MetaData.SETTING_READ_ONLY); diff --git a/src/main/java/org/elasticsearch/index/merge/MergeStats.java b/src/main/java/org/elasticsearch/index/merge/MergeStats.java index a5cf9cc62fd..055558cc563 100644 --- a/src/main/java/org/elasticsearch/index/merge/MergeStats.java +++ b/src/main/java/org/elasticsearch/index/merge/MergeStats.java @@ -19,8 +19,6 @@ package org.elasticsearch.index.merge; -import java.io.IOException; - import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -31,6 +29,11 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; +import java.io.IOException; + +/** + * + */ public class MergeStats implements Streamable, ToXContent { private long total; @@ -64,7 +67,12 @@ public class MergeStats implements Streamable, ToXContent { this.currentSizeInBytes += currentSizeInBytes; this.totalStoppedTimeInMillis += stoppedTimeMillis; this.totalThrottledTimeInMillis += throttledTimeMillis; - this.totalBytesPerSecAutoThrottle += (long) (mbPerSecAutoThrottle * 1024 * 1024); + long bytesPerSecAutoThrottle = (long) (mbPerSecAutoThrottle * 1024 * 1024); + if (this.totalBytesPerSecAutoThrottle == Long.MAX_VALUE || bytesPerSecAutoThrottle == Long.MAX_VALUE) { + this.totalBytesPerSecAutoThrottle = Long.MAX_VALUE; + } else { + this.totalBytesPerSecAutoThrottle += bytesPerSecAutoThrottle; + } } public void add(MergeStats mergeStats) { @@ -80,7 +88,11 @@ public class MergeStats implements Streamable, ToXContent { this.currentSizeInBytes += mergeStats.currentSizeInBytes; this.totalStoppedTimeInMillis += mergeStats.totalStoppedTimeInMillis; this.totalThrottledTimeInMillis += mergeStats.totalThrottledTimeInMillis; - this.totalBytesPerSecAutoThrottle += mergeStats.totalBytesPerSecAutoThrottle; + if (this.totalBytesPerSecAutoThrottle == Long.MAX_VALUE || mergeStats.totalBytesPerSecAutoThrottle == Long.MAX_VALUE) { + this.totalBytesPerSecAutoThrottle = Long.MAX_VALUE; + } else { + this.totalBytesPerSecAutoThrottle += mergeStats.totalBytesPerSecAutoThrottle; + } } /** diff --git a/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java b/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java index c4bdc179c69..5605603ce97 100644 --- a/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java +++ b/src/main/java/org/elasticsearch/index/merge/scheduler/ConcurrentMergeSchedulerProvider.java @@ -41,6 +41,9 @@ import java.io.IOException; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; +/** + * + */ public class ConcurrentMergeSchedulerProvider extends MergeSchedulerProvider { private final IndexSettingsService indexSettingsService; diff --git a/src/main/java/org/elasticsearch/index/merge/scheduler/MergeSchedulerProvider.java b/src/main/java/org/elasticsearch/index/merge/scheduler/MergeSchedulerProvider.java index 11719ec866e..de9a7d8bed6 100644 --- a/src/main/java/org/elasticsearch/index/merge/scheduler/MergeSchedulerProvider.java +++ b/src/main/java/org/elasticsearch/index/merge/scheduler/MergeSchedulerProvider.java @@ -34,6 +34,9 @@ import java.io.Closeable; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +/** + * + */ public abstract class MergeSchedulerProvider extends AbstractIndexShardComponent implements IndexShardComponent, Closeable { public static interface FailureListener { diff --git a/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java b/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java index 044db0c2c6b..7a17bcd46ff 100644 --- a/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java +++ b/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java @@ -51,6 +51,8 @@ public class IndexDynamicSettingsModule extends AbstractModule { public IndexDynamicSettingsModule() { indexDynamicSettings = new DynamicSettings(); + indexDynamicSettings.addDynamicSetting(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE); + indexDynamicSettings.addDynamicSetting(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE); indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT); indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT); indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.AUTO_THROTTLE); diff --git a/src/main/java/org/elasticsearch/index/store/DirectoryService.java b/src/main/java/org/elasticsearch/index/store/DirectoryService.java index 2fa4e8eb455..81d8910ed4c 100644 --- a/src/main/java/org/elasticsearch/index/store/DirectoryService.java +++ b/src/main/java/org/elasticsearch/index/store/DirectoryService.java @@ -39,6 +39,8 @@ public abstract class DirectoryService extends AbstractIndexShardComponent { public abstract Directory[] build() throws IOException; + public abstract long throttleTimeInNanos(); + /** * Creates a new Directory from the given distributor. * The default implementation returns a new {@link org.elasticsearch.index.store.DistributorDirectory} @@ -56,4 +58,4 @@ public abstract class DirectoryService extends AbstractIndexShardComponent { } return new DistributorDirectory(distributor); } -} +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/store/IndexStore.java b/src/main/java/org/elasticsearch/index/store/IndexStore.java index d22c993ae18..d979075694b 100644 --- a/src/main/java/org/elasticsearch/index/store/IndexStore.java +++ b/src/main/java/org/elasticsearch/index/store/IndexStore.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.store; +import org.apache.lucene.store.StoreRateLimiting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.ShardId; @@ -32,6 +33,12 @@ import java.nio.file.Path; */ public interface IndexStore extends Closeable { + /** + * Returns the rate limiting, either of the index is explicitly configured, or + * the node level one (defaults to the node level one). + */ + StoreRateLimiting rateLimiting(); + /** * The shard store class that should be used for each shard. */ diff --git a/src/main/java/org/elasticsearch/index/store/Store.java b/src/main/java/org/elasticsearch/index/store/Store.java index c5610d6f257..87df70c81f9 100644 --- a/src/main/java/org/elasticsearch/index/store/Store.java +++ b/src/main/java/org/elasticsearch/index/store/Store.java @@ -283,7 +283,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref public StoreStats stats() throws IOException { ensureOpen(); - return new StoreStats(Directories.estimateSize(directory)); + return new StoreStats(Directories.estimateSize(directory), directoryService.throttleTimeInNanos()); } public void renameFile(String from, String to) throws IOException { diff --git a/src/main/java/org/elasticsearch/index/store/StoreStats.java b/src/main/java/org/elasticsearch/index/store/StoreStats.java index 2db10ff0f80..3b7f52a6895 100644 --- a/src/main/java/org/elasticsearch/index/store/StoreStats.java +++ b/src/main/java/org/elasticsearch/index/store/StoreStats.java @@ -19,9 +19,6 @@ package org.elasticsearch.index.store; -import java.io.IOException; - -import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -31,18 +28,23 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; +import java.io.IOException; + /** */ public class StoreStats implements Streamable, ToXContent { private long sizeInBytes; + private long throttleTimeInNanos; + public StoreStats() { } - public StoreStats(long sizeInBytes) { + public StoreStats(long sizeInBytes, long throttleTimeInNanos) { this.sizeInBytes = sizeInBytes; + this.throttleTimeInNanos = throttleTimeInNanos; } public void add(StoreStats stats) { @@ -50,6 +52,7 @@ public class StoreStats implements Streamable, ToXContent { return; } sizeInBytes += stats.sizeInBytes; + throttleTimeInNanos += stats.throttleTimeInNanos; } @@ -69,6 +72,14 @@ public class StoreStats implements Streamable, ToXContent { return size(); } + public TimeValue throttleTime() { + return TimeValue.timeValueNanos(throttleTimeInNanos); + } + + public TimeValue getThrottleTime() { + return throttleTime(); + } + public static StoreStats readStoreStats(StreamInput in) throws IOException { StoreStats store = new StoreStats(); store.readFrom(in); @@ -78,17 +89,20 @@ public class StoreStats implements Streamable, ToXContent { @Override public void readFrom(StreamInput in) throws IOException { sizeInBytes = in.readVLong(); + throttleTimeInNanos = in.readVLong(); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(sizeInBytes); + out.writeVLong(throttleTimeInNanos); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.STORE); builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, sizeInBytes); + builder.timeValueField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, throttleTime()); builder.endObject(); return builder; } @@ -97,5 +111,8 @@ public class StoreStats implements Streamable, ToXContent { static final XContentBuilderString STORE = new XContentBuilderString("store"); static final XContentBuilderString SIZE = new XContentBuilderString("size"); static final XContentBuilderString SIZE_IN_BYTES = new XContentBuilderString("size_in_bytes"); + + static final XContentBuilderString THROTTLE_TIME = new XContentBuilderString("throttle_time"); + static final XContentBuilderString THROTTLE_TIME_IN_MILLIS = new XContentBuilderString("throttle_time_in_millis"); } } diff --git a/src/main/java/org/elasticsearch/index/store/distributor/AbstractDistributor.java b/src/main/java/org/elasticsearch/index/store/distributor/AbstractDistributor.java index 5baa0da1f36..18dac8b6d92 100644 --- a/src/main/java/org/elasticsearch/index/store/distributor/AbstractDistributor.java +++ b/src/main/java/org/elasticsearch/index/store/distributor/AbstractDistributor.java @@ -19,18 +19,18 @@ package org.elasticsearch.index.store.distributor; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.StoreUtils; +import org.elasticsearch.index.store.DirectoryService; +import org.elasticsearch.index.store.DirectoryUtils; + import java.io.IOException; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.elasticsearch.index.store.DirectoryService; -import org.elasticsearch.index.store.DirectoryUtils; -import org.elasticsearch.index.store.StoreUtils; - public abstract class AbstractDistributor implements Distributor { protected final Directory[] delegates; diff --git a/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java b/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java index 5be06b37a3b..61786c8e5d6 100644 --- a/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java +++ b/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java @@ -36,15 +36,27 @@ import org.elasticsearch.index.store.StoreException; /** */ -public abstract class FsDirectoryService extends DirectoryService { +public abstract class FsDirectoryService extends DirectoryService implements StoreRateLimiting.Listener, StoreRateLimiting.Provider { protected final IndexStore indexStore; + private final CounterMetric rateLimitingTimeInNanos = new CounterMetric(); + public FsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { super(shardId, indexSettings); this.indexStore = indexStore; } + @Override + public long throttleTimeInNanos() { + return rateLimitingTimeInNanos.count(); + } + + @Override + public StoreRateLimiting rateLimiting() { + return indexStore.rateLimiting(); + } + protected final LockFactory buildLockFactory() throws IOException { String fsLock = componentSettings.get("lock", componentSettings.get("fs_lock", "native")); LockFactory lockFactory; @@ -65,10 +77,16 @@ public abstract class FsDirectoryService extends DirectoryService { Directory[] dirs = new Directory[locations.length]; for (int i = 0; i < dirs.length; i++) { Files.createDirectories(locations[i]); - dirs[i] = newFSDirectory(locations[i], buildLockFactory()); + Directory wrapped = newFSDirectory(locations[i], buildLockFactory()); + dirs[i] = new RateLimitedFSDirectory(wrapped, this, this) ; } return dirs; } protected abstract Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException; + + @Override + public void onPause(long nanos) { + rateLimitingTimeInNanos.inc(nanos); + } } diff --git a/src/main/java/org/elasticsearch/index/store/support/AbstractIndexStore.java b/src/main/java/org/elasticsearch/index/store/support/AbstractIndexStore.java index 357e9ce771a..bb73c669047 100644 --- a/src/main/java/org/elasticsearch/index/store/support/AbstractIndexStore.java +++ b/src/main/java/org/elasticsearch/index/store/support/AbstractIndexStore.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.store.support; +import org.apache.lucene.store.StoreRateLimiting; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.common.io.FileSystemUtils; @@ -42,8 +43,37 @@ import java.nio.file.Path; */ public abstract class AbstractIndexStore extends AbstractIndexComponent implements IndexStore { + public static final String INDEX_STORE_THROTTLE_TYPE = "index.store.throttle.type"; + public static final String INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC = "index.store.throttle.max_bytes_per_sec"; + public static final String INDEX_FOLDER_NAME = "index"; public static final String TRANSLOG_FOLDER_NAME = "translog"; + + class ApplySettings implements IndexSettingsService.Listener { + @Override + public void onRefreshSettings(Settings settings) { + String rateLimitingType = settings.get(INDEX_STORE_THROTTLE_TYPE, AbstractIndexStore.this.rateLimitingType); + if (!rateLimitingType.equals(AbstractIndexStore.this.rateLimitingType)) { + logger.info("updating index.store.throttle.type from [{}] to [{}]", AbstractIndexStore.this.rateLimitingType, rateLimitingType); + if (rateLimitingType.equalsIgnoreCase("node")) { + AbstractIndexStore.this.rateLimitingType = rateLimitingType; + AbstractIndexStore.this.nodeRateLimiting = true; + } else { + StoreRateLimiting.Type.fromString(rateLimitingType); + AbstractIndexStore.this.rateLimitingType = rateLimitingType; + AbstractIndexStore.this.nodeRateLimiting = false; + AbstractIndexStore.this.rateLimiting.setType(rateLimitingType); + } + } + + ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, AbstractIndexStore.this.rateLimitingThrottle); + if (!rateLimitingThrottle.equals(AbstractIndexStore.this.rateLimitingThrottle)) { + logger.info("updating index.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", AbstractIndexStore.this.rateLimitingThrottle, rateLimitingThrottle, AbstractIndexStore.this.rateLimitingType); + AbstractIndexStore.this.rateLimitingThrottle = rateLimitingThrottle; + AbstractIndexStore.this.rateLimiting.setMaxRate(rateLimitingThrottle); + } + } + } private final NodeEnvironment nodeEnv; private final Path[] locations; @@ -52,11 +82,32 @@ public abstract class AbstractIndexStore extends AbstractIndexComponent implemen protected final IndicesStore indicesStore; + private volatile String rateLimitingType; + private volatile ByteSizeValue rateLimitingThrottle; + private volatile boolean nodeRateLimiting; + + private final StoreRateLimiting rateLimiting = new StoreRateLimiting(); + + private final ApplySettings applySettings = new ApplySettings(); + protected AbstractIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) { super(index, indexSettings); this.indexService = indexService; this.indicesStore = indicesStore; + this.rateLimitingType = indexSettings.get(INDEX_STORE_THROTTLE_TYPE, "none"); + if (rateLimitingType.equalsIgnoreCase("node")) { + nodeRateLimiting = true; + } else { + nodeRateLimiting = false; + rateLimiting.setType(rateLimitingType); + } + this.rateLimitingThrottle = indexSettings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(0)); + rateLimiting.setMaxRate(rateLimitingThrottle); + + logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle); + + indexService.settingsService().addListener(applySettings); this.nodeEnv = nodeEnv; if (nodeEnv.hasNodeFile()) { this.locations = nodeEnv.indexPaths(index); @@ -67,8 +118,15 @@ public abstract class AbstractIndexStore extends AbstractIndexComponent implemen @Override public void close() throws ElasticsearchException { + indexService.settingsService().removeListener(applySettings); } + @Override + public StoreRateLimiting rateLimiting() { + return nodeRateLimiting ? indicesStore.rateLimiting() : this.rateLimiting; + } + + @Override public boolean canDeleteUnallocated(ShardId shardId, @IndexSettings Settings indexSettings) { if (locations == null) { diff --git a/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index cb0b03da62e..558ccdfebbe 100644 --- a/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -19,6 +19,7 @@ package org.elasticsearch.indices.store; +import org.apache.lucene.store.StoreRateLimiting; import org.elasticsearch.Version; import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -59,10 +60,34 @@ import java.util.concurrent.atomic.AtomicInteger; */ public class IndicesStore extends AbstractComponent implements ClusterStateListener, Closeable { + public static final String INDICES_STORE_THROTTLE_TYPE = "indices.store.throttle.type"; + public static final String INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC = "indices.store.throttle.max_bytes_per_sec"; + public static final String ACTION_SHARD_EXISTS = "internal:index/shard/exists"; private static final EnumSet ACTIVE_STATES = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED); + class ApplySettings implements NodeSettingsService.Listener { + @Override + public void onRefreshSettings(Settings settings) { + String rateLimitingType = settings.get(INDICES_STORE_THROTTLE_TYPE, IndicesStore.this.rateLimitingType); + // try and parse the type + StoreRateLimiting.Type.fromString(rateLimitingType); + if (!rateLimitingType.equals(IndicesStore.this.rateLimitingType)) { + logger.info("updating indices.store.throttle.type from [{}] to [{}]", IndicesStore.this.rateLimitingType, rateLimitingType); + IndicesStore.this.rateLimitingType = rateLimitingType; + IndicesStore.this.rateLimiting.setType(rateLimitingType); + } + + ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndicesStore.this.rateLimitingThrottle); + if (!rateLimitingThrottle.equals(IndicesStore.this.rateLimitingThrottle)) { + logger.info("updating indices.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndicesStore.this.rateLimitingThrottle, rateLimitingThrottle, IndicesStore.this.rateLimitingType); + IndicesStore.this.rateLimitingThrottle = rateLimitingThrottle; + IndicesStore.this.rateLimiting.setMaxRate(rateLimitingThrottle); + } + } + } + private final NodeEnvironment nodeEnv; private final NodeSettingsService nodeSettingsService; @@ -72,6 +97,12 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe private final ClusterService clusterService; private final TransportService transportService; + private volatile String rateLimitingType; + private volatile ByteSizeValue rateLimitingThrottle; + private final StoreRateLimiting rateLimiting = new StoreRateLimiting(); + + private final ApplySettings applySettings = new ApplySettings(); + @Inject public IndicesStore(Settings settings, NodeEnvironment nodeEnv, NodeSettingsService nodeSettingsService, IndicesService indicesService, ClusterService clusterService, TransportService transportService) { @@ -83,6 +114,15 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe this.transportService = transportService; transportService.registerHandler(ACTION_SHARD_EXISTS, new ShardActiveRequestHandler()); + // we don't limit by default (we default to CMS's auto throttle instead): + this.rateLimitingType = componentSettings.get("throttle.type", StoreRateLimiting.Type.NONE.name()); + rateLimiting.setType(rateLimitingType); + this.rateLimitingThrottle = componentSettings.getAsBytesSize("throttle.max_bytes_per_sec", new ByteSizeValue(10240, ByteSizeUnit.MB)); + rateLimiting.setMaxRate(rateLimitingThrottle); + + logger.debug("using indices.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle); + + nodeSettingsService.addListener(applySettings); clusterService.addLast(this); } @@ -95,8 +135,13 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe this.transportService = null; } + public StoreRateLimiting rateLimiting() { + return this.rateLimiting; + } + @Override public void close() { + nodeSettingsService.removeListener(applySettings); clusterService.remove(this); } diff --git a/src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java b/src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java index 8aba2d37aa1..8485a5aae7d 100644 --- a/src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java +++ b/src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java @@ -179,6 +179,11 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase { public Directory[] build() throws IOException { return new Directory[]{ directory }; } + + @Override + public long throttleTimeInNanos() { + return 0; + } }; return new Store(shardId, EMPTY_SETTINGS, directoryService, new LeastUsedDistributor(directoryService), new DummyShardLock(shardId)); } diff --git a/src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java b/src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java index 43306d1b315..6caac7c3186 100644 --- a/src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java +++ b/src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java @@ -178,6 +178,11 @@ public class MergePolicySettingsTest extends ElasticsearchTestCase { public Directory[] build() throws IOException { return new Directory[] { new RAMDirectory() } ; } + + @Override + public long throttleTimeInNanos() { + return 0; + } }; return new Store(shardId, settings, directoryService, new LeastUsedDistributor(directoryService), new DummyShardLock(shardId)); } diff --git a/src/test/java/org/elasticsearch/index/store/StoreTest.java b/src/test/java/org/elasticsearch/index/store/StoreTest.java index 35dc4968a12..2dd834a6047 100644 --- a/src/test/java/org/elasticsearch/index/store/StoreTest.java +++ b/src/test/java/org/elasticsearch/index/store/StoreTest.java @@ -710,6 +710,11 @@ public class StoreTest extends ElasticsearchLuceneTestCase { public Directory[] build() throws IOException { return dirs; } + + @Override + public long throttleTimeInNanos() { + return random.nextInt(1000); + } } public static void assertConsistent(Store store, Store.MetadataSnapshot metadata) throws IOException { diff --git a/src/test/java/org/elasticsearch/index/store/distributor/DistributorTests.java b/src/test/java/org/elasticsearch/index/store/distributor/DistributorTests.java index 5d319692681..119c7636843 100644 --- a/src/test/java/org/elasticsearch/index/store/distributor/DistributorTests.java +++ b/src/test/java/org/elasticsearch/index/store/distributor/DistributorTests.java @@ -150,6 +150,11 @@ public class DistributorTests extends ElasticsearchTestCase { public Directory[] build() throws IOException { return directories; } + + @Override + public long throttleTimeInNanos() { + return 0; + } } public static class FakeFsDirectory extends FSDirectory { diff --git a/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java b/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java index d03577fb07d..e9e5af52ebd 100644 --- a/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java +++ b/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java @@ -125,6 +125,140 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest { } + // #6626: make sure we can update throttle settings and the changes take effect + @Test + @Slow + public void testUpdateThrottleSettings() { + + // No throttling at first, only 1 non-replicated shard, force lots of merging: + assertAcked(prepareCreate("test") + .setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "none") + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2") + .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "1") + .put(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT, "2") + )); + ensureGreen(); + long termUpto = 0; + for(int i=0;i<100;i++) { + // Provoke slowish merging by making many unique terms: + StringBuilder sb = new StringBuilder(); + for(int j=0;j<100;j++) { + sb.append(' '); + sb.append(termUpto++); + } + client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); + if (i % 2 == 0) { + refresh(); + } + } + + // No merge IO throttling should have happened: + NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); + for(NodeStats stats : nodesStats.getNodes()) { + assertThat(stats.getIndices().getStore().getThrottleTime().getMillis(), equalTo(0l)); + } + + logger.info("test: set low merge throttling"); + + // Now updates settings to turn on merge throttling lowish rate + client() + .admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, "1mb")) + .get(); + + // Make sure setting says it is in fact changed: + GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get(); + assertThat(getSettingsResponse.getSetting("test", AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE), equalTo("merge")); + + // Also make sure we see throttling kicking in: + boolean done = false; + while (done == false) { + // Provoke slowish merging by making many unique terms: + for(int i=0;i<5;i++) { + StringBuilder sb = new StringBuilder(); + for(int j=0;j<100;j++) { + sb.append(' '); + sb.append(termUpto++); + sb.append(" some random text that keeps repeating over and over again hambone"); + } + client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); + } + refresh(); + nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); + for(NodeStats stats : nodesStats.getNodes()) { + long throttleMillis = stats.getIndices().getStore().getThrottleTime().getMillis(); + if (throttleMillis > 0) { + done = true; + break; + } + } + } + + logger.info("test: disable merge throttling"); + + // Now updates settings to disable merge throttling + client() + .admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "none")) + .get(); + + // Optimize does a waitForMerges, which we must do to make sure all in-flight (throttled) merges finish: + logger.info("test: optimize"); + client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); + logger.info("test: optimize done"); + + // Record current throttling so far + long sumThrottleTime = 0; + nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); + for(NodeStats stats : nodesStats.getNodes()) { + sumThrottleTime += stats.getIndices().getStore().getThrottleTime().getMillis(); + } + + // Make sure no further throttling happens: + for(int i=0;i<100;i++) { + // Provoke slowish merging by making many unique terms: + StringBuilder sb = new StringBuilder(); + for(int j=0;j<100;j++) { + sb.append(' '); + sb.append(termUpto++); + } + client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); + if (i % 2 == 0) { + refresh(); + } + } + logger.info("test: done indexing after disabling throttling"); + + long newSumThrottleTime = 0; + nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); + for(NodeStats stats : nodesStats.getNodes()) { + newSumThrottleTime += stats.getIndices().getStore().getThrottleTime().getMillis(); + } + + // No additional merge IO throttling should have happened: + assertEquals(sumThrottleTime, newSumThrottleTime); + + // Optimize & flush and wait; else we sometimes get a "Delete Index failed - not acked" + // when ElasticsearchIntegrationTest.after tries to remove indices created by the test: + + // Wait for merges to finish + client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); + flush(); + + logger.info("test: test done"); + } + private static class MockAppender extends AppenderSkeleton { public boolean sawIndexWriterMessage; public boolean sawFlushDeletes; diff --git a/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java b/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java index 0d7621ac783..0adebd64bb3 100644 --- a/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java +++ b/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java @@ -301,6 +301,90 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest { assertThat(client().admin().indices().prepareStats("idx").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0l)); } + + @Test + public void nonThrottleStats() throws Exception { + assertAcked(prepareCreate("test") + .setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2") + .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "1") + .put(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT, "10000") + )); + ensureGreen(); + long termUpto = 0; + IndicesStatsResponse stats; + // Provoke slowish merging by making many unique terms: + for(int i=0; i<100; i++) { + StringBuilder sb = new StringBuilder(); + for(int j=0; j<100; j++) { + sb.append(' '); + sb.append(termUpto++); + sb.append(" some random text that keeps repeating over and over again hambone"); + } + client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); + } + refresh(); + stats = client().admin().indices().prepareStats().execute().actionGet(); + //nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); + + stats = client().admin().indices().prepareStats().execute().actionGet(); + assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0l)); + } + + @Test + public void throttleStats() throws Exception { + assertAcked(prepareCreate("test") + .setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge") + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER, "2") + .put(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT, "1") + .put(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT, "1") + .put("index.merge.policy.type", "tiered") + + )); + ensureGreen(); + long termUpto = 0; + IndicesStatsResponse stats; + // make sure we see throttling kicking in: + boolean done = false; + long start = System.currentTimeMillis(); + while (!done) { + for(int i=0; i<100; i++) { + // Provoke slowish merging by making many unique terms: + StringBuilder sb = new StringBuilder(); + for(int j=0; j<100; j++) { + sb.append(' '); + sb.append(termUpto++); + } + client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get(); + if (i % 2 == 0) { + refresh(); + } + } + refresh(); + stats = client().admin().indices().prepareStats().execute().actionGet(); + //nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get(); + done = stats.getPrimaries().getIndexing().getTotal().getThrottleTimeInMillis() > 0; + if (System.currentTimeMillis() - start > 300*1000) { //Wait 5 minutes for throttling to kick in + fail("index throttling didn't kick in after 5 minutes of intense merging"); + } + } + + // Optimize & flush and wait; else we sometimes get a "Delete Index failed - not acked" + // when ElasticsearchIntegrationTest.after tries to remove indices created by the test: + logger.info("test: now optimize"); + client().admin().indices().prepareOptimize("test").setWaitForMerge(true).get(); + flush(); + logger.info("test: test done"); + } + @Test public void simpleStats() throws Exception { createIndex("test1", "test2"); @@ -440,9 +524,6 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest { assertThat(stats.getTotal().getMerge(), notNullValue()); assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0l)); - assertThat(stats.getTotal().getMerge().getTotalStoppedTime(), notNullValue()); - assertThat(stats.getTotal().getMerge().getTotalThrottledTime(), notNullValue()); - assertThat(stats.getTotal().getMerge().getTotalBytesPerSecAutoThrottle(), greaterThan(0l)); } @Test diff --git a/src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java b/src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java index adab2a2df18..bfe2876c8dc 100644 --- a/src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java +++ b/src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java @@ -54,48 +54,63 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest { String storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); Path[] dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } + assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); createIndexWithStoreType("test", IndexStoreModule.Type.NIOFS, "random"); storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(random[niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(random[rate_limited(niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } + assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); createIndexWithStoreType("test", IndexStoreModule.Type.MMAPFS, "least_used"); storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[mmapfs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(mmapfs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), mmapfs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(mmapfs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } + assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); createIndexWithStoreType("test", IndexStoreModule.Type.SIMPLEFS, "least_used"); storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[simplefs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(simplefs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), simplefs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(simplefs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } + assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); createIndexWithStoreType("test", IndexStoreModule.Type.DEFAULT, "least_used"); storeString = getStoreDirectory("test", 0).toString(); logger.info(storeString); dataPaths = dataPaths(); - assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[default(mmapfs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(default(mmapfs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); assertThat(storeString.toLowerCase(Locale.ROOT), containsString("),niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); if (dataPaths.length > 1) { - assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), default(mmapfs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(default(mmapfs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); } + assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])")); + + createIndexWithoutRateLimitingStoreType("test", IndexStoreModule.Type.NIOFS, "least_used"); + storeString = getStoreDirectory("test", 0).toString(); + logger.info(storeString); + dataPaths = dataPaths(); + assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[niofs(" + dataPaths[0].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + if (dataPaths.length > 1) { + assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), niofs(" + dataPaths[1].toAbsolutePath().toString().toLowerCase(Locale.ROOT))); + } + assertThat(storeString, endsWith(")])")); } private void createIndexWithStoreType(String index, IndexStoreModule.Type storeType, String distributor) { @@ -106,11 +121,28 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest { .put("index.store.type", storeType.name()) .put("index.number_of_replicas", 0) .put("index.number_of_shards", 1) + .put("index.store.throttle.type", "merge") + .put("index.store.throttle.max_bytes_per_sec", "20mb") ) .execute().actionGet(); assertThat(client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet().isTimedOut(), equalTo(false)); } + private void createIndexWithoutRateLimitingStoreType(String index, IndexStoreModule.Type storeType, String distributor) { + cluster().wipeIndices(index); + client().admin().indices().prepareCreate(index) + .setSettings(settingsBuilder() + .put("index.store.distributor", distributor) + .put("index.store.type", storeType) + .put("index.store.throttle.type", "none") + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 1) + ) + .execute().actionGet(); + assertThat(client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet().isTimedOut(), equalTo(false)); + } + + private Path[] dataPaths() { Set nodes = internalCluster().nodesInclude("test"); assertThat(nodes.isEmpty(), equalTo(false)); diff --git a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java index 7c3a00ed0e2..fad835d5e24 100644 --- a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java @@ -633,6 +633,11 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests asyncIndexThreads[i].join(); } + logger.info("--> update index settings to back to normal"); + assertAcked(client().admin().indices().prepareUpdateSettings("test-*").setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "node") + )); + // Make sure that snapshot finished - doesn't matter if it failed or succeeded try { CreateSnapshotResponse snapshotResponse = snapshotResponseFuture.get(); @@ -674,6 +679,11 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests for (int i = 0; i < between(10, 500); i++) { index(name, "doc", Integer.toString(i), "foo", "bar" + i); } + + assertAcked(client().admin().indices().prepareUpdateSettings(name).setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "all") + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, between(100, 50000)) + )); } public static abstract class TestCustomMetaData implements MetaData.Custom { diff --git a/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java index 81d14aea5da..08bc0d3c1fc 100644 --- a/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java @@ -1361,6 +1361,12 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests { refresh(); assertThat(client.prepareCount("test-idx").get().getCount(), equalTo(100L)); + // Update settings to make sure that relocation is slow so we can start snapshot before relocation is finished + assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "all") + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, 100) + )); + logger.info("--> start relocations"); allowNodes("test-idx", internalCluster().numDataNodes()); @@ -1371,6 +1377,11 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests { logger.info("--> snapshot"); client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get(); + // Update settings to back to normal + assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(ImmutableSettings.builder() + .put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "node") + )); + logger.info("--> wait for snapshot to complete"); SnapshotInfo snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600)); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java index 6f4f0e94a4b..14c1aa06302 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java @@ -27,6 +27,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Predicate; import com.google.common.collect.Lists; +import org.apache.lucene.store.StoreRateLimiting; import org.apache.lucene.util.AbstractRandomizedTest; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; @@ -429,11 +430,23 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase setRandomTranslogSettings(random, builder); setRandomNormsLoading(random, builder); setRandomScriptingSettings(random, builder); + if (random.nextBoolean()) { + if (random.nextInt(10) == 0) { // do something crazy slow here + builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB)); + } else { + builder.put(IndicesStore.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(RandomInts.randomIntBetween(random, 10, 200), ByteSizeUnit.MB)); + } + } + if (random.nextBoolean()) { + builder.put(IndicesStore.INDICES_STORE_THROTTLE_TYPE, RandomPicks.randomFrom(random, StoreRateLimiting.Type.values())); + } if (random.nextBoolean()) { builder.put(StoreModule.DISTIBUTOR_KEY, random.nextBoolean() ? StoreModule.LEAST_USED_DISTRIBUTOR : StoreModule.RANDOM_WEIGHT_DISTRIBUTOR); } - + if (random.nextBoolean()) { + builder.put(ConcurrentMergeSchedulerProvider.AUTO_THROTTLE, false); + } if (random.nextBoolean()) { if (random.nextInt(10) == 0) { // do something crazy slow here diff --git a/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java index fccd693866b..bb2a262c278 100644 --- a/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java +++ b/src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.StoreRateLimiting; import org.apache.lucene.util.AbstractRandomizedTest; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; @@ -150,6 +151,21 @@ public class MockFSDirectoryService extends FsDirectoryService { } } + @Override + public void onPause(long nanos) { + delegateService.onPause(nanos); + } + + @Override + public StoreRateLimiting rateLimiting() { + return delegateService.rateLimiting(); + } + + @Override + public long throttleTimeInNanos() { + return delegateService.throttleTimeInNanos(); + } + @Override public Directory newFromDistributor(Distributor distributor) throws IOException { return helper.wrap(super.newFromDistributor(distributor)); From def2d34f803c95623566717d2dc3fe4ef3e1483c Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 14 Jan 2015 10:13:10 -0500 Subject: [PATCH 6/6] don't mention fixed throttling in the docs --- .../cluster/update-settings.asciidoc | 9 ------ docs/reference/index-modules/merge.asciidoc | 6 ---- docs/reference/index-modules/store.asciidoc | 31 ------------------- docs/reference/modules/indices.asciidoc | 13 -------- 4 files changed, 59 deletions(-) diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc index 9962cf2bb23..aded0fda24b 100644 --- a/docs/reference/cluster/update-settings.asciidoc +++ b/docs/reference/cluster/update-settings.asciidoc @@ -183,15 +183,6 @@ due to forced awareness or allocation filtering. `indices.recovery.max_bytes_per_sec`:: See <> -[float] -==== Store level throttling - -`indices.store.throttle.type`:: - See <> - -`indices.store.throttle.max_bytes_per_sec`:: - See <> - [float] [[logger]] === Logger diff --git a/docs/reference/index-modules/merge.asciidoc b/docs/reference/index-modules/merge.asciidoc index 6fd4d65793a..4d276f00037 100644 --- a/docs/reference/index-modules/merge.asciidoc +++ b/docs/reference/index-modules/merge.asciidoc @@ -7,12 +7,6 @@ where the index data is stored, and are immutable up to delete markers. Segments are, periodically, merged into larger segments to keep the index size at bay and expunge deletes. -The more segments one has in the Lucene index means slower searches and -more memory used. Segment merging is used to reduce the number of segments, -however merges can be expensive to perform, especially on low IO environments. -Merges can be throttled using <>. - - [float] [[policy]] === Policy diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc index f7fdb86ad9a..4a0a9de86a9 100644 --- a/docs/reference/index-modules/store.asciidoc +++ b/docs/reference/index-modules/store.asciidoc @@ -19,37 +19,6 @@ to the fact that there is no need for extra large JVM heaps (with their own consequences) for storing the index in memory. -[float] -[[store-throttling]] -=== Store Level Throttling - -The way Lucene, the IR library elasticsearch uses under the covers, -works is by creating immutable segments (up to deletes) and constantly -merging them (the merge policy settings allow to control how those -merges happen). The merge process happens in an asynchronous manner -without affecting the indexing / search speed. The problem though, -especially on systems with low IO, is that the merge process can be -expensive and affect search / index operation simply by the fact that -the box is now taxed with more IO happening. - -The store module allows to have throttling configured for merges (or -all) either on the node level, or on the index level. The node level -throttling will make sure that out of all the shards allocated on that -node, the merge process won't pass the specific setting bytes per -second. It can be set by setting `indices.store.throttle.type` to -`merge`, and setting `indices.store.throttle.max_bytes_per_sec` to -something like `5mb`. The node level settings can be changed dynamically -using the cluster update settings API. The default is disabled (set to `none`), -in favor of . - -If specific index level configuration is needed, regardless of the node -level settings, it can be set as well using the -`index.store.throttle.type`, and -`index.store.throttle.max_bytes_per_sec`. The default value for the type -is `node`, meaning it will throttle based on the node level settings and -participate in the global throttling happening. Both settings can be set -using the index update settings API dynamically. - [float] [[file-system]] === File system storage types diff --git a/docs/reference/modules/indices.asciidoc b/docs/reference/modules/indices.asciidoc index 16c22a9dba7..a27b26b846b 100644 --- a/docs/reference/modules/indices.asciidoc +++ b/docs/reference/modules/indices.asciidoc @@ -61,16 +61,3 @@ The following settings can be set to manage the recovery policy: `indices.recovery.max_bytes_per_sec`:: defaults to `20mb`. -[float] -[[throttling]] -=== Store level throttling - -The following settings can be set to control the store throttling: - -[horizontal] -`indices.store.throttle.type`:: - could be `merge` (default), `none` or `all`. See <>. - -`indices.store.throttle.max_bytes_per_sec`:: - defaults to `20mb`. -