Add disk threshold settings validation
This commit adds cross-settings validation for the low/high/flood stage disk watermark settings. This validation was enabled by the introduction of multiple settings validation. Relates #25600
This commit is contained in:
Jason Tedor
GitHub
parent
794257c421
commit
bc22c1c286
|
|
@ -21,6 +21,7 @@ package org.elasticsearch.gradle.test
|
|||
import org.apache.tools.ant.DefaultLogger
|
||||
import org.apache.tools.ant.taskdefs.condition.Os
|
||||
import org.elasticsearch.gradle.LoggedExec
|
||||
import org.elasticsearch.gradle.Version
|
||||
import org.elasticsearch.gradle.VersionProperties
|
||||
import org.elasticsearch.gradle.plugin.PluginBuildPlugin
|
||||
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
|
||||
|
|
@ -312,6 +313,9 @@ class ClusterFormationTasks {
|
|||
// Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space
|
||||
esConfig['cluster.routing.allocation.disk.watermark.low'] = '1b'
|
||||
esConfig['cluster.routing.allocation.disk.watermark.high'] = '1b'
|
||||
if (Version.fromString(node.nodeVersion).major >= 6) {
|
||||
esConfig['cluster.routing.allocation.disk.watermark.floodstage'] = '1b'
|
||||
}
|
||||
esConfig.putAll(node.config.settings)
|
||||
|
||||
Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,11 @@ import org.elasticsearch.common.unit.ByteSizeValue;
|
|||
import org.elasticsearch.common.unit.RatioValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Iterator;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A container to keep settings for disk thresholds up to date with cluster setting changes.
|
||||
*/
|
||||
|
|
@ -37,14 +42,17 @@ public class DiskThresholdSettings {
|
|||
public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING =
|
||||
new Setting<>("cluster.routing.allocation.disk.watermark.low", "85%",
|
||||
(s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.low"),
|
||||
new LowDiskWatermarkValidator(),
|
||||
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
||||
public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING =
|
||||
new Setting<>("cluster.routing.allocation.disk.watermark.high", "90%",
|
||||
(s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.high"),
|
||||
new HighDiskWatermarkValidator(),
|
||||
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
||||
public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING =
|
||||
new Setting<>("cluster.routing.allocation.disk.watermark.floodstage", "95%",
|
||||
(s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.floodstage"),
|
||||
new FloodStageValidator(),
|
||||
Setting.Property.Dynamic, Setting.Property.NodeScope);
|
||||
public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING =
|
||||
Setting.boolSetting("cluster.routing.allocation.disk.include_relocations", true,
|
||||
|
|
@ -84,6 +92,116 @@ public class DiskThresholdSettings {
|
|||
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING, this::setEnabled);
|
||||
}
|
||||
|
||||
static final class LowDiskWatermarkValidator implements Setting.Validator<String> {
|
||||
|
||||
@Override
|
||||
public void validate(String value, Map<Setting<String>, String> settings) {
|
||||
final String highWatermarkRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING);
|
||||
final String floodStageRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING);
|
||||
doValidate(value, highWatermarkRaw, floodStageRaw);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Setting<String>> settings() {
|
||||
return Arrays.asList(
|
||||
CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING,
|
||||
CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING)
|
||||
.iterator();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static final class HighDiskWatermarkValidator implements Setting.Validator<String> {
|
||||
|
||||
@Override
|
||||
public void validate(String value, Map<Setting<String>, String> settings) {
|
||||
final String lowWatermarkRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING);
|
||||
final String floodStageRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING);
|
||||
doValidate(lowWatermarkRaw, value, floodStageRaw);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Setting<String>> settings() {
|
||||
return Arrays.asList(
|
||||
CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING,
|
||||
CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING)
|
||||
.iterator();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static final class FloodStageValidator implements Setting.Validator<String> {
|
||||
|
||||
@Override
|
||||
public void validate(String value, Map<Setting<String>, String> settings) {
|
||||
final String lowWatermarkRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING);
|
||||
final String highWatermarkRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING);
|
||||
doValidate(lowWatermarkRaw, highWatermarkRaw, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Setting<String>> settings() {
|
||||
return Arrays.asList(
|
||||
CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING,
|
||||
CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING)
|
||||
.iterator();
|
||||
}
|
||||
}
|
||||
|
||||
private static void doValidate(String low, String high, String flood) {
|
||||
try {
|
||||
doValidateAsPercentage(low, high, flood);
|
||||
return; // early return so that we do not try to parse as bytes
|
||||
} catch (final ElasticsearchParseException e) {
|
||||
// swallow as we are now going to try to parse as bytes
|
||||
}
|
||||
try {
|
||||
doValidateAsBytes(low, high, flood);
|
||||
} catch (final ElasticsearchParseException e) {
|
||||
final String message = String.format(
|
||||
Locale.ROOT,
|
||||
"unable to consistently parse [%s=%s], [%s=%s], and [%s=%s] as percentage or bytes",
|
||||
CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(),
|
||||
low,
|
||||
CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
|
||||
high,
|
||||
CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(),
|
||||
flood);
|
||||
throw new IllegalArgumentException(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
private static void doValidateAsPercentage(final String low, final String high, final String flood) {
|
||||
final double lowWatermarkThreshold = thresholdPercentageFromWatermark(low, false);
|
||||
final double highWatermarkThreshold = thresholdPercentageFromWatermark(high, false);
|
||||
final double floodThreshold = thresholdPercentageFromWatermark(flood, false);
|
||||
if (lowWatermarkThreshold > highWatermarkThreshold) {
|
||||
throw new IllegalArgumentException(
|
||||
"low disk watermark [" + low + "] more than high disk watermark [" + high + "]");
|
||||
}
|
||||
if (highWatermarkThreshold > floodThreshold) {
|
||||
throw new IllegalArgumentException(
|
||||
"high disk watermark [" + high + "] more than flood stage disk watermark [" + flood + "]");
|
||||
}
|
||||
}
|
||||
|
||||
private static void doValidateAsBytes(final String low, final String high, final String flood) {
|
||||
final ByteSizeValue lowWatermarkBytes =
|
||||
thresholdBytesFromWatermark(low, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), false);
|
||||
final ByteSizeValue highWatermarkBytes =
|
||||
thresholdBytesFromWatermark(high, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), false);
|
||||
final ByteSizeValue floodStageBytes =
|
||||
thresholdBytesFromWatermark(flood, CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), false);
|
||||
if (lowWatermarkBytes.getBytes() < highWatermarkBytes.getBytes()) {
|
||||
throw new IllegalArgumentException(
|
||||
"low disk watermark [" + low + "] less than high disk watermark [" + high + "]");
|
||||
}
|
||||
if (highWatermarkBytes.getBytes() < floodStageBytes.getBytes()) {
|
||||
throw new IllegalArgumentException(
|
||||
"high disk watermark [" + high + "] less than flood stage disk watermark [" + flood + "]");
|
||||
}
|
||||
}
|
||||
|
||||
private void setIncludeRelocations(boolean includeRelocations) {
|
||||
this.includeRelocations = includeRelocations;
|
||||
}
|
||||
|
|
@ -178,13 +296,28 @@ public class DiskThresholdSettings {
|
|||
* Attempts to parse the watermark into a percentage, returning 100.0% if
|
||||
* it cannot be parsed.
|
||||
*/
|
||||
private double thresholdPercentageFromWatermark(String watermark) {
|
||||
private static double thresholdPercentageFromWatermark(String watermark) {
|
||||
return thresholdPercentageFromWatermark(watermark, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to parse the watermark into a percentage, returning 100.0% if it can not be parsed and the specified lenient parameter is
|
||||
* true, otherwise throwing an {@link ElasticsearchParseException}.
|
||||
*
|
||||
* @param watermark the watermark to parse as a percentage
|
||||
* @param lenient true if lenient parsing should be applied
|
||||
* @return the parsed percentage
|
||||
*/
|
||||
private static double thresholdPercentageFromWatermark(String watermark, boolean lenient) {
|
||||
try {
|
||||
return RatioValue.parseRatioValue(watermark).getAsPercent();
|
||||
} catch (ElasticsearchParseException ex) {
|
||||
// NOTE: this is not end-user leniency, since up above we check that it's a valid byte or percentage, and then store the two
|
||||
// cases separately
|
||||
return 100.0;
|
||||
if (lenient) {
|
||||
return 100.0;
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -192,13 +325,29 @@ public class DiskThresholdSettings {
|
|||
* Attempts to parse the watermark into a {@link ByteSizeValue}, returning
|
||||
* a ByteSizeValue of 0 bytes if the value cannot be parsed.
|
||||
*/
|
||||
private ByteSizeValue thresholdBytesFromWatermark(String watermark, String settingName) {
|
||||
private static ByteSizeValue thresholdBytesFromWatermark(String watermark, String settingName) {
|
||||
return thresholdBytesFromWatermark(watermark, settingName, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to parse the watermark into a {@link ByteSizeValue}, returning zero bytes if it can not be parsed and the specified lenient
|
||||
* parameter is true, otherwise throwing an {@link ElasticsearchParseException}.
|
||||
*
|
||||
* @param watermark the watermark to parse as a byte size
|
||||
* @param settingName the name of the setting
|
||||
* @param lenient true if lenient parsing should be applied
|
||||
* @return the parsed byte size value
|
||||
*/
|
||||
private static ByteSizeValue thresholdBytesFromWatermark(String watermark, String settingName, boolean lenient) {
|
||||
try {
|
||||
return ByteSizeValue.parseBytesSizeValue(watermark, settingName);
|
||||
} catch (ElasticsearchParseException ex) {
|
||||
// NOTE: this is not end-user leniency, since up above we check that it's a valid byte or percentage, and then store the two
|
||||
// cases separately
|
||||
return ByteSizeValue.parseBytesSizeValue("0b", settingName);
|
||||
if (lenient) {
|
||||
return ByteSizeValue.parseBytesSizeValue("0b", settingName);
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,12 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.hasToString;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
|
||||
public class DiskThresholdSettingsTests extends ESTestCase {
|
||||
|
||||
public void testDefaults() {
|
||||
|
|
@ -47,18 +53,154 @@ public class DiskThresholdSettingsTests extends ESTestCase {
|
|||
Settings newSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), false)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING.getKey(), false)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "70%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "500mb")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "500mb")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1000mb")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "250mb")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "30s")
|
||||
.build();
|
||||
nss.applySettings(newSettings);
|
||||
|
||||
assertEquals(ByteSizeValue.parseBytesSizeValue("0b", "test"), diskThresholdSettings.getFreeBytesThresholdHigh());
|
||||
assertEquals(30.0D, diskThresholdSettings.getFreeDiskThresholdHigh(), 0.0D);
|
||||
assertEquals(ByteSizeValue.parseBytesSizeValue("500mb", "test"), diskThresholdSettings.getFreeBytesThresholdLow());
|
||||
assertEquals(ByteSizeValue.parseBytesSizeValue("500mb", "test"), diskThresholdSettings.getFreeBytesThresholdHigh());
|
||||
assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdHigh(), 0.0D);
|
||||
assertEquals(ByteSizeValue.parseBytesSizeValue("1000mb", "test"), diskThresholdSettings.getFreeBytesThresholdLow());
|
||||
assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdLow(), 0.0D);
|
||||
assertEquals(ByteSizeValue.parseBytesSizeValue("250mb", "test"), diskThresholdSettings.getFreeBytesThresholdFloodStage());
|
||||
assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdFloodStage(), 0.0D);
|
||||
assertEquals(30L, diskThresholdSettings.getRerouteInterval().seconds());
|
||||
assertFalse(diskThresholdSettings.isEnabled());
|
||||
assertFalse(diskThresholdSettings.includeRelocations());
|
||||
}
|
||||
|
||||
public void testInvalidConstruction() {
|
||||
final Settings settings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "80%")
|
||||
.build();
|
||||
final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
|
||||
final IllegalArgumentException e =
|
||||
expectThrows(IllegalArgumentException.class, () -> new DiskThresholdSettings(settings, clusterSettings));
|
||||
assertThat(e, hasToString(containsString("low disk watermark [90%] more than high disk watermark [80%]")));
|
||||
}
|
||||
|
||||
public void testInvalidLowHighPercentageUpdate() {
|
||||
final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
|
||||
new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater
|
||||
|
||||
final Settings newSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "80%")
|
||||
.build();
|
||||
|
||||
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings));
|
||||
final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [90%]";
|
||||
assertThat(e, hasToString(containsString(expected)));
|
||||
assertNotNull(e.getCause());
|
||||
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
|
||||
final IllegalArgumentException cause = (IllegalArgumentException) e.getCause();
|
||||
assertThat(cause, hasToString(containsString("low disk watermark [90%] more than high disk watermark [80%]")));
|
||||
}
|
||||
|
||||
public void testInvalidHighFloodPercentageUpdate() {
|
||||
final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
|
||||
new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater
|
||||
|
||||
final Settings newSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "50%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "60%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "55%")
|
||||
.build();
|
||||
|
||||
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings));
|
||||
final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [50%]";
|
||||
assertThat(e, hasToString(containsString(expected)));
|
||||
assertNotNull(e.getCause());
|
||||
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
|
||||
final IllegalArgumentException cause = (IllegalArgumentException) e.getCause();
|
||||
assertThat(cause, hasToString(containsString("high disk watermark [60%] more than flood stage disk watermark [55%]")));
|
||||
}
|
||||
|
||||
public void testInvalidLowHighBytesUpdate() {
|
||||
final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
|
||||
new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater
|
||||
|
||||
final Settings newSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "500m")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1000m")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "250m")
|
||||
.build();
|
||||
|
||||
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings));
|
||||
final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [500m]";
|
||||
assertThat(e, hasToString(containsString(expected)));
|
||||
assertNotNull(e.getCause());
|
||||
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
|
||||
final IllegalArgumentException cause = (IllegalArgumentException) e.getCause();
|
||||
assertThat(cause, hasToString(containsString("low disk watermark [500m] less than high disk watermark [1000m]")));
|
||||
}
|
||||
|
||||
public void testInvalidHighFloodBytesUpdate() {
|
||||
final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
|
||||
new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater
|
||||
|
||||
final Settings newSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "500m")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1000m")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "750m")
|
||||
.build();
|
||||
|
||||
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings));
|
||||
final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [500m]";
|
||||
assertThat(e, hasToString(containsString(expected)));
|
||||
assertNotNull(e.getCause());
|
||||
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
|
||||
final IllegalArgumentException cause = (IllegalArgumentException) e.getCause();
|
||||
assertThat(cause, hasToString(containsString("low disk watermark [500m] less than high disk watermark [1000m]")));
|
||||
}
|
||||
|
||||
public void testIncompatibleThresholdUpdate() {
|
||||
final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
|
||||
new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater
|
||||
|
||||
final Settings newSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1000m")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "95%")
|
||||
.build();
|
||||
|
||||
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings));
|
||||
final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [90%]";
|
||||
assertThat(e, hasToString(containsString(expected)));
|
||||
assertNotNull(e.getCause());
|
||||
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
|
||||
final IllegalArgumentException cause = (IllegalArgumentException) e.getCause();
|
||||
final String incompatibleExpected = String.format(
|
||||
Locale.ROOT,
|
||||
"unable to consistently parse [%s=%s], [%s=%s], and [%s=%s] as percentage or bytes",
|
||||
DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(),
|
||||
"90%",
|
||||
DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
|
||||
"1000m",
|
||||
DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(),
|
||||
"95%");
|
||||
assertThat(cause, hasToString(containsString(incompatibleExpected)));
|
||||
}
|
||||
|
||||
public void testInvalidHighDiskThreshold() {
|
||||
final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
|
||||
new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater
|
||||
|
||||
final Settings newSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "75%")
|
||||
.build();
|
||||
|
||||
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings));
|
||||
final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.high] from [90%] to [75%]";
|
||||
assertThat(e, hasToString(containsString(expected)));
|
||||
assertNotNull(e.getCause());
|
||||
assertNotNull(e.getCause());
|
||||
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
|
||||
final IllegalArgumentException cause = (IllegalArgumentException) e.getCause();
|
||||
assertThat(cause, hasToString(containsString("low disk watermark [85%] more than high disk watermark [75%]")));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,7 +77,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
|
|||
Settings diskSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.7)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.8).build();
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.8)
|
||||
.build();
|
||||
|
||||
ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
|
||||
usagesBuilder.put("node1", new DiskUsage("node1", "node1", "/dev/null", 100, 10)); // 90% used
|
||||
|
|
@ -179,7 +180,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
|
|||
diskSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "60%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.7).build();
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.7)
|
||||
.build();
|
||||
|
||||
deciders = new AllocationDeciders(Settings.EMPTY,
|
||||
new HashSet<>(Arrays.asList(
|
||||
|
|
@ -209,7 +211,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
|
|||
diskSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.5)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.6).build();
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.6)
|
||||
.build();
|
||||
|
||||
deciders = new AllocationDeciders(Settings.EMPTY,
|
||||
new HashSet<>(Arrays.asList(
|
||||
|
|
@ -259,7 +262,9 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
|
|||
Settings diskSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "30b")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "9b").build();
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "9b")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "5b")
|
||||
.build();
|
||||
|
||||
ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
|
||||
usagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 10)); // 90% used
|
||||
|
|
@ -397,7 +402,9 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
|
|||
diskSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "40b")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "30b").build();
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "30b")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "20b")
|
||||
.build();
|
||||
|
||||
deciders = new AllocationDeciders(Settings.EMPTY,
|
||||
new HashSet<>(Arrays.asList(
|
||||
|
|
@ -427,7 +434,9 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
|
|||
diskSettings = Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "50b")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "40b").build();
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "40b")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "30b")
|
||||
.build();
|
||||
|
||||
deciders = new AllocationDeciders(Settings.EMPTY,
|
||||
new HashSet<>(Arrays.asList(
|
||||
|
|
|
|||
|
|
@ -71,10 +71,13 @@ public class MockDiskUsagesIT extends ESIntegTestCase {
|
|||
cis.setN2Usage(nodes.get(1), new DiskUsage(nodes.get(1), "n2", "/dev/null", 100, 50));
|
||||
cis.setN3Usage(nodes.get(2), new DiskUsage(nodes.get(2), "n3", "/dev/null", 100, 50));
|
||||
|
||||
final boolean watermarkBytes = randomBoolean(); // we have to consistently use bytes or percentage for the disk watermark settings
|
||||
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), randomFrom("20b", "80%"))
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), randomFrom("10b", "90%"))
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), randomFrom("0b", "100%"))
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "20b" : "80%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "10b" : "90%")
|
||||
.put(
|
||||
DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(),
|
||||
watermarkBytes ? "0b" : "100%")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "1ms")).get();
|
||||
// Create an index with 10 shards so we can check allocation for it
|
||||
prepareCreate("test").setSettings(Settings.builder()
|
||||
|
|
|
|||
|
|
@ -29,14 +29,21 @@ file or updated dynamically on a live cluster with the
|
|||
the node.
|
||||
|
||||
`cluster.routing.allocation.disk.watermark.floodstage`::
|
||||
+
|
||||
--
|
||||
Controls the floodstage watermark. It defaults to 95%, meaning ES enforces a read-only
|
||||
index block (`index.blocks.read_only_allow_delete`) on every index that has
|
||||
one or more shards allocated on the node that has at least one disk exceeding the floodstage.
|
||||
This is a last resort to prevent nodes from running out of disk space.
|
||||
The index block must be released manually once there is enough disk space available
|
||||
to allow indexing operations to continue.
|
||||
|
||||
Controls the flood stage watermark. It defaults to 95%, meaning ES enforces
|
||||
a read-only index block (`index.blocks.read_only_allow_delete`) on every
|
||||
index that has one or more shards allocated on the node that has at least
|
||||
one disk exceeding the flood stage. This is a last resort to prevent nodes
|
||||
from running out of disk space. The index block must be released manually
|
||||
once there is enough disk space available to allow indexing operations to
|
||||
continue.
|
||||
|
||||
NOTE: You can not mix the usage of percentage values and byte values within
|
||||
these settings. Either all are set to percentage values, or all are set to byte
|
||||
values. This is so that we can we validate that the settings are internally
|
||||
consistent (that is, the low disk threshold is not more than the high disk
|
||||
threshold, and the high disk threshold is not more than the flood stage
|
||||
threshold).
|
||||
|
||||
An example of resetting the read-only index block on the `twitter` index:
|
||||
|
||||
|
|
@ -72,18 +79,16 @@ free disk space. This can be confusing, since it flips the meaning of high and
|
|||
low. For example, it makes sense to set the low watermark to 10gb and the high
|
||||
watermark to 5gb, but not the other way around.
|
||||
|
||||
|
||||
An example of updating the low watermark to no more than 80% of the disk size,
|
||||
a high watermark of at least 50 gigabytes free, and a floodstage watermark of
|
||||
10 gigabytes free, and updating the information about the cluster every
|
||||
minute:
|
||||
An example of updating the low watermark to at least 100 gigabytes free, a high
|
||||
watermark of at least 50 gigabytes free, and a flood stage watermark of 10
|
||||
gigabytes free, and updating the information about the cluster every minute:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
PUT _cluster/settings
|
||||
{
|
||||
"transient": {
|
||||
"cluster.routing.allocation.disk.watermark.low": "80%",
|
||||
"cluster.routing.allocation.disk.watermark.low": "100gb",
|
||||
"cluster.routing.allocation.disk.watermark.high": "50gb",
|
||||
"cluster.routing.allocation.disk.watermark.floodstage": "10gb",
|
||||
"cluster.info.update.interval": "1m"
|
||||
|
|
|
|||
|
|
@ -1721,6 +1721,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
|
|||
// from failing on nodes without enough disk space
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b")
|
||||
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b")
|
||||
.put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 2048)
|
||||
// by default we never cache below 10k docs in a segment,
|
||||
// bypass this limit so that caching gets some testing in
|
||||
|
|
|
|||
|
|
@ -331,6 +331,7 @@ public final class InternalTestCluster extends TestCluster {
|
|||
// from failing on nodes without enough disk space
|
||||
builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b");
|
||||
builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b");
|
||||
builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b");
|
||||
// Some tests make use of scripting quite a bit, so increase the limit for integration tests
|
||||
builder.put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 1000);
|
||||
if (TEST_NIGHTLY) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue