diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 4dbf3efe595..5ecb24f4557 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -21,6 +21,7 @@ package org.elasticsearch.gradle.test import org.apache.tools.ant.DefaultLogger import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.LoggedExec +import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.elasticsearch.gradle.plugin.PluginPropertiesExtension @@ -312,6 +313,9 @@ class ClusterFormationTasks { // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space esConfig['cluster.routing.allocation.disk.watermark.low'] = '1b' esConfig['cluster.routing.allocation.disk.watermark.high'] = '1b' + if (Version.fromString(node.nodeVersion).major >= 6) { + esConfig['cluster.routing.allocation.disk.watermark.floodstage'] = '1b' + } esConfig.putAll(node.config.settings) Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup) diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java index 71e4347ad69..8f21f29bea2 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java @@ -27,6 +27,11 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.RatioValue; import org.elasticsearch.common.unit.TimeValue; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; + /** * A container to keep settings for disk thresholds up to date with cluster setting changes. */ @@ -37,14 +42,17 @@ public class DiskThresholdSettings { public static final Setting CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING = new Setting<>("cluster.routing.allocation.disk.watermark.low", "85%", (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.low"), + new LowDiskWatermarkValidator(), Setting.Property.Dynamic, Setting.Property.NodeScope); public static final Setting CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING = new Setting<>("cluster.routing.allocation.disk.watermark.high", "90%", (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.high"), + new HighDiskWatermarkValidator(), Setting.Property.Dynamic, Setting.Property.NodeScope); public static final Setting CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING = new Setting<>("cluster.routing.allocation.disk.watermark.floodstage", "95%", (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.floodstage"), + new FloodStageValidator(), Setting.Property.Dynamic, Setting.Property.NodeScope); public static final Setting CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING = Setting.boolSetting("cluster.routing.allocation.disk.include_relocations", true, @@ -84,6 +92,116 @@ public class DiskThresholdSettings { clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING, this::setEnabled); } + static final class LowDiskWatermarkValidator implements Setting.Validator { + + @Override + public void validate(String value, Map, String> settings) { + final String highWatermarkRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING); + final String floodStageRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING); + doValidate(value, highWatermarkRaw, floodStageRaw); + } + + @Override + public Iterator> settings() { + return Arrays.asList( + CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING, + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING) + .iterator(); + } + + } + + static final class HighDiskWatermarkValidator implements Setting.Validator { + + @Override + public void validate(String value, Map, String> settings) { + final String lowWatermarkRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING); + final String floodStageRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING); + doValidate(lowWatermarkRaw, value, floodStageRaw); + } + + @Override + public Iterator> settings() { + return Arrays.asList( + CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING, + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING) + .iterator(); + } + + } + + static final class FloodStageValidator implements Setting.Validator { + + @Override + public void validate(String value, Map, String> settings) { + final String lowWatermarkRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING); + final String highWatermarkRaw = settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING); + doValidate(lowWatermarkRaw, highWatermarkRaw, value); + } + + @Override + public Iterator> settings() { + return Arrays.asList( + CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING, + CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING) + .iterator(); + } + } + + private static void doValidate(String low, String high, String flood) { + try { + doValidateAsPercentage(low, high, flood); + return; // early return so that we do not try to parse as bytes + } catch (final ElasticsearchParseException e) { + // swallow as we are now going to try to parse as bytes + } + try { + doValidateAsBytes(low, high, flood); + } catch (final ElasticsearchParseException e) { + final String message = String.format( + Locale.ROOT, + "unable to consistently parse [%s=%s], [%s=%s], and [%s=%s] as percentage or bytes", + CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), + low, + CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), + high, + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + flood); + throw new IllegalArgumentException(message, e); + } + } + + private static void doValidateAsPercentage(final String low, final String high, final String flood) { + final double lowWatermarkThreshold = thresholdPercentageFromWatermark(low, false); + final double highWatermarkThreshold = thresholdPercentageFromWatermark(high, false); + final double floodThreshold = thresholdPercentageFromWatermark(flood, false); + if (lowWatermarkThreshold > highWatermarkThreshold) { + throw new IllegalArgumentException( + "low disk watermark [" + low + "] more than high disk watermark [" + high + "]"); + } + if (highWatermarkThreshold > floodThreshold) { + throw new IllegalArgumentException( + "high disk watermark [" + high + "] more than flood stage disk watermark [" + flood + "]"); + } + } + + private static void doValidateAsBytes(final String low, final String high, final String flood) { + final ByteSizeValue lowWatermarkBytes = + thresholdBytesFromWatermark(low, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), false); + final ByteSizeValue highWatermarkBytes = + thresholdBytesFromWatermark(high, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), false); + final ByteSizeValue floodStageBytes = + thresholdBytesFromWatermark(flood, CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), false); + if (lowWatermarkBytes.getBytes() < highWatermarkBytes.getBytes()) { + throw new IllegalArgumentException( + "low disk watermark [" + low + "] less than high disk watermark [" + high + "]"); + } + if (highWatermarkBytes.getBytes() < floodStageBytes.getBytes()) { + throw new IllegalArgumentException( + "high disk watermark [" + high + "] less than flood stage disk watermark [" + flood + "]"); + } + } + private void setIncludeRelocations(boolean includeRelocations) { this.includeRelocations = includeRelocations; } @@ -178,13 +296,28 @@ public class DiskThresholdSettings { * Attempts to parse the watermark into a percentage, returning 100.0% if * it cannot be parsed. */ - private double thresholdPercentageFromWatermark(String watermark) { + private static double thresholdPercentageFromWatermark(String watermark) { + return thresholdPercentageFromWatermark(watermark, true); + } + + /** + * Attempts to parse the watermark into a percentage, returning 100.0% if it can not be parsed and the specified lenient parameter is + * true, otherwise throwing an {@link ElasticsearchParseException}. + * + * @param watermark the watermark to parse as a percentage + * @param lenient true if lenient parsing should be applied + * @return the parsed percentage + */ + private static double thresholdPercentageFromWatermark(String watermark, boolean lenient) { try { return RatioValue.parseRatioValue(watermark).getAsPercent(); } catch (ElasticsearchParseException ex) { // NOTE: this is not end-user leniency, since up above we check that it's a valid byte or percentage, and then store the two // cases separately - return 100.0; + if (lenient) { + return 100.0; + } + throw ex; } } @@ -192,13 +325,29 @@ public class DiskThresholdSettings { * Attempts to parse the watermark into a {@link ByteSizeValue}, returning * a ByteSizeValue of 0 bytes if the value cannot be parsed. */ - private ByteSizeValue thresholdBytesFromWatermark(String watermark, String settingName) { + private static ByteSizeValue thresholdBytesFromWatermark(String watermark, String settingName) { + return thresholdBytesFromWatermark(watermark, settingName, true); + } + + /** + * Attempts to parse the watermark into a {@link ByteSizeValue}, returning zero bytes if it can not be parsed and the specified lenient + * parameter is true, otherwise throwing an {@link ElasticsearchParseException}. + * + * @param watermark the watermark to parse as a byte size + * @param settingName the name of the setting + * @param lenient true if lenient parsing should be applied + * @return the parsed byte size value + */ + private static ByteSizeValue thresholdBytesFromWatermark(String watermark, String settingName, boolean lenient) { try { return ByteSizeValue.parseBytesSizeValue(watermark, settingName); } catch (ElasticsearchParseException ex) { // NOTE: this is not end-user leniency, since up above we check that it's a valid byte or percentage, and then store the two // cases separately - return ByteSizeValue.parseBytesSizeValue("0b", settingName); + if (lenient) { + return ByteSizeValue.parseBytesSizeValue("0b", settingName); + } + throw ex; } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java index fa4518218c9..342fcea7dde 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java @@ -24,6 +24,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.test.ESTestCase; +import java.util.Locale; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.instanceOf; + public class DiskThresholdSettingsTests extends ESTestCase { public void testDefaults() { @@ -47,18 +53,154 @@ public class DiskThresholdSettingsTests extends ESTestCase { Settings newSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), false) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING.getKey(), false) - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "70%") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "500mb") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "500mb") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1000mb") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "250mb") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "30s") .build(); nss.applySettings(newSettings); - assertEquals(ByteSizeValue.parseBytesSizeValue("0b", "test"), diskThresholdSettings.getFreeBytesThresholdHigh()); - assertEquals(30.0D, diskThresholdSettings.getFreeDiskThresholdHigh(), 0.0D); - assertEquals(ByteSizeValue.parseBytesSizeValue("500mb", "test"), diskThresholdSettings.getFreeBytesThresholdLow()); + assertEquals(ByteSizeValue.parseBytesSizeValue("500mb", "test"), diskThresholdSettings.getFreeBytesThresholdHigh()); + assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdHigh(), 0.0D); + assertEquals(ByteSizeValue.parseBytesSizeValue("1000mb", "test"), diskThresholdSettings.getFreeBytesThresholdLow()); assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdLow(), 0.0D); + assertEquals(ByteSizeValue.parseBytesSizeValue("250mb", "test"), diskThresholdSettings.getFreeBytesThresholdFloodStage()); + assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdFloodStage(), 0.0D); assertEquals(30L, diskThresholdSettings.getRerouteInterval().seconds()); assertFalse(diskThresholdSettings.isEnabled()); assertFalse(diskThresholdSettings.includeRelocations()); } + + public void testInvalidConstruction() { + final Settings settings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "80%") + .build(); + final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final IllegalArgumentException e = + expectThrows(IllegalArgumentException.class, () -> new DiskThresholdSettings(settings, clusterSettings)); + assertThat(e, hasToString(containsString("low disk watermark [90%] more than high disk watermark [80%]"))); + } + + public void testInvalidLowHighPercentageUpdate() { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater + + final Settings newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "80%") + .build(); + + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); + final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [90%]"; + assertThat(e, hasToString(containsString(expected))); + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); + assertThat(cause, hasToString(containsString("low disk watermark [90%] more than high disk watermark [80%]"))); + } + + public void testInvalidHighFloodPercentageUpdate() { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater + + final Settings newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "50%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "60%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "55%") + .build(); + + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); + final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [50%]"; + assertThat(e, hasToString(containsString(expected))); + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); + assertThat(cause, hasToString(containsString("high disk watermark [60%] more than flood stage disk watermark [55%]"))); + } + + public void testInvalidLowHighBytesUpdate() { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater + + final Settings newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "500m") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1000m") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "250m") + .build(); + + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); + final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [500m]"; + assertThat(e, hasToString(containsString(expected))); + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); + assertThat(cause, hasToString(containsString("low disk watermark [500m] less than high disk watermark [1000m]"))); + } + + public void testInvalidHighFloodBytesUpdate() { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater + + final Settings newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "500m") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1000m") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "750m") + .build(); + + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); + final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [500m]"; + assertThat(e, hasToString(containsString(expected))); + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); + assertThat(cause, hasToString(containsString("low disk watermark [500m] less than high disk watermark [1000m]"))); + } + + public void testIncompatibleThresholdUpdate() { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater + + final Settings newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1000m") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "95%") + .build(); + + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); + final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [90%]"; + assertThat(e, hasToString(containsString(expected))); + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); + final String incompatibleExpected = String.format( + Locale.ROOT, + "unable to consistently parse [%s=%s], [%s=%s], and [%s=%s] as percentage or bytes", + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), + "90%", + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), + "1000m", + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + "95%"); + assertThat(cause, hasToString(containsString(incompatibleExpected))); + } + + public void testInvalidHighDiskThreshold() { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater + + final Settings newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "75%") + .build(); + + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); + final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.high] from [90%] to [75%]"; + assertThat(e, hasToString(containsString(expected))); + assertNotNull(e.getCause()); + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); + assertThat(cause, hasToString(containsString("low disk watermark [85%] more than high disk watermark [75%]"))); + } + } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 805a8259bd0..322d4b25ec9 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -77,7 +77,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { Settings diskSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.7) - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.8).build(); + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.8) + .build(); ImmutableOpenMap.Builder usagesBuilder = ImmutableOpenMap.builder(); usagesBuilder.put("node1", new DiskUsage("node1", "node1", "/dev/null", 100, 10)); // 90% used @@ -179,7 +180,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { diskSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "60%") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.7).build(); + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.7) + .build(); deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList( @@ -209,7 +211,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { diskSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.5) - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.6).build(); + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.6) + .build(); deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList( @@ -259,7 +262,9 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { Settings diskSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "30b") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "9b").build(); + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "9b") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "5b") + .build(); ImmutableOpenMap.Builder usagesBuilder = ImmutableOpenMap.builder(); usagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 10)); // 90% used @@ -397,7 +402,9 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { diskSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "40b") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "30b").build(); + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "30b") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "20b") + .build(); deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList( @@ -427,7 +434,9 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { diskSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "50b") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "40b").build(); + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "40b") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "30b") + .build(); deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList( diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java index 8b0b8917d26..1ad18c1f69f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java @@ -71,10 +71,13 @@ public class MockDiskUsagesIT extends ESIntegTestCase { cis.setN2Usage(nodes.get(1), new DiskUsage(nodes.get(1), "n2", "/dev/null", 100, 50)); cis.setN3Usage(nodes.get(2), new DiskUsage(nodes.get(2), "n3", "/dev/null", 100, 50)); + final boolean watermarkBytes = randomBoolean(); // we have to consistently use bytes or percentage for the disk watermark settings client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder() - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), randomFrom("20b", "80%")) - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), randomFrom("10b", "90%")) - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), randomFrom("0b", "100%")) + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "20b" : "80%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "10b" : "90%") + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + watermarkBytes ? "0b" : "100%") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "1ms")).get(); // Create an index with 10 shards so we can check allocation for it prepareCreate("test").setSettings(Settings.builder() diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc index e63746ee9f4..a2a431299ae 100644 --- a/docs/reference/modules/cluster/disk_allocator.asciidoc +++ b/docs/reference/modules/cluster/disk_allocator.asciidoc @@ -29,14 +29,21 @@ file or updated dynamically on a live cluster with the the node. `cluster.routing.allocation.disk.watermark.floodstage`:: -+ --- - Controls the floodstage watermark. It defaults to 95%, meaning ES enforces a read-only - index block (`index.blocks.read_only_allow_delete`) on every index that has - one or more shards allocated on the node that has at least one disk exceeding the floodstage. - This is a last resort to prevent nodes from running out of disk space. - The index block must be released manually once there is enough disk space available - to allow indexing operations to continue. + + Controls the flood stage watermark. It defaults to 95%, meaning ES enforces + a read-only index block (`index.blocks.read_only_allow_delete`) on every + index that has one or more shards allocated on the node that has at least + one disk exceeding the flood stage. This is a last resort to prevent nodes + from running out of disk space. The index block must be released manually + once there is enough disk space available to allow indexing operations to + continue. + +NOTE: You can not mix the usage of percentage values and byte values within +these settings. Either all are set to percentage values, or all are set to byte +values. This is so that we can we validate that the settings are internally +consistent (that is, the low disk threshold is not more than the high disk +threshold, and the high disk threshold is not more than the flood stage +threshold). An example of resetting the read-only index block on the `twitter` index: @@ -72,18 +79,16 @@ free disk space. This can be confusing, since it flips the meaning of high and low. For example, it makes sense to set the low watermark to 10gb and the high watermark to 5gb, but not the other way around. - -An example of updating the low watermark to no more than 80% of the disk size, -a high watermark of at least 50 gigabytes free, and a floodstage watermark of -10 gigabytes free, and updating the information about the cluster every -minute: +An example of updating the low watermark to at least 100 gigabytes free, a high +watermark of at least 50 gigabytes free, and a flood stage watermark of 10 +gigabytes free, and updating the information about the cluster every minute: [source,js] -------------------------------------------------- PUT _cluster/settings { "transient": { - "cluster.routing.allocation.disk.watermark.low": "80%", + "cluster.routing.allocation.disk.watermark.low": "100gb", "cluster.routing.allocation.disk.watermark.high": "50gb", "cluster.routing.allocation.disk.watermark.floodstage": "10gb", "cluster.info.update.interval": "1m" diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 357658d1575..6d93fe7099f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1721,6 +1721,7 @@ public abstract class ESIntegTestCase extends ESTestCase { // from failing on nodes without enough disk space .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b") .put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 2048) // by default we never cache below 10k docs in a segment, // bypass this limit so that caching gets some testing in diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 888807bf67c..72d2a690b5e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -331,6 +331,7 @@ public final class InternalTestCluster extends TestCluster { // from failing on nodes without enough disk space builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b"); builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b"); + builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b"); // Some tests make use of scripting quite a bit, so increase the limit for integration tests builder.put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 1000); if (TEST_NIGHTLY) {