From 1f0673c9bd7e26ddf5ce55c6a184db2dbf4c778a Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Aug 2016 09:26:20 -0400 Subject: [PATCH] Default max local storage nodes to one This commit defaults the max local storage nodes to one. The motivation for this change is that a default value greather than one is dangerous as users sometimes end up unknowingly starting a second node and start thinking that they have encountered data loss. Relates #19964 --- .../gradle/test/ClusterFormationTasks.groovy | 1 + .../elasticsearch/env/NodeEnvironment.java | 14 ++++++++++--- .../org/elasticsearch/tribe/TribeService.java | 3 +++ .../cluster/ClusterInfoServiceIT.java | 2 ++ .../DiscoveryWithServiceDisruptionsIT.java | 2 ++ .../env/NodeEnvironmentTests.java | 16 ++++++--------- .../memory/breaker/CircuitBreakerNoopIT.java | 2 ++ .../java/org/elasticsearch/tribe/TribeIT.java | 3 ++- .../migration/migrate_5_0/settings.asciidoc | 11 ++++++++++ .../elasticsearch/tribe/TribeUnitTests.java | 4 +++- .../elasticsearch/test/ESIntegTestCase.java | 2 ++ .../test/InternalTestCluster.java | 5 +++-- .../ClusterDiscoveryConfiguration.java | 3 ++- .../test/test/InternalTestClusterTests.java | 20 ++++++++++++++----- 14 files changed, 65 insertions(+), 23 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 442882dfe99..b1f07265019 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -261,6 +261,7 @@ class ClusterFormationTasks { 'node.attr.testattr' : 'test', 'repositories.url.allowed_urls': 'http://snapshot.test*' ] + esConfig['node.max_local_storage_nodes'] = node.config.numNodes esConfig['http.port'] = node.config.httpPort esConfig['transport.tcp.port'] = node.config.transportPort esConfig.putAll(node.config.settings) diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 367131d93cd..59ef122760b 100644 --- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -68,6 +68,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.Set; @@ -151,7 +152,7 @@ public final class NodeEnvironment implements Closeable { /** * Maximum number of data nodes that should run in an environment. */ - public static final Setting MAX_LOCAL_STORAGE_NODES_SETTING = Setting.intSetting("node.max_local_storage_nodes", 50, 1, + public static final Setting MAX_LOCAL_STORAGE_NODES_SETTING = Setting.intSetting("node.max_local_storage_nodes", 1, 1, Property.NodeScope); /** @@ -244,8 +245,15 @@ public final class NodeEnvironment implements Closeable { } if (locks[0] == null) { - throw new IllegalStateException("Failed to obtain node lock, is the following location writable?: " - + Arrays.toString(environment.dataWithClusterFiles()), lastException); + final String message = String.format( + Locale.ROOT, + "failed to obtain node locks, tried [%s] with lock id%s;" + + " maybe these locations are not writable or multiple nodes were started without increasing [%s] (was [%d])?", + Arrays.toString(environment.dataWithClusterFiles()), + maxLocalStorageNodes == 1 ? " [0]" : "s [0--" + (maxLocalStorageNodes - 1) + "]", + MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), + maxLocalStorageNodes); + throw new IllegalStateException(message, lastException); } this.nodeMetaData = loadOrCreateNodeMetaData(settings, startupTraceLogger, nodePaths); this.logger = Loggers.getLogger(getClass(), Node.addNodeNameIfNeeded(settings, this.nodeMetaData.nodeId())); diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java index 40c805e0b00..3ca80155270 100644 --- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java +++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java @@ -117,6 +117,9 @@ public class TribeService extends AbstractLifecycleComponent { sb.put(Node.NODE_MASTER_SETTING.getKey(), false); sb.put(Node.NODE_DATA_SETTING.getKey(), false); sb.put(Node.NODE_INGEST_SETTING.getKey(), false); + if (!NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.exists(settings)) { + sb.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), nodesSettings.size()); + } sb.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local"); // a tribe node should not use zen discovery // nothing is going to be discovered, since no master will be elected sb.put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0); diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index e1f5a1a719b..3cafff08a07 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.store.Store; @@ -118,6 +119,7 @@ public class ClusterInfoServiceIT extends ESIntegTestCase { protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() // manual collection or upon cluster forming. + .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2) .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), "1s") .build(); } diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java index ab42abd4aac..e5de81f7380 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java @@ -55,6 +55,7 @@ import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.ZenPingService; import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.indices.store.IndicesStoreIntegrationIT; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -207,6 +208,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { // TODO: Rarely use default settings form some of these Settings nodeSettings = Settings.builder() .put(settings) + .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 4) .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNode) .build(); diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 96c52d9dc8e..ee403bfe910 100644 --- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -74,18 +74,14 @@ public class NodeEnvironmentTests extends ESTestCase { } public void testNodeLockSingleEnvironment() throws IOException { - final Settings settings = buildEnvSettings(Settings.builder() - .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 1).build()); + final Settings settings = buildEnvSettings(Settings.builder().put("node.max_local_storage_nodes", 1).build()); NodeEnvironment env = newNodeEnvironment(settings); List dataPaths = Environment.PATH_DATA_SETTING.get(settings); - try { - // Reuse the same location and attempt to lock again - new NodeEnvironment(settings, new Environment(settings)); - fail("env has already locked all the data directories it is allowed"); - } catch (IllegalStateException ex) { - assertThat(ex.getMessage(), containsString("Failed to obtain node lock")); - } + // Reuse the same location and attempt to lock again + IllegalStateException ex = + expectThrows(IllegalStateException.class, () -> new NodeEnvironment(settings, new Environment(settings))); + assertThat(ex.getMessage(), containsString("failed to obtain node lock")); // Close the environment that holds the lock and make sure we can get the lock after release env.close(); @@ -121,7 +117,7 @@ public class NodeEnvironmentTests extends ESTestCase { } public void testNodeLockMultipleEnvironment() throws IOException { - final Settings settings = buildEnvSettings(Settings.EMPTY); + final Settings settings = buildEnvSettings(Settings.builder().put("node.max_local_storage_nodes", 2).build()); final NodeEnvironment first = newNodeEnvironment(settings); List dataPaths = Environment.PATH_DATA_SETTING.get(settings); NodeEnvironment second = new NodeEnvironment(settings, new Environment(settings)); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java index 02eab6dc0aa..627eb74007b 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java @@ -22,6 +22,7 @@ package org.elasticsearch.indices.memory.breaker; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; @@ -40,6 +41,7 @@ public class CircuitBreakerNoopIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() + .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2) .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING.getKey(), "noop") // This is set low, because if the "noop" is not a noop, it will break .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "10b") diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java index e1df7201fbe..01e6a490dad 100644 --- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java +++ b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; @@ -128,6 +129,7 @@ public class TribeIT extends ESIntegTestCase { tribe1Defaults.putArray("tribe.t2." + UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), getUnicastHosts(cluster2.client())); Settings merged = Settings.builder() + .put(internalCluster().getDefaultSettings()) .put("tribe.t1.cluster.name", internalCluster().getClusterName()) .put("tribe.t2.cluster.name", cluster2.getClusterName()) .put("tribe.t1.transport.type", "local") @@ -142,7 +144,6 @@ public class TribeIT extends ESIntegTestCase { .put(tribe1Defaults.build()) .put(tribe2Defaults.build()) - .put(internalCluster().getDefaultSettings()) .put("node.name", "tribe_node") // make sure we can identify threads from this node .build(); diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc index 76ee65a6abe..f6875f60c60 100644 --- a/docs/reference/migration/migrate_5_0/settings.asciidoc +++ b/docs/reference/migration/migrate_5_0/settings.asciidoc @@ -310,3 +310,14 @@ The unit 'w' representing weeks is no longer supported. Fractional time values (e.g., 0.5s) are no longer supported. For example, this means when setting timeouts "0.5s" will be rejected and should instead be input as "500ms". + +==== Node max local storage nodes + +Previous versions of Elasticsearch defaulted to allowing multiple nodes to share the same data +directory (up to 50). This can be confusing where users accidentally startup multiple nodes and end +up thinking that they've lost data because the second node will start with an empty data directory. +While the default of allowing multiple nodes is friendly to playing with forming a small cluster on +a laptop, and end-users do sometimes run multiple nodes on the same host, this tends to be the +exception. Keeping with Elasticsearch's continual movement towards safer out-of-the-box defaults, +and optimizing for the norm instead of the exception, the default for +`node.max_local_storage_nodes` is now one. diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java index 95550f6f654..247caa42210 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java @@ -60,7 +60,9 @@ public class TribeUnitTests extends ESTestCase { .put(NetworkModule.HTTP_ENABLED.getKey(), false) .put("transport.type", "local") .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local") - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2) + .build(); tribe1 = new TribeClientNode( Settings.builder() diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 63aa4484255..dffa285101f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -31,6 +31,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.script.ScriptService; import org.elasticsearch.transport.MockTcpTransportPlugin; import org.elasticsearch.action.ActionListener; @@ -1630,6 +1631,7 @@ public abstract class ESIntegTestCase extends ESTestCase { */ protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder() + .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), Integer.MAX_VALUE) // Default the watermarks to absurdly low to prevent the tests // from failing on nodes without enough disk space .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b") diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index c121164fb01..cc78afb987f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -175,8 +175,8 @@ public final class InternalTestCluster extends TestCluster { public final int HTTP_BASE_PORT = GLOBAL_HTTP_BASE_PORT + CLUSTER_BASE_PORT_OFFSET; - static final int DEFAULT_LOW_NUM_MASTER_NODES = 1; - static final int DEFAULT_HIGH_NUM_MASTER_NODES = 3; + public static final int DEFAULT_LOW_NUM_MASTER_NODES = 1; + public static final int DEFAULT_HIGH_NUM_MASTER_NODES = 3; static final int DEFAULT_MIN_NUM_DATA_NODES = 1; static final int DEFAULT_MAX_NUM_DATA_NODES = TEST_NIGHTLY ? 6 : 3; @@ -300,6 +300,7 @@ public final class InternalTestCluster extends TestCluster { builder.put(Environment.PATH_DATA_SETTING.getKey(), dataPath.toString()); } } + builder.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), Integer.MAX_VALUE); builder.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), baseDir.resolve("custom")); builder.put(Environment.PATH_HOME_SETTING.getKey(), baseDir); builder.put(Environment.PATH_REPO_SETTING.getKey(), baseDir.resolve("repos")); diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java index 67d7b99171d..b393498ec89 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java +++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.network.NetworkUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.discovery.DiscoveryModule; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.NodeConfigurationSource; import org.elasticsearch.transport.TransportSettings; @@ -108,7 +109,7 @@ public class ClusterDiscoveryConfiguration extends NodeConfigurationSource { @Override public Settings nodeSettings(int nodeOrdinal) { - Settings.Builder builder = Settings.builder(); + Settings.Builder builder = Settings.builder().put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numOfNodes); String[] unicastHosts = new String[unicastHostOrdinals.length]; if (nodeOrdinal >= unicastHostPorts.length) { diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index d4af031aa84..f0b7454fe9d 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -128,12 +128,16 @@ public class InternalTestClusterTests extends ESTestCase { boolean masterNodes = randomBoolean(); int minNumDataNodes = randomIntBetween(0, 3); int maxNumDataNodes = randomIntBetween(minNumDataNodes, 4); + int numClientNodes = randomIntBetween(0, 2); final String clusterName1 = "shared1"; final String clusterName2 = "shared2"; NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() { @Override public Settings nodeSettings(int nodeOrdinal) { return Settings.builder() + .put( + NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), + 2 * ((masterNodes ? InternalTestCluster.DEFAULT_HIGH_NUM_MASTER_NODES : 0) + maxNumDataNodes + numClientNodes)) .put(NetworkModule.HTTP_ENABLED.getKey(), false) .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local") .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build(); @@ -145,7 +149,7 @@ public class InternalTestClusterTests extends ESTestCase { .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build(); } }; - int numClientNodes = randomIntBetween(0, 2); + boolean enableHttpPipelining = randomBoolean(); String nodePrefix = "foobar"; @@ -187,13 +191,17 @@ public class InternalTestClusterTests extends ESTestCase { long clusterSeed = randomLong(); boolean masterNodes = randomBoolean(); // we need one stable node - int minNumDataNodes = 2; - int maxNumDataNodes = 2; + final int minNumDataNodes = 2; + final int maxNumDataNodes = 2; + final int numClientNodes = randomIntBetween(0, 2); final String clusterName1 = "shared1"; NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() { @Override public Settings nodeSettings(int nodeOrdinal) { return Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false) + .put( + NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), + 2 + (masterNodes ? InternalTestCluster.DEFAULT_HIGH_NUM_MASTER_NODES : 0) + maxNumDataNodes + numClientNodes) .put(NetworkModule.TRANSPORT_TYPE_KEY, "local") .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local") .build(); @@ -203,7 +211,7 @@ public class InternalTestClusterTests extends ESTestCase { return Settings.builder() .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build(); } - }; int numClientNodes = randomIntBetween(0, 2); + }; boolean enableHttpPipelining = randomBoolean(); String nodePrefix = "test"; Path baseDir = createTempDir(); @@ -269,11 +277,13 @@ public class InternalTestClusterTests extends ESTestCase { public void testDifferentRolesMaintainPathOnRestart() throws Exception { final Path baseDir = createTempDir(); + final int numNodes = 5; InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, true, 0, 0, "test", new NodeConfigurationSource() { @Override public Settings nodeSettings(int nodeOrdinal) { return Settings.builder() + .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numNodes) .put(NetworkModule.HTTP_ENABLED.getKey(), false) .put(NetworkModule.TRANSPORT_TYPE_KEY, "local") .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local") @@ -289,7 +299,7 @@ public class InternalTestClusterTests extends ESTestCase { cluster.beforeTest(random(), 0.0); try { Map> pathsPerRole = new HashMap<>(); - for (int i = 0; i < 5; i++) { + for (int i = 0; i < numNodes; i++) { final DiscoveryNode.Role role = randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST); final String node; switch (role) {