[Zen2] Change unsafe bootstrap nodes count to nodes list in tests (#36559)

This commit modifies ESSingleNodeTestCase and ESIntegTestCase and
several concrete test classes to use node names when bootstrapping the
cluster.

Today ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING
setting is used to bootstrap clusters in tests. Instead, we want to use
ClusterBootrstapService.INITIAL_MASTER_NODES_SETTING and get rid of
the former setting eventually.

There were two main problems when refactoring InternalTestCluster:

1. Nodes are created one-by-one in buildNode method. And node.name
is created in this method as well. It's not suitable for bootstrapping,
because we need to have the names of all master eligible nodes in
advance, before creating the node with bootstrapping configuration set.
We address this issue by separating buildNode into two methods:
getNodeSettings and buildNode. We first iterate over all nodes to
get nodes settings, then change the setting for the bootstrapping node
and then proceed with building the node.
2. If autoManageMinMasterNodes = false, there is no way for the test to
set the list of bootstrapping nodes because node names are not known in
advance. This problem is solved by adding updateNodesSettings method
to NodeConfigurationSource and ESIntegTestCase (which could be
overridden by concrete integration test class). Once we have the list
of settings for all nodes, the integration test class is allowed to
update it. In our case, we update the
ClusterBootrstapService.INITIAL_MASTER_NODES_SETTING setting.

modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java

@@ -43,8 +43,10 @@ import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.core.Is.is;
 
@@ -60,11 +62,25 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase {
             .put(TestZenDiscovery.USE_ZEN2.getKey(), true)
             .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE);
 
-        if (nodeOrdinal == 0) {
+        return builder.build();
-            builder.put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 2);
     }
 
-        return builder.build();
+    @Override
+    protected List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
+        final Settings firstNodeSettings = allNodesSettings.get(0);
+        final List<Settings> otherNodesSettings = allNodesSettings.subList(1, allNodesSettings.size());
+        final List<String> masterNodeNames = allNodesSettings.stream()
+                .filter(org.elasticsearch.node.Node.NODE_MASTER_SETTING::get)
+                .map(org.elasticsearch.node.Node.NODE_NAME_SETTING::get)
+                .collect(Collectors.toList());
+        final List<Settings> updatedSettings = new ArrayList<>();
+
+        updatedSettings.add(Settings.builder().put(firstNodeSettings)
+                .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), masterNodeNames)
+                .build());
+        updatedSettings.addAll(otherNodesSettings);
+
+        return updatedSettings;
     }
 
     @Override

server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java

@@ -31,6 +31,8 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -40,6 +42,23 @@ import static org.hamcrest.Matchers.hasItem;
 public class RecoverAfterNodesIT extends ESIntegTestCase {
     private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10);
 
+    @Override
+    protected List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
+        if (internalCluster().numDataAndMasterNodes() == 0) {
+            final Settings firstNodeSettings = allNodesSettings.get(0);
+            final List<Settings> otherNodesSettings = allNodesSettings.subList(1, allNodesSettings.size());
+
+            final List<Settings> updatedSettings = new ArrayList<>();
+            updatedSettings.add(Settings.builder().put(firstNodeSettings)
+                    .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
+                            Node.NODE_NAME_SETTING.get(firstNodeSettings)).build());
+            updatedSettings.addAll(otherNodesSettings);
+
+            return updatedSettings;
+        }
+        return super.addExtraClusterBootstrapSettings(allNodesSettings);
+    }
+
     public Set<ClusterBlock> waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException {
         long start = System.currentTimeMillis();
         Set<ClusterBlock> blocks;
@@ -60,9 +79,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
 
     public void testRecoverAfterNodes() throws Exception {
         logger.info("--> start node (1)");
-        Client clientNode1 = startNode(Settings.builder()
+        Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
-                .put("gateway.recover_after_nodes", 3)
-                .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1);
         assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -89,8 +106,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         logger.info("--> start master_node (1)");
         Client master1 = startNode(Settings.builder()
             .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false)
-            .put(Node.NODE_MASTER_SETTING.getKey(), true)
+            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
-            .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1);
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -136,8 +152,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         Client master1 = startNode(Settings.builder()
             .put("gateway.recover_after_data_nodes", 2)
             .put(Node.NODE_DATA_SETTING.getKey(), false)
-            .put(Node.NODE_MASTER_SETTING.getKey(), true)
+            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
-            .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1);
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -1947,6 +1947,11 @@ public abstract class ESIntegTestCase extends ESTestCase {
                     .put(ESIntegTestCase.this.nodeSettings(nodeOrdinal)).build();
             }
 
+            @Override
+            public List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
+                return ESIntegTestCase.this.addExtraClusterBootstrapSettings(allNodesSettings);
+            }
+
             @Override
             public Path nodeConfigPath(int nodeOrdinal) {
                 return ESIntegTestCase.this.nodeConfigPath(nodeOrdinal);
@@ -1975,6 +1980,19 @@ public abstract class ESIntegTestCase extends ESTestCase {
         };
     }
 
+    /**
+     * This method is called before starting a collection of nodes.
+     * At this point the test has a holistic view on all nodes settings and might perform settings adjustments as needed.
+     * For instance, the test could retrieve master node names and fill in
+     * {@link org.elasticsearch.cluster.coordination.ClusterBootstrapService#INITIAL_MASTER_NODES_SETTING} setting.
+     *
+     * @param allNodesSettings list of node settings before update
+     * @return list of node settings after update
+     */
+    protected List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
+        return allNodesSettings;
+    }
+
     /**
      * Iff this returns true mock transport implementations are used for the test runs. Otherwise not mock transport impls are used.
      * The default is {@code true}.

test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java

@@ -62,7 +62,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 
-import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING;
+import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
 import static org.elasticsearch.discovery.zen.SettingsBasedHostsProvider.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
@@ -179,6 +179,8 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
 
     private Node newNode() {
         final Path tempDir = createTempDir();
+        final String nodeName = nodeSettings().get(Node.NODE_NAME_SETTING.getKey(), "node_s_0");
+
         Settings settings = Settings.builder()
             .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", random().nextLong()))
             .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
@@ -186,7 +188,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             // TODO: use a consistent data path for custom paths
             // This needs to tie into the ESIntegTestCase#indexSettings() method
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
-            .put("node.name", "node_s_0")
+            .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
             .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "1000/1m")
             .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
             .put("transport.type", getTestTransportType())
@@ -201,9 +203,10 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             // turn it off for these tests.
             .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), false)
             .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()) // empty list disables a port scan for other nodes
-            .put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1)
+            .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeName)
             .put(nodeSettings()) // allow test cases to provide their own settings or override these
             .build();
+
         Collection<Class<? extends Plugin>> plugins = getPlugins();
         if (plugins.contains(getTestTransportPlugin()) == false) {
             plugins = new ArrayList<>(plugins);

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -148,7 +148,7 @@ import java.util.stream.Stream;
 import static java.util.Collections.emptyList;
 import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY;
 import static org.apache.lucene.util.LuceneTestCase.rarely;
-import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING;
+import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.discovery.DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING;
@@ -525,10 +525,15 @@ public final class InternalTestCluster extends TestCluster {
         if (randomNodeAndClient != null) {
             return randomNodeAndClient;
         }
-        final int ord = nextNodeId.getAndIncrement();
         final Runnable onTransportServiceStarted = () -> {}; // do not create unicast host file for this one node.
-        final Settings settings = Settings.builder().put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1).build();
+
-        final NodeAndClient buildNode = buildNode(ord, random.nextLong(), settings, false, 1, onTransportServiceStarted);
+        final int nodeId = nextNodeId.getAndIncrement();
+        final Settings settings = getNodeSettings(nodeId, random.nextLong(), Settings.EMPTY, 1);
+        final Settings nodeSettings = Settings.builder()
+                .putList(INITIAL_MASTER_NODES_SETTING.getKey(), Node.NODE_NAME_SETTING.get(settings))
+                .put(settings)
+                .build();
+        final NodeAndClient buildNode = buildNode(nodeId, nodeSettings, false, onTransportServiceStarted);
         assert nodes.isEmpty();
         buildNode.startNode();
         publishNode(buildNode);
@@ -598,36 +603,51 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
-    /**
+    private Settings getNodeSettings(final int nodeId, final long seed, final Settings extraSettings, final int defaultMinMasterNodes) {
-     * builds a new node given the settings.
+        final Settings settings = getSettings(nodeId, seed, extraSettings);
-     *
+
-     * @param settings              the settings to use
+        final String name = buildNodeName(nodeId, settings);
-     * @param defaultMinMasterNodes min_master_nodes value to use if min_master_nodes is auto managed
+
-     * @param onTransportServiceStarted callback to run when transport service is started
+        final Settings.Builder updatedSettings = Settings.builder()
-     */
+                .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home
-    private NodeAndClient buildNode(Settings settings, int defaultMinMasterNodes, Runnable onTransportServiceStarted) {
+                .put(settings)
-        int ord = nextNodeId.getAndIncrement();
+                .put("node.name", name)
-        return buildNode(ord, random.nextLong(), settings, false, defaultMinMasterNodes, onTransportServiceStarted);
+                .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);
+
+        final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()).equals("single-node");
+        if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) {
+            assert updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null :
+                    "min master nodes may not be set when auto managed";
+            assert updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null :
+                    "automatically managing min master nodes require nodes to complete a join cycle" +
+                            " when starting";
+            updatedSettings
+                    // don't wait too long not to slow down tests
+                    .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s")
+                    .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes);
+        } else if (!usingSingleNodeDiscovery && updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) {
+            throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured");
+        }
+
+        return updatedSettings.build();
     }
 
     /**
      * builds a new node
      *
-     * @param nodeId                the node internal id (see {@link NodeAndClient#nodeAndClientId()}
+     * @param nodeId                    node ordinal
-     * @param seed                  the node's random seed
      * @param settings                  the settings to use
      * @param reuseExisting             if a node with the same name is already part of {@link #nodes}, no new node will be built and
      *                                  the method will return the existing one
-     * @param defaultMinMasterNodes min_master_nodes value to use if min_master_nodes is auto managed
      * @param onTransportServiceStarted callback to run when transport service is started
      */
-    private NodeAndClient buildNode(int nodeId, long seed, Settings settings,
+    private NodeAndClient buildNode(int nodeId, Settings settings,
-                                    boolean reuseExisting, int defaultMinMasterNodes, Runnable onTransportServiceStarted) {
+                                    boolean reuseExisting, Runnable onTransportServiceStarted) {
         assert Thread.holdsLock(this);
         ensureOpen();
-        settings = getSettings(nodeId, seed, settings);
         Collection<Class<? extends Plugin>> plugins = getPlugins();
-        String name = buildNodeName(nodeId, settings);
+        String name = settings.get("node.name");
+
         if (reuseExisting && nodes.containsKey(name)) {
             onTransportServiceStarted.run(); // reusing an existing node implies its transport service already started
             return nodes.get(name);
@@ -635,34 +655,14 @@ public final class InternalTestCluster extends TestCluster {
             assert reuseExisting == true || nodes.containsKey(name) == false :
                     "node name [" + name + "] already exists but not allowed to use it";
         }
-        Settings.Builder finalSettings = Settings.builder()
-            .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home
-            .put(settings)
-            .put("node.name", name)
-            .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);
 
-        final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(finalSettings.build()).equals("single-node");
+        SecureSettings secureSettings = Settings.builder().put(settings).getSecureSettings();
-        if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) {
-            assert finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null :
-                "min master nodes may not be set when auto managed";
-            assert finalSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null :
-                "automatically managing min master nodes require nodes to complete a join cycle" +
-                    " when starting";
-            finalSettings
-                // don't wait too long not to slow down tests
-                .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s")
-                .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes);
-        } else if (!usingSingleNodeDiscovery && finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) {
-            throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured");
-        }
-        SecureSettings secureSettings = finalSettings.getSecureSettings();
         if (secureSettings instanceof MockSecureSettings) {
             // we clone this here since in the case of a node restart we might need it again
             secureSettings = ((MockSecureSettings) secureSettings).clone();
         }
-        final Settings nodeSettings = finalSettings.build();
         MockNode node = new MockNode(
-                nodeSettings,
+                settings,
                 plugins,
                 nodeConfigurationSource.nodeConfigPath(nodeId),
                 forbidPrivateIndexSettings);
@@ -677,13 +677,15 @@ public final class InternalTestCluster extends TestCluster {
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
-        return new NodeAndClient(name, node, nodeSettings, nodeId);
+        return new NodeAndClient(name, node, settings, nodeId);
+    }
+
+    private String getNodePrefix(Settings settings) {
+        return nodePrefix + getRoleSuffix(settings);
     }
 
     private String buildNodeName(int id, Settings settings) {
-        String prefix = nodePrefix;
+        return getNodePrefix(settings) + id;
-        prefix = prefix + getRoleSuffix(settings);
-        return prefix + id;
     }
 
     /**
@@ -1087,49 +1089,52 @@ public final class InternalTestCluster extends TestCluster {
         final List<NodeAndClient> toStartAndPublish = new ArrayList<>(); // we want to start nodes in one go due to min master nodes
         final Runnable onTransportServiceStarted = () -> rebuildUnicastHostFiles(toStartAndPublish);
 
-        final int bootstrapNodeIndex;
+        final List<Settings> settings = new ArrayList<>();
+
+        for (int i = 0; i < numSharedDedicatedMasterNodes; i++) {
+            final Settings.Builder extraSettings = Settings.builder();
+            extraSettings.put(Node.NODE_MASTER_SETTING.getKey(), true);
+            extraSettings.put(Node.NODE_DATA_SETTING.getKey(), false);
+            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes));
+        }
+        for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) {
+            final Settings.Builder extraSettings = Settings.builder();
+            if (numSharedDedicatedMasterNodes > 0) {
+                // if we don't have dedicated master nodes, keep things default
+                extraSettings.put(Node.NODE_MASTER_SETTING.getKey(), false).build();
+                extraSettings.put(Node.NODE_DATA_SETTING.getKey(), true).build();
+            }
+            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes));
+        }
+        for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes;
+             i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
+            final Builder extraSettings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)
+                .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
+            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes));
+        }
+
+        int bootstrapNodeIndex = -1;
+        final List<String> masterNodeNames = settings.stream()
+                .filter(Node.NODE_MASTER_SETTING::get)
+                .map(Node.NODE_NAME_SETTING::get)
+                .collect(Collectors.toList());
+
         if (prevNodeCount == 0 && autoManageMinMasterNodes) {
             if (numSharedDedicatedMasterNodes > 0) {
                 bootstrapNodeIndex = RandomNumbers.randomIntBetween(random, 0, numSharedDedicatedMasterNodes - 1);
             } else if (numSharedDataNodes > 0) {
                 bootstrapNodeIndex = RandomNumbers.randomIntBetween(random, 0, numSharedDataNodes - 1);
-            } else {
-                bootstrapNodeIndex = -1;
             }
-        } else {
-            bootstrapNodeIndex = -1;
         }
 
-        for (int i = 0; i < numSharedDedicatedMasterNodes; i++) {
+        final List<Settings> updatedSettings = nodeConfigurationSource.addExtraClusterBootstrapSettings(settings);
-            final Settings.Builder settings = Settings.builder();
+
-            settings.put(Node.NODE_MASTER_SETTING.getKey(), true);
+        for (int i = 0; i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
-            settings.put(Node.NODE_DATA_SETTING.getKey(), false);
+            Settings nodeSettings = updatedSettings.get(i);
             if (i == bootstrapNodeIndex) {
-                settings.put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), numSharedDedicatedMasterNodes);
+                nodeSettings = Settings.builder().putList(INITIAL_MASTER_NODES_SETTING.getKey(), masterNodeNames).put(nodeSettings).build();
             }
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes,
+            final NodeAndClient nodeAndClient = buildNode(i, nodeSettings, true, onTransportServiceStarted);
-                onTransportServiceStarted);
-            toStartAndPublish.add(nodeAndClient);
-        }
-        for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) {
-            final Settings.Builder settings = Settings.builder();
-            if (numSharedDedicatedMasterNodes > 0) {
-                // if we don't have dedicated master nodes, keep things default
-                settings.put(Node.NODE_MASTER_SETTING.getKey(), false).build();
-                settings.put(Node.NODE_DATA_SETTING.getKey(), true).build();
-            } else if (i == bootstrapNodeIndex) {
-                settings.put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), numSharedDataNodes);
-            }
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes,
-                onTransportServiceStarted);
-            toStartAndPublish.add(nodeAndClient);
-        }
-        for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes;
-             i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
-            final Builder settings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)
-                .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes,
-                onTransportServiceStarted);
             toStartAndPublish.add(nodeAndClient);
         }
 
@@ -1948,8 +1953,8 @@ public final class InternalTestCluster extends TestCluster {
     /**
      * Starts multiple nodes with the given settings and returns their names
      */
-    public synchronized List<String> startNodes(Settings... settings) {
+    public synchronized List<String> startNodes(Settings... extraSettings) {
-        final int newMasterCount = Math.toIntExact(Stream.of(settings).filter(Node.NODE_MASTER_SETTING::get).count());
+        final int newMasterCount = Math.toIntExact(Stream.of(extraSettings).filter(Node.NODE_MASTER_SETTING::get).count());
         final int defaultMinMasterNodes;
         if (autoManageMinMasterNodes) {
             defaultMinMasterNodes = getMinMasterNodes(getMasterNodesCount() + newMasterCount);
@@ -1958,19 +1963,39 @@ public final class InternalTestCluster extends TestCluster {
         }
         final List<NodeAndClient> nodes = new ArrayList<>();
         final int prevMasterCount = getMasterNodesCount();
-        int bootstrapMasterNodeIndex = prevMasterCount == 0 && autoManageMinMasterNodes && newMasterCount > 0 && Arrays.stream(settings)
+        int bootstrapMasterNodeIndex =
+                prevMasterCount == 0 && autoManageMinMasterNodes && newMasterCount > 0 && Arrays.stream(extraSettings)
             .allMatch(s -> Node.NODE_MASTER_SETTING.get(s) == false || TestZenDiscovery.USE_ZEN2.get(s) == true)
             ? RandomNumbers.randomIntBetween(random, 0, newMasterCount - 1) : -1;
 
-        for (Settings nodeSettings : settings) {
+        final int numOfNodes = extraSettings.length;
+        final int firstNodeId = nextNodeId.getAndIncrement();
+        final List<Settings> settings = new ArrayList<>();
+        for (int i = 0; i < numOfNodes; i++) {
+            settings.add(getNodeSettings(firstNodeId + i, random.nextLong(), extraSettings[i], defaultMinMasterNodes));
+        }
+        nextNodeId.set(firstNodeId + numOfNodes);
+
+        final List<String> initialMasterNodes = settings.stream()
+                .filter(Node.NODE_MASTER_SETTING::get)
+                .map(Node.NODE_NAME_SETTING::get)
+                .collect(Collectors.toList());
+
+        final List<Settings> updatedSettings = nodeConfigurationSource.addExtraClusterBootstrapSettings(settings);
+
+        for (int i = 0; i < numOfNodes; i++) {
+            final Settings nodeSettings = updatedSettings.get(i);
             final Builder builder = Settings.builder();
             if (Node.NODE_MASTER_SETTING.get(nodeSettings)) {
                 if (bootstrapMasterNodeIndex == 0) {
-                    builder.put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), newMasterCount);
+                    builder.putList(INITIAL_MASTER_NODES_SETTING.getKey(), initialMasterNodes);
                 }
                 bootstrapMasterNodeIndex -= 1;
             }
-            nodes.add(buildNode(builder.put(nodeSettings).build(), defaultMinMasterNodes, () -> rebuildUnicastHostFiles(nodes)));
+
+            final NodeAndClient nodeAndClient =
+                    buildNode(firstNodeId + i, builder.put(nodeSettings).build(), false, () -> rebuildUnicastHostFiles(nodes));
+            nodes.add(nodeAndClient);
         }
         startAndPublishNodesAndClients(nodes);
         if (autoManageMinMasterNodes) {

test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java

@@ -24,6 +24,7 @@ import org.elasticsearch.plugins.Plugin;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 public abstract class NodeConfigurationSource {
 
@@ -51,6 +52,10 @@ public abstract class NodeConfigurationSource {
 
     public abstract Path nodeConfigPath(int nodeOrdinal);
 
+    public List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
+        return allNodesSettings;
+    }
+
     /** Returns plugins that should be loaded on the node */
     public Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.emptyList();