From ca92d74e7e7102ffe52a9b2d6db3c89696b6dc67 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 20 Dec 2018 15:20:33 +0100 Subject: [PATCH] [Zen2] Change unsafe bootstrap nodes count to nodes list in tests (#36559) This commit modifies ESSingleNodeTestCase and ESIntegTestCase and several concrete test classes to use node names when bootstrapping the cluster. Today ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING setting is used to bootstrap clusters in tests. Instead, we want to use ClusterBootrstapService.INITIAL_MASTER_NODES_SETTING and get rid of the former setting eventually. There were two main problems when refactoring InternalTestCluster: 1. Nodes are created one-by-one in buildNode method. And node.name is created in this method as well. It's not suitable for bootstrapping, because we need to have the names of all master eligible nodes in advance, before creating the node with bootstrapping configuration set. We address this issue by separating buildNode into two methods: getNodeSettings and buildNode. We first iterate over all nodes to get nodes settings, then change the setting for the bootstrapping node and then proceed with building the node. 2. If autoManageMinMasterNodes = false, there is no way for the test to set the list of bootstrapping nodes because node names are not known in advance. This problem is solved by adding updateNodesSettings method to NodeConfigurationSource and ESIntegTestCase (which could be overridden by concrete integration test class). Once we have the list of settings for all nodes, the integration test class is allowed to update it. In our case, we update the ClusterBootrstapService.INITIAL_MASTER_NODES_SETTING setting. --- .../rest/discovery/Zen2RestApiIT.java | 24 +- .../gateway/RecoverAfterNodesIT.java | 29 ++- .../elasticsearch/test/ESIntegTestCase.java | 18 ++ .../test/ESSingleNodeTestCase.java | 9 +- .../test/InternalTestCluster.java | 205 ++++++++++-------- .../test/NodeConfigurationSource.java | 5 + 6 files changed, 186 insertions(+), 104 deletions(-) diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java index 7695c2bb596..cfa5a3f6d79 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java @@ -43,8 +43,10 @@ import org.elasticsearch.test.discovery.TestZenDiscovery; import org.hamcrest.Matchers; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.hamcrest.core.Is.is; @@ -60,13 +62,27 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase { .put(TestZenDiscovery.USE_ZEN2.getKey(), true) .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE); - if (nodeOrdinal == 0) { - builder.put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 2); - } - return builder.build(); } + @Override + protected List addExtraClusterBootstrapSettings(List allNodesSettings) { + final Settings firstNodeSettings = allNodesSettings.get(0); + final List otherNodesSettings = allNodesSettings.subList(1, allNodesSettings.size()); + final List masterNodeNames = allNodesSettings.stream() + .filter(org.elasticsearch.node.Node.NODE_MASTER_SETTING::get) + .map(org.elasticsearch.node.Node.NODE_NAME_SETTING::get) + .collect(Collectors.toList()); + final List updatedSettings = new ArrayList<>(); + + updatedSettings.add(Settings.builder().put(firstNodeSettings) + .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), masterNodeNames) + .build()); + updatedSettings.addAll(otherNodesSettings); + + return updatedSettings; + } + @Override protected boolean addMockHttpTransport() { return false; // enable http diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index 94aa403c3d3..e97f69b6d49 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -31,6 +31,8 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import static org.hamcrest.Matchers.equalTo; @@ -40,6 +42,23 @@ import static org.hamcrest.Matchers.hasItem; public class RecoverAfterNodesIT extends ESIntegTestCase { private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10); + @Override + protected List addExtraClusterBootstrapSettings(List allNodesSettings) { + if (internalCluster().numDataAndMasterNodes() == 0) { + final Settings firstNodeSettings = allNodesSettings.get(0); + final List otherNodesSettings = allNodesSettings.subList(1, allNodesSettings.size()); + + final List updatedSettings = new ArrayList<>(); + updatedSettings.add(Settings.builder().put(firstNodeSettings) + .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), + Node.NODE_NAME_SETTING.get(firstNodeSettings)).build()); + updatedSettings.addAll(otherNodesSettings); + + return updatedSettings; + } + return super.addExtraClusterBootstrapSettings(allNodesSettings); + } + public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException { long start = System.currentTimeMillis(); Set blocks; @@ -60,9 +79,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase { public void testRecoverAfterNodes() throws Exception { logger.info("--> start node (1)"); - Client clientNode1 = startNode(Settings.builder() - .put("gateway.recover_after_nodes", 3) - .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1); + Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -89,8 +106,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase { logger.info("--> start master_node (1)"); Client master1 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true) - .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -136,8 +152,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase { Client master1 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true) - .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 1dc3c375ad8..e17377b500d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1947,6 +1947,11 @@ public abstract class ESIntegTestCase extends ESTestCase { .put(ESIntegTestCase.this.nodeSettings(nodeOrdinal)).build(); } + @Override + public List addExtraClusterBootstrapSettings(List allNodesSettings) { + return ESIntegTestCase.this.addExtraClusterBootstrapSettings(allNodesSettings); + } + @Override public Path nodeConfigPath(int nodeOrdinal) { return ESIntegTestCase.this.nodeConfigPath(nodeOrdinal); @@ -1975,6 +1980,19 @@ public abstract class ESIntegTestCase extends ESTestCase { }; } + /** + * This method is called before starting a collection of nodes. + * At this point the test has a holistic view on all nodes settings and might perform settings adjustments as needed. + * For instance, the test could retrieve master node names and fill in + * {@link org.elasticsearch.cluster.coordination.ClusterBootstrapService#INITIAL_MASTER_NODES_SETTING} setting. + * + * @param allNodesSettings list of node settings before update + * @return list of node settings after update + */ + protected List addExtraClusterBootstrapSettings(List allNodesSettings) { + return allNodesSettings; + } + /** * Iff this returns true mock transport implementations are used for the test runs. Otherwise not mock transport impls are used. * The default is {@code true}. diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 5fbb5da14db..32aa1c2107d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -62,7 +62,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING; +import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; import static org.elasticsearch.discovery.zen.SettingsBasedHostsProvider.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -179,6 +179,8 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { private Node newNode() { final Path tempDir = createTempDir(); + final String nodeName = nodeSettings().get(Node.NODE_NAME_SETTING.getKey(), "node_s_0"); + Settings settings = Settings.builder() .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", random().nextLong())) .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) @@ -186,7 +188,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { // TODO: use a consistent data path for custom paths // This needs to tie into the ESIntegTestCase#indexSettings() method .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent()) - .put("node.name", "node_s_0") + .put(Node.NODE_NAME_SETTING.getKey(), nodeName) .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "1000/1m") .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created .put("transport.type", getTestTransportType()) @@ -201,9 +203,10 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { // turn it off for these tests. .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), false) .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()) // empty list disables a port scan for other nodes - .put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1) + .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeName) .put(nodeSettings()) // allow test cases to provide their own settings or override these .build(); + Collection> plugins = getPlugins(); if (plugins.contains(getTestTransportPlugin()) == false) { plugins = new ArrayList<>(plugins); diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index db719389665..e6e11dacb74 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -148,7 +148,7 @@ import java.util.stream.Stream; import static java.util.Collections.emptyList; import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY; import static org.apache.lucene.util.LuceneTestCase.rarely; -import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING; +import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.discovery.DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING; @@ -525,10 +525,15 @@ public final class InternalTestCluster extends TestCluster { if (randomNodeAndClient != null) { return randomNodeAndClient; } - final int ord = nextNodeId.getAndIncrement(); final Runnable onTransportServiceStarted = () -> {}; // do not create unicast host file for this one node. - final Settings settings = Settings.builder().put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1).build(); - final NodeAndClient buildNode = buildNode(ord, random.nextLong(), settings, false, 1, onTransportServiceStarted); + + final int nodeId = nextNodeId.getAndIncrement(); + final Settings settings = getNodeSettings(nodeId, random.nextLong(), Settings.EMPTY, 1); + final Settings nodeSettings = Settings.builder() + .putList(INITIAL_MASTER_NODES_SETTING.getKey(), Node.NODE_NAME_SETTING.get(settings)) + .put(settings) + .build(); + final NodeAndClient buildNode = buildNode(nodeId, nodeSettings, false, onTransportServiceStarted); assert nodes.isEmpty(); buildNode.startNode(); publishNode(buildNode); @@ -598,71 +603,66 @@ public final class InternalTestCluster extends TestCluster { } } - /** - * builds a new node given the settings. - * - * @param settings the settings to use - * @param defaultMinMasterNodes min_master_nodes value to use if min_master_nodes is auto managed - * @param onTransportServiceStarted callback to run when transport service is started - */ - private NodeAndClient buildNode(Settings settings, int defaultMinMasterNodes, Runnable onTransportServiceStarted) { - int ord = nextNodeId.getAndIncrement(); - return buildNode(ord, random.nextLong(), settings, false, defaultMinMasterNodes, onTransportServiceStarted); + private Settings getNodeSettings(final int nodeId, final long seed, final Settings extraSettings, final int defaultMinMasterNodes) { + final Settings settings = getSettings(nodeId, seed, extraSettings); + + final String name = buildNodeName(nodeId, settings); + + final Settings.Builder updatedSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home + .put(settings) + .put("node.name", name) + .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed); + + final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()).equals("single-node"); + if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) { + assert updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null : + "min master nodes may not be set when auto managed"; + assert updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null : + "automatically managing min master nodes require nodes to complete a join cycle" + + " when starting"; + updatedSettings + // don't wait too long not to slow down tests + .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s") + .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes); + } else if (!usingSingleNodeDiscovery && updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) { + throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured"); + } + + return updatedSettings.build(); } /** * builds a new node * - * @param nodeId the node internal id (see {@link NodeAndClient#nodeAndClientId()} - * @param seed the node's random seed - * @param settings the settings to use - * @param reuseExisting if a node with the same name is already part of {@link #nodes}, no new node will be built and - * the method will return the existing one - * @param defaultMinMasterNodes min_master_nodes value to use if min_master_nodes is auto managed + * @param nodeId node ordinal + * @param settings the settings to use + * @param reuseExisting if a node with the same name is already part of {@link #nodes}, no new node will be built and + * the method will return the existing one * @param onTransportServiceStarted callback to run when transport service is started */ - private NodeAndClient buildNode(int nodeId, long seed, Settings settings, - boolean reuseExisting, int defaultMinMasterNodes, Runnable onTransportServiceStarted) { + private NodeAndClient buildNode(int nodeId, Settings settings, + boolean reuseExisting, Runnable onTransportServiceStarted) { assert Thread.holdsLock(this); ensureOpen(); - settings = getSettings(nodeId, seed, settings); Collection> plugins = getPlugins(); - String name = buildNodeName(nodeId, settings); + String name = settings.get("node.name"); + if (reuseExisting && nodes.containsKey(name)) { onTransportServiceStarted.run(); // reusing an existing node implies its transport service already started return nodes.get(name); } else { assert reuseExisting == true || nodes.containsKey(name) == false : - "node name [" + name + "] already exists but not allowed to use it"; + "node name [" + name + "] already exists but not allowed to use it"; } - Settings.Builder finalSettings = Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home - .put(settings) - .put("node.name", name) - .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed); - final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(finalSettings.build()).equals("single-node"); - if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) { - assert finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null : - "min master nodes may not be set when auto managed"; - assert finalSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null : - "automatically managing min master nodes require nodes to complete a join cycle" + - " when starting"; - finalSettings - // don't wait too long not to slow down tests - .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s") - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes); - } else if (!usingSingleNodeDiscovery && finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) { - throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured"); - } - SecureSettings secureSettings = finalSettings.getSecureSettings(); + SecureSettings secureSettings = Settings.builder().put(settings).getSecureSettings(); if (secureSettings instanceof MockSecureSettings) { // we clone this here since in the case of a node restart we might need it again secureSettings = ((MockSecureSettings) secureSettings).clone(); } - final Settings nodeSettings = finalSettings.build(); MockNode node = new MockNode( - nodeSettings, + settings, plugins, nodeConfigurationSource.nodeConfigPath(nodeId), forbidPrivateIndexSettings); @@ -677,13 +677,15 @@ public final class InternalTestCluster extends TestCluster { } catch (IOException e) { throw new UncheckedIOException(e); } - return new NodeAndClient(name, node, nodeSettings, nodeId); + return new NodeAndClient(name, node, settings, nodeId); + } + + private String getNodePrefix(Settings settings) { + return nodePrefix + getRoleSuffix(settings); } private String buildNodeName(int id, Settings settings) { - String prefix = nodePrefix; - prefix = prefix + getRoleSuffix(settings); - return prefix + id; + return getNodePrefix(settings) + id; } /** @@ -1087,49 +1089,52 @@ public final class InternalTestCluster extends TestCluster { final List toStartAndPublish = new ArrayList<>(); // we want to start nodes in one go due to min master nodes final Runnable onTransportServiceStarted = () -> rebuildUnicastHostFiles(toStartAndPublish); - final int bootstrapNodeIndex; + final List settings = new ArrayList<>(); + + for (int i = 0; i < numSharedDedicatedMasterNodes; i++) { + final Settings.Builder extraSettings = Settings.builder(); + extraSettings.put(Node.NODE_MASTER_SETTING.getKey(), true); + extraSettings.put(Node.NODE_DATA_SETTING.getKey(), false); + settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes)); + } + for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) { + final Settings.Builder extraSettings = Settings.builder(); + if (numSharedDedicatedMasterNodes > 0) { + // if we don't have dedicated master nodes, keep things default + extraSettings.put(Node.NODE_MASTER_SETTING.getKey(), false).build(); + extraSettings.put(Node.NODE_DATA_SETTING.getKey(), true).build(); + } + settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes)); + } + for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes; + i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) { + final Builder extraSettings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false) + .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false); + settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes)); + } + + int bootstrapNodeIndex = -1; + final List masterNodeNames = settings.stream() + .filter(Node.NODE_MASTER_SETTING::get) + .map(Node.NODE_NAME_SETTING::get) + .collect(Collectors.toList()); + if (prevNodeCount == 0 && autoManageMinMasterNodes) { if (numSharedDedicatedMasterNodes > 0) { bootstrapNodeIndex = RandomNumbers.randomIntBetween(random, 0, numSharedDedicatedMasterNodes - 1); } else if (numSharedDataNodes > 0) { bootstrapNodeIndex = RandomNumbers.randomIntBetween(random, 0, numSharedDataNodes - 1); - } else { - bootstrapNodeIndex = -1; } - } else { - bootstrapNodeIndex = -1; } - for (int i = 0; i < numSharedDedicatedMasterNodes; i++) { - final Settings.Builder settings = Settings.builder(); - settings.put(Node.NODE_MASTER_SETTING.getKey(), true); - settings.put(Node.NODE_DATA_SETTING.getKey(), false); + final List updatedSettings = nodeConfigurationSource.addExtraClusterBootstrapSettings(settings); + + for (int i = 0; i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) { + Settings nodeSettings = updatedSettings.get(i); if (i == bootstrapNodeIndex) { - settings.put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), numSharedDedicatedMasterNodes); + nodeSettings = Settings.builder().putList(INITIAL_MASTER_NODES_SETTING.getKey(), masterNodeNames).put(nodeSettings).build(); } - NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes, - onTransportServiceStarted); - toStartAndPublish.add(nodeAndClient); - } - for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) { - final Settings.Builder settings = Settings.builder(); - if (numSharedDedicatedMasterNodes > 0) { - // if we don't have dedicated master nodes, keep things default - settings.put(Node.NODE_MASTER_SETTING.getKey(), false).build(); - settings.put(Node.NODE_DATA_SETTING.getKey(), true).build(); - } else if (i == bootstrapNodeIndex) { - settings.put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), numSharedDataNodes); - } - NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes, - onTransportServiceStarted); - toStartAndPublish.add(nodeAndClient); - } - for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes; - i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) { - final Builder settings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false) - .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false); - NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes, - onTransportServiceStarted); + final NodeAndClient nodeAndClient = buildNode(i, nodeSettings, true, onTransportServiceStarted); toStartAndPublish.add(nodeAndClient); } @@ -1948,8 +1953,8 @@ public final class InternalTestCluster extends TestCluster { /** * Starts multiple nodes with the given settings and returns their names */ - public synchronized List startNodes(Settings... settings) { - final int newMasterCount = Math.toIntExact(Stream.of(settings).filter(Node.NODE_MASTER_SETTING::get).count()); + public synchronized List startNodes(Settings... extraSettings) { + final int newMasterCount = Math.toIntExact(Stream.of(extraSettings).filter(Node.NODE_MASTER_SETTING::get).count()); final int defaultMinMasterNodes; if (autoManageMinMasterNodes) { defaultMinMasterNodes = getMinMasterNodes(getMasterNodesCount() + newMasterCount); @@ -1958,19 +1963,39 @@ public final class InternalTestCluster extends TestCluster { } final List nodes = new ArrayList<>(); final int prevMasterCount = getMasterNodesCount(); - int bootstrapMasterNodeIndex = prevMasterCount == 0 && autoManageMinMasterNodes && newMasterCount > 0 && Arrays.stream(settings) + int bootstrapMasterNodeIndex = + prevMasterCount == 0 && autoManageMinMasterNodes && newMasterCount > 0 && Arrays.stream(extraSettings) .allMatch(s -> Node.NODE_MASTER_SETTING.get(s) == false || TestZenDiscovery.USE_ZEN2.get(s) == true) ? RandomNumbers.randomIntBetween(random, 0, newMasterCount - 1) : -1; - for (Settings nodeSettings : settings) { + final int numOfNodes = extraSettings.length; + final int firstNodeId = nextNodeId.getAndIncrement(); + final List settings = new ArrayList<>(); + for (int i = 0; i < numOfNodes; i++) { + settings.add(getNodeSettings(firstNodeId + i, random.nextLong(), extraSettings[i], defaultMinMasterNodes)); + } + nextNodeId.set(firstNodeId + numOfNodes); + + final List initialMasterNodes = settings.stream() + .filter(Node.NODE_MASTER_SETTING::get) + .map(Node.NODE_NAME_SETTING::get) + .collect(Collectors.toList()); + + final List updatedSettings = nodeConfigurationSource.addExtraClusterBootstrapSettings(settings); + + for (int i = 0; i < numOfNodes; i++) { + final Settings nodeSettings = updatedSettings.get(i); final Builder builder = Settings.builder(); if (Node.NODE_MASTER_SETTING.get(nodeSettings)) { if (bootstrapMasterNodeIndex == 0) { - builder.put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), newMasterCount); + builder.putList(INITIAL_MASTER_NODES_SETTING.getKey(), initialMasterNodes); } bootstrapMasterNodeIndex -= 1; } - nodes.add(buildNode(builder.put(nodeSettings).build(), defaultMinMasterNodes, () -> rebuildUnicastHostFiles(nodes))); + + final NodeAndClient nodeAndClient = + buildNode(firstNodeId + i, builder.put(nodeSettings).build(), false, () -> rebuildUnicastHostFiles(nodes)); + nodes.add(nodeAndClient); } startAndPublishNodesAndClients(nodes); if (autoManageMinMasterNodes) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java b/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java index 60c69bbd6c6..5ed21d64c68 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java +++ b/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java @@ -24,6 +24,7 @@ import org.elasticsearch.plugins.Plugin; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; +import java.util.List; public abstract class NodeConfigurationSource { @@ -51,6 +52,10 @@ public abstract class NodeConfigurationSource { public abstract Path nodeConfigPath(int nodeOrdinal); + public List addExtraClusterBootstrapSettings(List allNodesSettings) { + return allNodesSettings; + } + /** Returns plugins that should be loaded on the node */ public Collection> nodePlugins() { return Collections.emptyList();