[Zen2] Remove initial master node count setting (#37150)

The `cluster.unsafe_initial_master_node_count` setting was introduced as a
temporary measure while the design of `cluster.initial_master_nodes` was being
finalised. This commit removes this temporary setting, replacing it with usages
of `cluster.initial_master_nodes` where appropriate.

server/src/main/java/org/elasticsearch/action/admin/cluster/bootstrap/GetDiscoveredNodesRequest.java

@@ -35,8 +35,6 @@ import java.util.List;
  */
 public class GetDiscoveredNodesRequest extends ActionRequest {
 
-    private int waitForNodes = 1;
-
     @Nullable // if the request should wait indefinitely
     private TimeValue timeout = TimeValue.timeValueSeconds(30);
 
@@ -47,35 +45,10 @@ public class GetDiscoveredNodesRequest extends ActionRequest {
 
     public GetDiscoveredNodesRequest(StreamInput in) throws IOException {
         super(in);
-        waitForNodes = in.readInt();
         timeout = in.readOptionalTimeValue();
         requiredNodes = in.readList(StreamInput::readString);
     }
 
-    /**
-     * Sometimes it is useful only to receive a successful response after discovering a certain number of master-eligible nodes. This
-     * parameter controls this behaviour.
-     *
-     * @param waitForNodes the minimum number of nodes to have discovered before this request will receive a successful response. Must
-     *                     be at least 1, because we always discover the local node.
-     */
-    public void setWaitForNodes(int waitForNodes) {
-        if (waitForNodes < 1) {
-            throw new IllegalArgumentException("always finds at least one node, waiting for [" + waitForNodes + "] is not allowed");
-        }
-        this.waitForNodes = waitForNodes;
-    }
-
-    /**
-     * Sometimes it is useful only to receive a successful response after discovering a certain number of master-eligible nodes. This
-     * parameter controls this behaviour.
-     *
-     * @return the minimum number of nodes to have discovered before this request will receive a successful response.
-     */
-    public int getWaitForNodes() {
-        return waitForNodes;
-    }
-
     /**
      * Sometimes it is useful to wait until enough nodes have been discovered, rather than failing immediately. This parameter controls how
      * long to wait, and defaults to 30s.
@@ -133,7 +106,6 @@ public class GetDiscoveredNodesRequest extends ActionRequest {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeInt(waitForNodes);
         out.writeOptionalTimeValue(timeout);
         out.writeStringList(requiredNodes);
     }
@@ -141,8 +113,7 @@ public class GetDiscoveredNodesRequest extends ActionRequest {
     @Override
     public String toString() {
         return "GetDiscoveredNodesRequest{" +
-            "waitForNodes=" + waitForNodes +
+            "timeout=" + timeout +
-            ", timeout=" + timeout +
             ", requiredNodes=" + requiredNodes + "}";
     }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/bootstrap/TransportGetDiscoveredNodesAction.java

@@ -152,10 +152,6 @@ public class TransportGetDiscoveredNodesAction extends HandledTransportAction<Ge
     }
 
     private static boolean checkWaitRequirements(GetDiscoveredNodesRequest request, Set<DiscoveryNode> nodes) {
-        if (nodes.size() < request.getWaitForNodes()) {
-            return false;
-        }
-
         List<String> requirements = request.getRequiredNodes();
         final Set<DiscoveryNode> selectedNodes = new HashSet<>();
         for (final String requirement : requirements) {

server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java

@@ -54,11 +54,6 @@ public class ClusterBootstrapService {
 
     private static final Logger logger = LogManager.getLogger(ClusterBootstrapService.class);
 
-    // The number of master-eligible nodes which, if discovered, can be used to bootstrap the cluster. This setting is unsafe in the event
-    // that more master nodes are started than expected.
-    public static final Setting<Integer> INITIAL_MASTER_NODE_COUNT_SETTING =
-        Setting.intSetting("cluster.unsafe_initial_master_node_count", 0, 0, Property.NodeScope);
-
     public static final Setting<List<String>> INITIAL_MASTER_NODES_SETTING =
         Setting.listSetting("cluster.initial_master_nodes", Collections.emptyList(), Function.identity(), Property.NodeScope);
 
@@ -66,7 +61,6 @@ public class ClusterBootstrapService {
         Setting.timeSetting("discovery.unconfigured_bootstrap_timeout",
             TimeValue.timeValueSeconds(3), TimeValue.timeValueMillis(1), Property.NodeScope);
 
-    private final int initialMasterNodeCount;
     private final List<String> initialMasterNodes;
     @Nullable
     private final TimeValue unconfiguredBootstrapTimeout;
@@ -74,15 +68,14 @@ public class ClusterBootstrapService {
     private volatile boolean running;
 
     public ClusterBootstrapService(Settings settings, TransportService transportService) {
-        initialMasterNodeCount = INITIAL_MASTER_NODE_COUNT_SETTING.get(settings);
         initialMasterNodes = INITIAL_MASTER_NODES_SETTING.get(settings);
         unconfiguredBootstrapTimeout = discoveryIsConfigured(settings) ? null : UNCONFIGURED_BOOTSTRAP_TIMEOUT_SETTING.get(settings);
         this.transportService = transportService;
     }
 
     public static boolean discoveryIsConfigured(Settings settings) {
-        return Stream.of(DISCOVERY_HOSTS_PROVIDER_SETTING, DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING,
+        return Stream.of(DISCOVERY_HOSTS_PROVIDER_SETTING, DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING, INITIAL_MASTER_NODES_SETTING)
-            INITIAL_MASTER_NODE_COUNT_SETTING, INITIAL_MASTER_NODES_SETTING).anyMatch(s -> s.exists(settings));
+            .anyMatch(s -> s.exists(settings));
     }
 
     public void start() {
@@ -144,17 +137,14 @@ public class ClusterBootstrapService {
                 });
 
             }
-        } else if (initialMasterNodeCount > 0 || initialMasterNodes.isEmpty() == false) {
+        } else if (initialMasterNodes.isEmpty() == false) {
-            logger.debug("unsafely waiting for discovery of [{}] master-eligible nodes", initialMasterNodeCount);
+            logger.debug("waiting for discovery of master-eligible nodes matching {}", initialMasterNodes);
 
             final ThreadContext threadContext = transportService.getThreadPool().getThreadContext();
             try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
                 threadContext.markAsSystemContext();
 
                 final GetDiscoveredNodesRequest request = new GetDiscoveredNodesRequest();
-                if (initialMasterNodeCount > 0) {
-                    request.setWaitForNodes(initialMasterNodeCount);
-                }
                 request.setRequiredNodes(initialMasterNodes);
                 request.setTimeout(null);
                 logger.trace("sending {}", request);
@@ -162,7 +152,6 @@ public class ClusterBootstrapService {
                     new TransportResponseHandler<GetDiscoveredNodesResponse>() {
                         @Override
                         public void handleResponse(GetDiscoveredNodesResponse response) {
-                            assert response.getNodes().size() >= initialMasterNodeCount;
                             assert response.getNodes().stream().allMatch(DiscoveryNode::isMasterNode);
                             logger.debug("discovered {}, starting to bootstrap", response.getNodes());
                             awaitBootstrap(response.getBootstrapConfiguration());

server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java

@@ -42,7 +42,6 @@ import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
-import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING;
 
 public class ClusterFormationFailureHelper {
     private static final Logger logger = LogManager.getLogger(ClusterFormationFailureHelper.class);
@@ -148,23 +147,13 @@ public class ClusterFormationFailureHelper {
 
                 final String bootstrappingDescription;
 
-                if (INITIAL_MASTER_NODE_COUNT_SETTING.get(Settings.EMPTY).equals(INITIAL_MASTER_NODE_COUNT_SETTING.get(settings))
+                if (INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY).equals(INITIAL_MASTER_NODES_SETTING.get(settings))) {
-                    && INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY).equals(INITIAL_MASTER_NODES_SETTING.get(settings))) {
                     bootstrappingDescription = "[" + INITIAL_MASTER_NODES_SETTING.getKey() + "] is empty on this node";
-                } else if (INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY).equals(INITIAL_MASTER_NODES_SETTING.get(settings))) {
+                } else {
-                    bootstrappingDescription = String.format(Locale.ROOT,
-                        "this node must discover at least [%d] master-eligible nodes to bootstrap a cluster",
-                        INITIAL_MASTER_NODE_COUNT_SETTING.get(settings));
-                } else if (INITIAL_MASTER_NODE_COUNT_SETTING.get(settings) <= INITIAL_MASTER_NODES_SETTING.get(settings).size()) {
                     // TODO update this when we can bootstrap on only a quorum of the initial nodes
                     bootstrappingDescription = String.format(Locale.ROOT,
                         "this node must discover master-eligible nodes %s to bootstrap a cluster",
                         INITIAL_MASTER_NODES_SETTING.get(settings));
-                } else {
-                    // TODO update this when we can bootstrap on only a quorum of the initial nodes
-                    bootstrappingDescription = String.format(Locale.ROOT,
-                        "this node must discover at least [%d] master-eligible nodes, including %s, to bootstrap a cluster",
-                        INITIAL_MASTER_NODE_COUNT_SETTING.get(settings), INITIAL_MASTER_NODES_SETTING.get(settings));
                 }
 
                 return String.format(Locale.ROOT,

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -485,7 +485,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     Reconfigurator.CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION,
                     TransportAddVotingConfigExclusionsAction.MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING,
                     ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING,
-                    ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING,
                     ClusterBootstrapService.UNCONFIGURED_BOOTSTRAP_TIMEOUT_SETTING,
                     LagDetector.CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING,
                     DiscoveryUpgradeService.BWC_PING_TIMEOUT_SETTING,

server/src/test/java/org/elasticsearch/action/admin/cluster/bootstrap/GetDiscoveredNodesRequestTests.java

@@ -31,20 +31,6 @@ import static org.hamcrest.core.Is.is;
 
 public class GetDiscoveredNodesRequestTests extends ESTestCase {
 
-    public void testWaitForNodesValidation() {
-        final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
-        assertThat("default value is 1", getDiscoveredNodesRequest.getWaitForNodes(), is(1));
-
-        final int newWaitForNodes = randomIntBetween(1, 10);
-        getDiscoveredNodesRequest.setWaitForNodes(newWaitForNodes);
-        assertThat("value updated", getDiscoveredNodesRequest.getWaitForNodes(), is(newWaitForNodes));
-
-        final IllegalArgumentException exception
-            = expectThrows(IllegalArgumentException.class, () -> getDiscoveredNodesRequest.setWaitForNodes(randomIntBetween(-10, 0)));
-        assertThat(exception.getMessage(), startsWith("always finds at least one node, waiting for "));
-        assertThat(exception.getMessage(), endsWith(" is not allowed"));
-    }
-
     public void testTimeoutValidation() {
         final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
         assertThat("default value is 30s", getDiscoveredNodesRequest.getTimeout(), is(TimeValue.timeValueSeconds(30)));
@@ -65,10 +51,6 @@ public class GetDiscoveredNodesRequestTests extends ESTestCase {
     public void testSerialization() throws IOException {
         final GetDiscoveredNodesRequest originalRequest = new GetDiscoveredNodesRequest();
 
-        if (randomBoolean()) {
-            originalRequest.setWaitForNodes(randomIntBetween(1, 10));
-        }
-
         if (randomBoolean()) {
             originalRequest.setTimeout(TimeValue.parseTimeValue(randomTimeValue(), "timeout"));
         } else if (randomBoolean()) {
@@ -77,7 +59,6 @@ public class GetDiscoveredNodesRequestTests extends ESTestCase {
 
         final GetDiscoveredNodesRequest deserialized = copyWriteable(originalRequest, writableRegistry(), GetDiscoveredNodesRequest::new);
 
-        assertThat(deserialized.getWaitForNodes(), equalTo(originalRequest.getWaitForNodes()));
         assertThat(deserialized.getTimeout(), equalTo(originalRequest.getTimeout()));
     }
 }

server/src/test/java/org/elasticsearch/action/admin/cluster/bootstrap/TransportGetDiscoveredNodesActionTests.java

@@ -72,6 +72,7 @@ import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING;
 import static org.elasticsearch.discovery.PeerFinder.REQUEST_PEERS_ACTION_NAME;
 import static org.elasticsearch.transport.TransportService.HANDSHAKE_ACTION_NAME;
@@ -204,8 +205,8 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
 
         {
             final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
-            getDiscoveredNodesRequest.setWaitForNodes(2);
             getDiscoveredNodesRequest.setTimeout(null);
+            getDiscoveredNodesRequest.setRequiredNodes(singletonList("not-a-node"));
             transportService.sendRequest(localNode, GetDiscoveredNodesAction.NAME, getDiscoveredNodesRequest, new ResponseHandler() {
                 @Override
                 public void handleResponse(GetDiscoveredNodesResponse response) {
@@ -221,8 +222,8 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
 
         {
             final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
-            getDiscoveredNodesRequest.setWaitForNodes(2);
             getDiscoveredNodesRequest.setTimeout(TimeValue.ZERO);
+            getDiscoveredNodesRequest.setRequiredNodes(singletonList("not-a-node"));
 
             final CountDownLatch countDownLatch = new CountDownLatch(1);
             transportService.sendRequest(localNode, GetDiscoveredNodesAction.NAME, getDiscoveredNodesRequest, new ResponseHandler() {
@@ -254,7 +255,6 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
 
         final CountDownLatch countDownLatch = new CountDownLatch(1);
         final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
-        getDiscoveredNodesRequest.setWaitForNodes(2);
         getDiscoveredNodesRequest.setTimeout(null);
         transportService.sendRequest(localNode, GetDiscoveredNodesAction.NAME, getDiscoveredNodesRequest, new ResponseHandler() {
             @Override
@@ -283,8 +283,8 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
 
         final CountDownLatch countDownLatch = new CountDownLatch(1);
         final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
-        getDiscoveredNodesRequest.setWaitForNodes(3);
         getDiscoveredNodesRequest.setTimeout(null);
+        getDiscoveredNodesRequest.setRequiredNodes(singletonList("not-a-node"));
         transportService.sendRequest(localNode, GetDiscoveredNodesAction.NAME, getDiscoveredNodesRequest, new ResponseHandler() {
             @Override
             public void handleResponse(GetDiscoveredNodesResponse response) {
@@ -342,7 +342,6 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
     public void testGetsDiscoveredNodesWithZeroTimeout() throws InterruptedException {
         setupGetDiscoveredNodesAction();
         final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
-        getDiscoveredNodesRequest.setWaitForNodes(2);
         getDiscoveredNodesRequest.setTimeout(TimeValue.ZERO);
         assertWaitConditionMet(getDiscoveredNodesRequest);
     }
@@ -377,7 +376,6 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
         final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
         String name = localNode.getName();
         getDiscoveredNodesRequest.setRequiredNodes(Arrays.asList(name, name));
-        getDiscoveredNodesRequest.setWaitForNodes(1);
         getDiscoveredNodesRequest.setTimeout(TimeValue.ZERO);
         assertWaitConditionFailedOnDuplicate(getDiscoveredNodesRequest, "[" + localNode + "] matches [" + name + ", " + name + ']');
     }
@@ -396,7 +394,6 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
         final CountDownLatch latch = new CountDownLatch(1);
         final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
         getDiscoveredNodesRequest.setRequiredNodes(Arrays.asList(localNode.getAddress().toString(), "_missing"));
-        getDiscoveredNodesRequest.setWaitForNodes(1);
         getDiscoveredNodesRequest.setTimeout(TimeValue.ZERO);
         transportService.sendRequest(localNode, GetDiscoveredNodesAction.NAME, getDiscoveredNodesRequest, new ResponseHandler() {
             @Override
@@ -423,8 +420,7 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
 
         final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
         final String ip = localNode.getAddress().getAddress();
-        getDiscoveredNodesRequest.setRequiredNodes(Collections.singletonList(ip));
+        getDiscoveredNodesRequest.setRequiredNodes(Arrays.asList(ip, "not-a-node"));
-        getDiscoveredNodesRequest.setWaitForNodes(2);
 
         final CountDownLatch countDownLatch = new CountDownLatch(1);
         transportService.sendRequest(localNode, GetDiscoveredNodesAction.NAME, getDiscoveredNodesRequest, new ResponseHandler() {
@@ -480,7 +476,7 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
         executeRequestPeersAction();
 
         final GetDiscoveredNodesRequest getDiscoveredNodesRequest = new GetDiscoveredNodesRequest();
-        getDiscoveredNodesRequest.setWaitForNodes(2);
+        getDiscoveredNodesRequest.setRequiredNodes(Arrays.asList(localNode.getName(), otherNode.getName()));
         assertWaitConditionMet(getDiscoveredNodesRequest);
     }
 

server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java

@@ -729,7 +729,6 @@ public class BootstrapChecksTests extends AbstractBootstrapCheckTestCase {
 
         ensureChecksPass.accept(Settings.builder().putList(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()));
         ensureChecksPass.accept(Settings.builder().putList(SettingsBasedHostsProvider.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()));
-        ensureChecksPass.accept(Settings.builder().put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 0));
         ensureChecksPass.accept(Settings.builder().putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey()));
     }
 }

server/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java

@@ -28,24 +28,28 @@ import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.coordination.ClusterBootstrapService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
-import org.elasticsearch.discovery.zen.ElectMasterService;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.InternalTestCluster.RestartCallback;
+import org.elasticsearch.test.disruption.NetworkDisruption;
+import org.elasticsearch.test.disruption.NetworkDisruption.IsolateAllNodes;
+import org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect;
+import org.elasticsearch.test.transport.MockTransportService;
 
-import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -53,7 +57,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThro
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
 public class NoMasterNodeIT extends ESIntegTestCase {
 
     @Override
@@ -61,106 +65,105 @@ public class NoMasterNodeIT extends ESIntegTestCase {
         return 2;
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(MockTransportService.TestPlugin.class);
+    }
+
     public void testNoMasterActions() throws Exception {
         Settings settings = Settings.builder()
             .put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), true)
-            .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
             .put(DiscoverySettings.NO_MASTER_BLOCK_SETTING.getKey(), "all")
-            .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 3)
             .build();
 
         final TimeValue timeout = TimeValue.timeValueMillis(10);
 
-        internalCluster().startNodes(3, settings);
+        final List<String> nodes = internalCluster().startNodes(3, settings);
 
         createIndex("test");
         client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
-        internalCluster().stopRandomDataNode();
 
-        internalCluster().restartRandomDataNode(new RestartCallback() {
+        final NetworkDisruption disruptionScheme
-            @Override
+            = new NetworkDisruption(new IsolateAllNodes(new HashSet<>(nodes)), new NetworkDisconnect());
-            public Settings onNodeStopped(String nodeName) throws Exception {
+        internalCluster().setDisruptionScheme(disruptionScheme);
+        disruptionScheme.startDisrupting();
 
-                final Client remainingClient = client(Arrays.stream(
+        final Client clientToMasterlessNode = client();
-                    internalCluster().getNodeNames()).filter(n -> n.equals(nodeName) == false).findAny().get());
 
         assertBusy(() -> {
-                    ClusterState state = remainingClient.admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
+            ClusterState state = clientToMasterlessNode.admin().cluster().prepareState().setLocal(true)
+                .execute().actionGet().getState();
             assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID));
         });
 
-                assertThrows(remainingClient.prepareGet("test", "type1", "1"),
+        assertThrows(clientToMasterlessNode.prepareGet("test", "type1", "1"),
             ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-                assertThrows(remainingClient.prepareGet("no_index", "type1", "1"),
+        assertThrows(clientToMasterlessNode.prepareGet("no_index", "type1", "1"),
             ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-                assertThrows(remainingClient.prepareMultiGet().add("test", "type1", "1"),
+        assertThrows(clientToMasterlessNode.prepareMultiGet().add("test", "type1", "1"),
             ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-                assertThrows(remainingClient.prepareMultiGet().add("no_index", "type1", "1"),
+        assertThrows(clientToMasterlessNode.prepareMultiGet().add("no_index", "type1", "1"),
             ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-                assertThrows(remainingClient.admin().indices().prepareAnalyze("test", "this is a test"),
+        assertThrows(clientToMasterlessNode.admin().indices().prepareAnalyze("test", "this is a test"),
             ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-                assertThrows(remainingClient.admin().indices().prepareAnalyze("no_index", "this is a test"),
+        assertThrows(clientToMasterlessNode.admin().indices().prepareAnalyze("no_index", "this is a test"),
             ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-                assertThrows(remainingClient.prepareSearch("test").setSize(0),
+        assertThrows(clientToMasterlessNode.prepareSearch("test").setSize(0),
             ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-                assertThrows(remainingClient.prepareSearch("no_index").setSize(0),
+        assertThrows(clientToMasterlessNode.prepareSearch("no_index").setSize(0),
             ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
         checkUpdateAction(false, timeout,
-                    remainingClient.prepareUpdate("test", "type1", "1")
+            clientToMasterlessNode.prepareUpdate("test", "type1", "1")
                 .setScript(new Script(
                     ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script",
                     Collections.emptyMap())).setTimeout(timeout));
 
         checkUpdateAction(true, timeout,
-                    remainingClient.prepareUpdate("no_index", "type1", "1")
+            clientToMasterlessNode.prepareUpdate("no_index", "type1", "1")
                 .setScript(new Script(
                     ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script",
                     Collections.emptyMap())).setTimeout(timeout));
 
 
-                checkWriteAction(remainingClient.prepareIndex("test", "type1", "1")
+        checkWriteAction(clientToMasterlessNode.prepareIndex("test", "type1", "1")
             .setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
 
-                checkWriteAction(remainingClient.prepareIndex("no_index", "type1", "1")
+        checkWriteAction(clientToMasterlessNode.prepareIndex("no_index", "type1", "1")
             .setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
 
-                BulkRequestBuilder bulkRequestBuilder = remainingClient.prepareBulk();
+        BulkRequestBuilder bulkRequestBuilder = clientToMasterlessNode.prepareBulk();
-                bulkRequestBuilder.add(remainingClient.prepareIndex("test", "type1", "1")
+        bulkRequestBuilder.add(clientToMasterlessNode.prepareIndex("test", "type1", "1")
             .setSource(XContentFactory.jsonBuilder().startObject().endObject()));
-                bulkRequestBuilder.add(remainingClient.prepareIndex("test", "type1", "2")
+        bulkRequestBuilder.add(clientToMasterlessNode.prepareIndex("test", "type1", "2")
             .setSource(XContentFactory.jsonBuilder().startObject().endObject()));
         bulkRequestBuilder.setTimeout(timeout);
         checkWriteAction(bulkRequestBuilder);
 
-                bulkRequestBuilder = remainingClient.prepareBulk();
+        bulkRequestBuilder = clientToMasterlessNode.prepareBulk();
-                bulkRequestBuilder.add(remainingClient.prepareIndex("no_index", "type1", "1")
+        bulkRequestBuilder.add(clientToMasterlessNode.prepareIndex("no_index", "type1", "1")
             .setSource(XContentFactory.jsonBuilder().startObject().endObject()));
-                bulkRequestBuilder.add(remainingClient.prepareIndex("no_index", "type1", "2")
+        bulkRequestBuilder.add(clientToMasterlessNode.prepareIndex("no_index", "type1", "2")
             .setSource(XContentFactory.jsonBuilder().startObject().endObject()));
         bulkRequestBuilder.setTimeout(timeout);
         checkWriteAction(bulkRequestBuilder);
 
-                return Settings.EMPTY;
+        disruptionScheme.stopDisrupting();
-            }
-        });
-
-        internalCluster().startNode(settings);
 
         client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("3").execute().actionGet();
     }
@@ -192,12 +195,10 @@ public class NoMasterNodeIT extends ESIntegTestCase {
     public void testNoMasterActionsWriteMasterBlock() throws Exception {
         Settings settings = Settings.builder()
             .put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false)
-            .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
             .put(DiscoverySettings.NO_MASTER_BLOCK_SETTING.getKey(), "write")
-            .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 3)
             .build();
 
-        internalCluster().startNodes(3, settings);
+        final List<String> nodes = internalCluster().startNodes(3, settings);
 
         prepareCreate("test1").setSettings(
             Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)).get();
@@ -213,38 +214,37 @@ public class NoMasterNodeIT extends ESIntegTestCase {
         ClusterStateResponse clusterState = client().admin().cluster().prepareState().get();
         logger.info("Cluster state:\n{}", clusterState.getState());
 
-        internalCluster().stopRandomDataNode();
+        final NetworkDisruption disruptionScheme
-        internalCluster().restartRandomDataNode(new RestartCallback() {
+            = new NetworkDisruption(new IsolateAllNodes(new HashSet<>(nodes)), new NetworkDisconnect());
-            @Override
+        internalCluster().setDisruptionScheme(disruptionScheme);
-            public Settings onNodeStopped(String nodeName) throws Exception {
+        disruptionScheme.startDisrupting();
 
-                final Client remainingClient = client(Arrays.stream(
+        final Client clientToMasterlessNode = client();
-                    internalCluster().getNodeNames()).filter(n -> n.equals(nodeName) == false).findAny().get());
 
         assertTrue(awaitBusy(() -> {
-                        ClusterState state = remainingClient.admin().cluster().prepareState().setLocal(true).get().getState();
+                ClusterState state = clientToMasterlessNode.admin().cluster().prepareState().setLocal(true).get().getState();
                 return state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID);
             }
         ));
 
-                GetResponse getResponse = remainingClient.prepareGet("test1", "type1", "1").get();
+        GetResponse getResponse = clientToMasterlessNode.prepareGet("test1", "type1", "1").get();
         assertExists(getResponse);
 
-                SearchResponse countResponse = remainingClient.prepareSearch("test1").setAllowPartialSearchResults(true).setSize(0).get();
+        SearchResponse countResponse = clientToMasterlessNode.prepareSearch("test1").setAllowPartialSearchResults(true).setSize(0).get();
         assertHitCount(countResponse, 1L);
 
         logger.info("--> here 3");
-                SearchResponse searchResponse = remainingClient.prepareSearch("test1").setAllowPartialSearchResults(true).get();
+        SearchResponse searchResponse = clientToMasterlessNode.prepareSearch("test1").setAllowPartialSearchResults(true).get();
         assertHitCount(searchResponse, 1L);
 
-                countResponse = remainingClient.prepareSearch("test2").setAllowPartialSearchResults(true).setSize(0).get();
+        countResponse = clientToMasterlessNode.prepareSearch("test2").setAllowPartialSearchResults(true).setSize(0).get();
         assertThat(countResponse.getTotalShards(), equalTo(3));
         assertThat(countResponse.getSuccessfulShards(), equalTo(1));
 
         TimeValue timeout = TimeValue.timeValueMillis(200);
         long now = System.currentTimeMillis();
         try {
-                    remainingClient.prepareUpdate("test1", "type1", "1")
+            clientToMasterlessNode.prepareUpdate("test1", "type1", "1")
                 .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").setTimeout(timeout).get();
             fail("Expected ClusterBlockException");
         } catch (ClusterBlockException e) {
@@ -256,20 +256,15 @@ public class NoMasterNodeIT extends ESIntegTestCase {
         }
 
         try {
-                    remainingClient.prepareIndex("test1", "type1", "1")
+            clientToMasterlessNode.prepareIndex("test1", "type1", "1")
                 .setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout).get();
             fail("Expected ClusterBlockException");
         } catch (ClusterBlockException e) {
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
         }
 
-                logger.info("finished assertions, restarting node [{}]", nodeName);
+        disruptionScheme.stopDisrupting();
 
-                return Settings.EMPTY;
-            }
-        });
-
-        internalCluster().startNode(settings);
         client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("3").get();
     }
 }

server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java

@@ -51,7 +51,6 @@ import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
-import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING;
 import static org.elasticsearch.common.settings.Settings.builder;
 import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING;
 import static org.elasticsearch.discovery.zen.SettingsBasedHostsProvider.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING;
@@ -83,17 +82,9 @@ public class ClusterBootstrapServiceTests extends ESTestCase {
         transportService = transport.createTransportService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(),
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, emptySet());
 
-        clusterBootstrapService = new ClusterBootstrapService(builder().put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 3).build(),
+        clusterBootstrapService = new ClusterBootstrapService(Settings.builder().putList(INITIAL_MASTER_NODES_SETTING.getKey(),
+            localNode.getName(), otherNode1.getName(), otherNode2.getName()).build(),
             transportService);
-
-        final Settings settings;
-        if (randomBoolean()) {
-            settings = Settings.builder().put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 3).build();
-        } else {
-            settings = Settings.builder()
-                .putList(INITIAL_MASTER_NODES_SETTING.getKey(), localNode.getName(), otherNode1.getName(), otherNode2.getName()).build();
-        }
-        clusterBootstrapService = new ClusterBootstrapService(settings, transportService);
     }
 
     private DiscoveryNode newDiscoveryNode(String nodeName) {
@@ -126,10 +117,6 @@ public class ClusterBootstrapServiceTests extends ESTestCase {
         testConfiguredIfSettingSet(builder().putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()));
     }
 
-    public void testDoesNothingByDefaultIfMasterNodeCountConfigured() {
-        testConfiguredIfSettingSet(builder().put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 0));
-    }
-
     public void testDoesNothingByDefaultIfMasterNodesConfigured() {
         testConfiguredIfSettingSet(builder().putList(INITIAL_MASTER_NODES_SETTING.getKey()));
     }

server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java

@@ -39,7 +39,6 @@ import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
-import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING;
 import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -172,31 +171,11 @@ public class ClusterFormationFailureHelperTests extends ESTestCase {
                 "[cluster.initial_master_nodes] is empty on this node: have discovered [" + otherNode + "]; " +
                 "discovery will continue using [] from hosts providers and [" + localNode + "] from last-known cluster state"));
 
-        assertThat(new ClusterFormationState(Settings.builder().put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 2).build(),
-                clusterState, emptyList(), emptyList()).getDescription(),
-            is("master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and " +
-                "this node must discover at least [2] master-eligible nodes to bootstrap a cluster: have discovered []; " +
-                "discovery will continue using [] from hosts providers and [" + localNode + "] from last-known cluster state"));
-
         assertThat(new ClusterFormationState(Settings.builder().putList(INITIAL_MASTER_NODES_SETTING.getKey(), "other").build(),
                 clusterState, emptyList(), emptyList()).getDescription(),
             is("master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and " +
                 "this node must discover master-eligible nodes [other] to bootstrap a cluster: have discovered []; " +
                 "discovery will continue using [] from hosts providers and [" + localNode + "] from last-known cluster state"));
-
-        assertThat(new ClusterFormationState(Settings.builder().putList(INITIAL_MASTER_NODES_SETTING.getKey(), "other")
-                .put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1).build(),
-                clusterState, emptyList(), emptyList()).getDescription(),
-            is("master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and " +
-                "this node must discover master-eligible nodes [other] to bootstrap a cluster: have discovered []; " +
-                "discovery will continue using [] from hosts providers and [" + localNode + "] from last-known cluster state"));
-
-        assertThat(new ClusterFormationState(Settings.builder().putList(INITIAL_MASTER_NODES_SETTING.getKey(), "other")
-                .put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 2).build(),
-                clusterState, emptyList(), emptyList()).getDescription(),
-            is("master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and " +
-                "this node must discover at least [2] master-eligible nodes, including [other], to bootstrap a cluster: have discovered " +
-                "[]; discovery will continue using [] from hosts providers and [" + localNode + "] from last-known cluster state"));
     }
 
     private static VotingConfiguration config(String[] nodeIds) {

test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java

@@ -56,13 +56,16 @@ import java.util.Random;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
-import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING;
+import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.DATA;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER;
 import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING;
 import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
+import static org.elasticsearch.node.Node.NODE_MASTER_SETTING;
+import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists;
 import static org.hamcrest.Matchers.equalTo;
@@ -187,6 +190,7 @@ public class InternalTestClusterTests extends ESTestCase {
         final String clusterName1 = "shared1";
         final String clusterName2 = "shared2";
         String transportClient = getTestTransportType();
+        final long bootstrapNodeSelectionSeed = randomLong();
         NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {
             @Override
             public Settings nodeSettings(int nodeOrdinal) {
@@ -200,12 +204,19 @@ public class InternalTestClusterTests extends ESTestCase {
                 if (autoManageMinMasterNodes == false) {
                     assert minNumDataNodes == maxNumDataNodes;
                     assert masterNodes == false;
-                    settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1)
+                    settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1);
-                        .put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), minNumDataNodes);
                 }
                 return settings.build();
             }
 
+            @Override
+            public List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
+                if (autoManageMinMasterNodes) {
+                    return allNodesSettings;
+                }
+                return addBootstrapConfiguration(new Random(bootstrapNodeSelectionSeed), allNodesSettings);
+            }
+
             @Override
             public Path nodeConfigPath(int nodeOrdinal) {
                 return null;
@@ -255,6 +266,19 @@ public class InternalTestClusterTests extends ESTestCase {
         }
     }
 
+    private static List<Settings> addBootstrapConfiguration(Random random, List<Settings> allNodesSettings) {
+        final List<Settings> updatedSettings = new ArrayList<>(allNodesSettings);
+        final int bootstrapIndex = randomFrom(random, IntStream.range(0, updatedSettings.size())
+            .filter(i -> NODE_MASTER_SETTING.get(allNodesSettings.get(i))).boxed().collect(Collectors.toList()));
+        final Settings settings = updatedSettings.get(bootstrapIndex);
+        assertFalse(INITIAL_MASTER_NODES_SETTING.exists(settings));
+        assertTrue(NODE_MASTER_SETTING.get(settings));
+        updatedSettings.set(bootstrapIndex,
+            Settings.builder().put(settings).putList(INITIAL_MASTER_NODES_SETTING.getKey(), allNodesSettings.stream()
+                .filter(NODE_MASTER_SETTING::get).map(NODE_NAME_SETTING::get).collect(Collectors.toList())).build());
+        return updatedSettings;
+    }
+
     public void testDataFolderAssignmentAndCleaning() throws IOException, InterruptedException {
         long clusterSeed = randomLong();
         boolean masterNodes = randomBoolean();
@@ -376,6 +400,9 @@ public class InternalTestClusterTests extends ESTestCase {
         String transportClient = getTestTransportType();
         InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, false,
                 false, 0, 0, "test", new NodeConfigurationSource() {
+
+            private boolean bootstrapConfigurationSet;
+
             @Override
             public Settings nodeSettings(int nodeOrdinal) {
                 return Settings.builder()
@@ -390,6 +417,16 @@ public class InternalTestClusterTests extends ESTestCase {
                         .build();
             }
 
+            @Override
+            public List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
+                if (bootstrapConfigurationSet || allNodesSettings.stream().noneMatch(NODE_MASTER_SETTING::get)) {
+                    return allNodesSettings;
+                }
+
+                bootstrapConfigurationSet = true;
+                return addBootstrapConfiguration(random(), allNodesSettings);
+            }
+
             @Override
             public Path nodeConfigPath(int nodeOrdinal) {
                 return null;
@@ -413,7 +450,6 @@ public class InternalTestClusterTests extends ESTestCase {
         final long masterCount = roles.stream().filter(role -> role == MASTER).count();
         final Settings minMasterNodes = Settings.builder()
             .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), masterCount / 2 + 1)
-            .put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), masterCount)
             .build();
 
         try {