Deprecate minimum_master_nodes (#37868)

Today we pass `discovery.zen.minimum_master_nodes` to nodes started up in
tests, but for 7.x nodes this setting is not required as it has no effect.
This commit removes this setting so that nodes are started with more realistic
configurations, and deprecates it.

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -373,18 +373,15 @@ class ClusterFormationTasks {
                 'path.repo'                    : "${node.sharedDir}/repo",
                 'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists
-                'node.attr.testattr'           : 'test'
+                'node.attr.testattr'           : 'test',
+                // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
+                'discovery.initial_state_timeout' : '0s'
         ]
         int minimumMasterNodes = node.config.minimumMasterNodes.call()
-        if (minimumMasterNodes > 0) {
+        if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) {
             esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes
         }
-        if (minimumMasterNodes > 1) {
+        if (node.nodeVersion.before("7.0.0") && esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
-            // don't wait for state.. just start up quickly
-            // this will also allow new and old nodes in the BWC case to become the master
-            esConfig['discovery.initial_state_timeout'] = '0s'
-        }
-        if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
             // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct
             // its mistake. Instead, only wait 5s to do another round of pinging.
             // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation

docs/reference/migration/migrate_7_0/discovery.asciidoc

@@ -13,8 +13,8 @@ settings summary>> for an example, and the
 <<modules-discovery-bootstrap-cluster,cluster bootstrapping reference
 documentation>> describes this setting in more detail.
 
-The `discovery.zen.minimum_master_nodes` setting is required during a rolling
+The `discovery.zen.minimum_master_nodes` setting is permitted, but ignored, on
-upgrade from 6.x, but can be removed in all other circumstances.
+7.x nodes.
 
 [float]
 ==== Removing master-eligible nodes sometimes requires voting exclusions

modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java

@@ -33,9 +33,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
@@ -58,11 +56,7 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        final Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal))
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(TestZenDiscovery.USE_ZEN2.getKey(), true).build();
-            .put(TestZenDiscovery.USE_ZEN2.getKey(), true)
-            .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE);
-
-        return builder.build();
     }
 
     @Override

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java

@@ -21,6 +21,7 @@ package org.elasticsearch.discovery.ec2;
 
 
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -41,13 +42,14 @@ public class Ec2DiscoveryUpdateSettingsTests extends AbstractAwsTestCase {
                 .build();
         internalCluster().startNode(nodeSettings);
 
-        // We try to update minimum_master_nodes now
+        // We try to update a setting now
-        ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings()
+        final String expectedValue = UUIDs.randomBase64UUID(random());
-                .setPersistentSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", 1))
+        final String settingName = "cluster.routing.allocation.exclude.any_attribute";
-                .setTransientSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", 1))
+        final ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder().put(settingName, expectedValue))
                 .get();
 
-        Integer min = response.getPersistentSettings().getAsInt("discovery.zen.minimum_master_nodes", null);
+        final String value = response.getPersistentSettings().get(settingName);
-        assertThat(min, is(1));
+        assertThat(value, is(expectedValue));
     }
 }

qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java

@@ -40,7 +40,7 @@ import java.util.List;
  * The intention is to confirm that users can still run their Elasticsearch instances with previous configurations.
  */
 public class CustomLoggingConfigIT extends ESRestTestCase {
-    private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*started.*";
+    private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*recovered.*cluster_state.*";
 
     public void testSuccessfulStartupWithCustomConfig() throws Exception {
         assertBusy(() -> {

qa/rolling-upgrade/build.gradle

@@ -77,7 +77,6 @@ for (Version version : bwcVersions.wireCompatible) {
       dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
       clusterName = 'rolling-upgrade'
       otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
-      minimumMasterNodes = { 2 }
       autoSetInitialMasterNodes = false
       /* Override the data directory so the new node always gets the node we
       * just stopped's data directory. */

rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml

@@ -4,22 +4,22 @@
       cluster.put_settings:
         body:
           transient:
-            discovery.zen.minimum_master_nodes: 1
+            cluster.routing.allocation.enable: "none"
         flat_settings: true
 
-  - match: {transient: {discovery.zen.minimum_master_nodes: "1"}}
+  - match: {transient: {cluster.routing.allocation.enable: "none"}}
 
   - do:
       cluster.get_settings:
         flat_settings: true
 
-  - match: {transient: {discovery.zen.minimum_master_nodes: "1"}}
+  - match: {transient: {cluster.routing.allocation.enable: "none"}}
 
   - do:
       cluster.put_settings:
         body:
           transient:
-            discovery.zen.minimum_master_nodes: null
+            cluster.routing.allocation.enable: null
         flat_settings: true
 
   - match: {transient: {}}
@@ -35,22 +35,22 @@
       cluster.put_settings:
         body:
           persistent:
-            cluster.routing.allocation.disk.threshold_enabled: false
+            cluster.routing.allocation.enable: "none"
         flat_settings: true
 
-  - match: {persistent: {cluster.routing.allocation.disk.threshold_enabled: "false"}}
+  - match: {persistent: {cluster.routing.allocation.enable: "none"}}
 
   - do:
       cluster.get_settings:
         flat_settings: true
 
-  - match: {persistent: {cluster.routing.allocation.disk.threshold_enabled: "false"}}
+  - match: {persistent: {cluster.routing.allocation.enable: "none"}}
 
   - do:
       cluster.put_settings:
         body:
           persistent:
-            cluster.routing.allocation.disk.threshold_enabled: null
+            cluster.routing.allocation.enable: null
         flat_settings: true
 
   - match: {persistent: {}}

server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java

@@ -42,7 +42,7 @@ public class ElectMasterService {
     private static final Logger logger = LogManager.getLogger(ElectMasterService.class);
 
     public static final Setting<Integer> DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING =
-        Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope);
+        Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope, Property.Deprecated);
 
     private volatile int minimumMasterNodes;
 

server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java

@@ -23,7 +23,6 @@ import org.elasticsearch.cluster.coordination.ClusterBootstrapService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
-import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -51,8 +50,7 @@ public class IndicesExistsIT extends ESIntegTestCase {
 
     public void testIndexExistsWithBlocksInPlace() throws IOException {
         Settings settings = Settings.builder()
-            .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99)
+            .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99).build();
-            .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE).build();
         String node = internalCluster().startNode(settings);
 
         assertThrows(client(node).admin().indices().prepareExists("test").setMasterNodeTimeout(TimeValue.timeValueSeconds(0)),

server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java

@@ -26,14 +26,13 @@ import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExc
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.coordination.ClusterBootstrapService;
+import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
-import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.node.Node;
@@ -102,7 +101,6 @@ public class MinimumMasterNodesIT extends ESIntegTestCase {
         bootstrapNodeId = 2;
 
         Settings settings = Settings.builder()
-                .put("discovery.zen.minimum_master_nodes", 2)
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms")
                 .put("discovery.initial_state_timeout", "500ms")
                 .build();
@@ -237,7 +235,6 @@ public class MinimumMasterNodesIT extends ESIntegTestCase {
         bootstrapNodeId = 3;
 
         Settings settings = Settings.builder()
-                .put("discovery.zen.minimum_master_nodes", 3)
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "1s")
                 .put("discovery.initial_state_timeout", "500ms")
                 .build();
@@ -316,11 +313,9 @@ public class MinimumMasterNodesIT extends ESIntegTestCase {
         Settings settings = Settings.builder()
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms")
                 .put("discovery.initial_state_timeout", "500ms")
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)
                 .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "100ms") // speed things up
                 .build();
 
-
         internalCluster().startNodes(3, settings);
         ensureGreen(); // ensure cluster state is recovered before we disrupt things
 

server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java

@@ -38,7 +38,6 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 
-import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
@@ -47,12 +46,6 @@ import static org.hamcrest.Matchers.nullValue;
 @TestLogging("_root:DEBUG,org.elasticsearch.action.admin.cluster.state:TRACE")
 public class SpecificMasterNodesIT extends ESIntegTestCase {
 
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
-            .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build();
-    }
-
     @Override
     protected List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
         // if it's the first master in the cluster bootstrap the cluster with this node name

server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java

@@ -28,7 +28,6 @@ import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoverySettings;
-import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.NodeMetaData;
@@ -155,7 +154,6 @@ public class UnsafeBootstrapMasterIT extends ESIntegTestCase {
     public void testNotBootstrappedCluster() throws Exception {
         internalCluster().startNode(
                 Settings.builder()
-                        .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
                         .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup
                         .build());
         assertBusy(() -> {
@@ -172,9 +170,7 @@ public class UnsafeBootstrapMasterIT extends ESIntegTestCase {
 
     public void testNoManifestFile() throws IOException {
         bootstrapNodeId = 1;
-        internalCluster().startNode(Settings.builder()
+        internalCluster().startNode();
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
-                .build());
         ensureStableCluster(1);
         NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class);
         internalCluster().stopRandomDataNode();
@@ -186,9 +182,7 @@ public class UnsafeBootstrapMasterIT extends ESIntegTestCase {
 
     public void testNoMetaData() throws IOException {
         bootstrapNodeId = 1;
-        internalCluster().startNode(Settings.builder()
+        internalCluster().startNode();
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
-                .build());
         ensureStableCluster(1);
         NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class);
         internalCluster().stopRandomDataNode();
@@ -201,9 +195,7 @@ public class UnsafeBootstrapMasterIT extends ESIntegTestCase {
 
     public void testAbortedByUser() throws IOException {
         bootstrapNodeId = 1;
-        internalCluster().startNode(Settings.builder()
+        internalCluster().startNode();
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
-                .build());
         ensureStableCluster(1);
         internalCluster().stopRandomDataNode();
 
@@ -213,13 +205,9 @@ public class UnsafeBootstrapMasterIT extends ESIntegTestCase {
 
     public void test3MasterNodes2Failed() throws Exception {
         bootstrapNodeId = 3;
-        List<String> masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.builder()
+        List<String> masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.EMPTY);
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
-                .build());
 
-        String dataNode = internalCluster().startDataOnlyNode(Settings.builder()
+        String dataNode = internalCluster().startDataOnlyNode();
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
-                .build());
         createIndex("test");
 
         Client dataNodeClient = internalCluster().client(dataNode);
@@ -246,9 +234,7 @@ public class UnsafeBootstrapMasterIT extends ESIntegTestCase {
                 String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT,
                         metaData.coordinationMetaData().term(), metaData.version())));
 
-        internalCluster().startMasterOnlyNode(Settings.builder()
+        internalCluster().startMasterOnlyNode();
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
-                .build());
 
         assertBusy(() -> {
             ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true)

server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java

@@ -34,6 +34,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDeci
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.MetaStateService;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -50,6 +51,7 @@ import static org.elasticsearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM;
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING;
 import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
+import static org.elasticsearch.test.InternalTestCluster.REMOVED_MINIMUM_MASTER_NODES;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
@@ -194,7 +196,8 @@ public class Zen1IT extends ESIntegTestCase {
                 }
                 ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get();
                 assertFalse(nodeName, clusterHealthResponse.isTimedOut());
-                return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS)).build();
+                return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS)
+                    .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES)).build();
             }
         });
 
@@ -289,6 +292,7 @@ public class Zen1IT extends ESIntegTestCase {
                 return Coordinator.addZen1Attribute(false, Settings.builder())
                     .put(ZEN2_SETTINGS)
                     .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames)
+                    .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES)
                     .build();
             }
         });

server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java

@@ -35,7 +35,6 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -125,11 +124,6 @@ public class MasterDisruptionIT extends AbstractDisruptionTestCase {
 
         ensureStableCluster(3);
 
-        logger.info("--> reducing min master nodes to 2");
-        assertAcked(client().admin().cluster().prepareUpdateSettings()
-                .setTransientSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2))
-                .get());
-
         String master = internalCluster().getMasterName();
         String nonMaster = null;
         for (String node : internalCluster().getNodeNames()) {
@@ -154,7 +148,7 @@ public class MasterDisruptionIT extends AbstractDisruptionTestCase {
      */
     @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.test.disruption:TRACE")
     public void testStaleMasterNotHijackingMajority() throws Exception {
-        // 3 node cluster with unicast discovery and minimum_master_nodes set to the default of 2:
+        // 3 node cluster with unicast discovery:
         final List<String> nodes = startCluster(3);
 
         // Save the current master node as old master node, because that node will get frozen

server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java

@@ -1,66 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.discovery.zen;
-
-import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
-import static org.elasticsearch.test.InternalTestCluster.nameFilter;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.isIn;
-
-@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0)
-public class MinimumMasterNodesInClusterStateIT extends ESIntegTestCase {
-
-    public void testMasterPublishes() throws Exception {
-        final String firstNode = internalCluster().startNode();
-
-        {
-            final ClusterState localState
-                = client(firstNode).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState();
-            assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1));
-            assertFalse(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(localState.metaData().settings()));
-        }
-
-        final List<String> secondThirdNodes = internalCluster().startNodes(2);
-        assertThat(internalCluster().getMasterName(), equalTo(firstNode));
-
-        final List<String> allNodes = Stream.concat(Stream.of(firstNode), secondThirdNodes.stream()).collect(Collectors.toList());
-        for (final String node : allNodes) {
-            final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState();
-            assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1));
-            assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2));
-        }
-
-        internalCluster().stopRandomNode(nameFilter(firstNode));
-        assertThat(internalCluster().getMasterName(), isIn(secondThirdNodes));
-
-        for (final String node : secondThirdNodes) {
-            final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState();
-            assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(2));
-            assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2));
-        }
-    }
-}

server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java

@@ -268,6 +268,9 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
             IOUtils.close(toClose);
             terminate(threadPool);
         }
+
+        assertWarnings("[discovery.zen.minimum_master_nodes] setting was deprecated in Elasticsearch and will be removed in a future " +
+            "release! See the breaking changes documentation for the next major version.");
     }
 
     public void testPendingCSQueueIsClearedWhenClusterStatePublished() throws Exception {
@@ -318,6 +321,9 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
             IOUtils.close(toClose);
             terminate(threadPool);
         }
+
+        assertWarnings("[discovery.zen.minimum_master_nodes] setting was deprecated in Elasticsearch and will be removed in a future " +
+            "release! See the breaking changes documentation for the next major version.");
     }
 
     private class AwaitingPublishListener implements ActionListener<Void> {

server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java

@@ -25,7 +25,6 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.coordination.ClusterBootstrapService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -59,7 +58,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         return super.addExtraClusterBootstrapSettings(allNodesSettings);
     }
 
-    public Set<ClusterBlock> waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException {
+    public Set<ClusterBlock> waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) {
         long start = System.currentTimeMillis();
         Set<ClusterBlock> blocks;
         do {
@@ -70,22 +69,20 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         return blocks;
     }
 
-    public Client startNode(Settings.Builder settings, int minMasterNodes) {
+    public Client startNode(Settings.Builder settings) {
-        String name = internalCluster().startNode(
+        String name = internalCluster().startNode(Settings.builder().put(settings.build()));
-            Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes)
-                .put(settings.build()));
         return internalCluster().client(name);
     }
 
     public void testRecoverAfterNodes() throws Exception {
         logger.info("--> start node (1)");
-        Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
+        Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
         assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start node (2)");
-        Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
+        Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
         Thread.sleep(BLOCK_WAIT_TIMEOUT.millis());
         assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
@@ -95,7 +92,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
 
         logger.info("--> start node (3)");
-        Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
+        Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
 
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode1).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode2).isEmpty(), equalTo(true));
@@ -106,7 +103,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         logger.info("--> start master_node (1)");
         Client master1 = startNode(Settings.builder()
             .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false)
-            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
+            .put(Node.NODE_MASTER_SETTING.getKey(), true));
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -114,7 +111,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         logger.info("--> start data_node (1)");
         Client data1 = startNode(Settings.builder()
             .put("gateway.recover_after_master_nodes", 2)
-            .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
+            .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -125,7 +122,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         logger.info("--> start data_node (2)");
         Client data2 = startNode(Settings.builder()
             .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true)
-            .put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
+            .put(Node.NODE_MASTER_SETTING.getKey(), false));
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -140,7 +137,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         Client master2 = startNode(Settings.builder()
             .put("gateway.recover_after_master_nodes", 2)
             .put(Node.NODE_DATA_SETTING.getKey(), false)
-            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
+            .put(Node.NODE_MASTER_SETTING.getKey(), true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true));
@@ -152,7 +149,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         Client master1 = startNode(Settings.builder()
             .put("gateway.recover_after_data_nodes", 2)
             .put(Node.NODE_DATA_SETTING.getKey(), false)
-            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
+            .put(Node.NODE_MASTER_SETTING.getKey(), true));
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -161,7 +158,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         Client data1 = startNode(Settings.builder()
             .put("gateway.recover_after_data_nodes", 2)
             .put(Node.NODE_DATA_SETTING.getKey(), true)
-            .put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
+            .put(Node.NODE_MASTER_SETTING.getKey(), false));
         assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -173,7 +170,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         Client master2 = startNode(Settings.builder()
             .put("gateway.recover_after_data_nodes", 2)
             .put(Node.NODE_DATA_SETTING.getKey(), false)
-            .put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
+            .put(Node.NODE_MASTER_SETTING.getKey(), true));
         assertThat(master2.admin().cluster().prepareState().setLocal(true).execute().actionGet()
                 .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
                 hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@@ -188,7 +185,7 @@ public class RecoverAfterNodesIT extends ESIntegTestCase {
         Client data2 = startNode(Settings.builder()
             .put("gateway.recover_after_data_nodes", 2)
             .put(Node.NODE_DATA_SETTING.getKey(), true)
-            .put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
+            .put(Node.NODE_MASTER_SETTING.getKey(), false));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true));
         assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true));

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -1087,17 +1087,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return lastKnownCount.get();
     }
 
-
-    /**
-     * Sets the cluster's minimum master node and make sure the response is acknowledge.
-     * Note: this doesn't guarantee that the new setting has taken effect, just that it has been received by all nodes.
-     */
-    public void setMinimumMasterNodes(int n) {
-        assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
-            Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), n))
-            .get().isAcknowledged());
-    }
-
     /**
      * Prints the current cluster state as debug logging.
      */

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -33,8 +33,8 @@ import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
-import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
+import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
@@ -157,11 +157,15 @@ import static org.elasticsearch.test.ESTestCase.assertBusy;
 import static org.elasticsearch.test.ESTestCase.awaitBusy;
 import static org.elasticsearch.test.ESTestCase.getTestTransportType;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.discovery.TestZenDiscovery.USE_ZEN2;
+import static org.elasticsearch.test.discovery.TestZenDiscovery.usingZen1;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -615,19 +619,28 @@ public final class InternalTestCluster extends TestCluster {
                 .put("node.name", name)
                 .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);
 
-        final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()).equals("single-node");
+        final String discoveryType = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build());
-        if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) {
+        final boolean usingSingleNodeDiscovery = discoveryType.equals("single-node");
-            assert updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null :
+        final boolean usingZen1 = TestZenDiscovery.usingZen1(updatedSettings.build());
-                    "min master nodes may not be set when auto managed";
+        if (usingSingleNodeDiscovery == false) {
-            assert updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null :
+            if (autoManageMinMasterNodes) {
-                    "automatically managing min master nodes require nodes to complete a join cycle" +
+                assertThat("min master nodes may not be set when auto managed",
-                            " when starting";
+                    updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), nullValue());
-            updatedSettings
+                assertThat("automatically managing min master nodes require nodes to complete a join cycle when starting",
-                    // don't wait too long not to slow down tests
+                    updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()), nullValue());
-                    .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s")
+
-                    .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes);
+                if (usingZen1) {
-        } else if (!usingSingleNodeDiscovery && updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) {
+                    updatedSettings
-            throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured");
+                        // don't wait too long not to slow down tests
+                        .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s")
+                        .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes);
+                }
+            } else {
+                if (usingZen1) {
+                    assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured",
+                        updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), not(nullValue()));
+                }
+            }
         }
 
         return updatedSettings.build();
@@ -822,6 +835,8 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
+    public static final int REMOVED_MINIMUM_MASTER_NODES = Integer.MAX_VALUE;
+
     private final class NodeAndClient implements Closeable {
         private MockNode node;
         private final Settings originalNodeSettings;
@@ -933,8 +948,10 @@ public final class InternalTestCluster extends TestCluster {
             Settings.Builder newSettings = Settings.builder();
             newSettings.put(callbackSettings);
             if (minMasterNodes >= 0) {
-                assert DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed";
+                if (usingZen1(newSettings.build())) {
-                newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes);
+                    assertFalse("min master nodes is auto managed", DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()));
+                    newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes);
+                }
                 if (INITIAL_MASTER_NODES_SETTING.exists(callbackSettings) == false) {
                     newSettings.putList(INITIAL_MASTER_NODES_SETTING.getKey());
                 }
@@ -966,10 +983,23 @@ public final class InternalTestCluster extends TestCluster {
                     .put(newSettings)
                     .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), newIdSeed)
                     .build();
-            final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(finalSettings).equals("single-node");
+            if (usingZen1(finalSettings)) {
-            if (usingSingleNodeDiscovery == false && DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) {
+                if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) {
-                throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() +
+                    throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() +
-                    " is not configured after restart of [" + name + "]");
+                        " is not configured after restart of [" + name + "]");
+                }
+            } else {
+                if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings)) {
+                    // simulating an upgrade from Zen1 to Zen2, but there's no way to remove a setting when restarting a node, so
+                    // you have to set it to REMOVED_MINIMUM_MASTER_NODES (== Integer.MAX_VALUE) to indicate its removal:
+                    assertTrue(USE_ZEN2.exists(finalSettings));
+                    assertTrue(USE_ZEN2.get(finalSettings));
+                    assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(finalSettings), equalTo(REMOVED_MINIMUM_MASTER_NODES));
+
+                    final Builder builder = Settings.builder().put(finalSettings);
+                    builder.remove(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey());
+                    finalSettings = builder.build();
+                }
             }
             Collection<Class<? extends Plugin>> plugins = node.getClasspathPlugins();
             node = new MockNode(finalSettings, plugins);

test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java

@@ -60,6 +60,7 @@ public class TestZenDiscovery extends ZenDiscovery {
 
     public static final Setting<Boolean> USE_ZEN2 =
         Setting.boolSetting("discovery.zen.use_zen2", true, Setting.Property.NodeScope);
+    private static final String TEST_ZEN_DISCOVERY_TYPE = "test-zen";
 
     /** A plugin which installs mock discovery and configures it to be used. */
     public static class TestPlugin extends Plugin implements DiscoveryPlugin {
@@ -97,7 +98,7 @@ public class TestZenDiscovery extends ZenDiscovery {
         @Override
         public Settings additionalSettings() {
             return Settings.builder()
-                .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen")
+                .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), TEST_ZEN_DISCOVERY_TYPE)
                 .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey())
                 .build();
         }
@@ -124,4 +125,9 @@ public class TestZenDiscovery extends ZenDiscovery {
     public ZenPing getZenPing() {
         return zenPing;
     }
+
+    public static boolean usingZen1(Settings settings) {
+        return DiscoveryModule.ZEN_DISCOVERY_TYPE.equals(DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings))
+            || USE_ZEN2.get(settings) == false;
+    }
 }

test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java

@@ -63,7 +63,6 @@ import static org.elasticsearch.cluster.node.DiscoveryNode.Role.DATA;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER;
 import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING;
-import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.node.Node.NODE_MASTER_SETTING;
 import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
@@ -101,7 +100,6 @@ public class InternalTestClusterTests extends ESTestCase {
             nodePrefix, Collections.emptyList(), Function.identity());
         // TODO: this is not ideal - we should have a way to make sure ports are initialized in the same way
         assertClusters(cluster0, cluster1, false);
-
     }
 
     /**
@@ -140,36 +138,6 @@ public class InternalTestClusterTests extends ESTestCase {
         }
     }
 
-    private void assertMMNinNodeSetting(InternalTestCluster cluster, int masterNodes) {
-        for (final String node : cluster.getNodeNames()) {
-            assertMMNinNodeSetting(node, cluster, masterNodes);
-        }
-    }
-
-    private void assertMMNinNodeSetting(String node, InternalTestCluster cluster, int masterNodes) {
-        final int minMasterNodes = masterNodes / 2 + 1;
-        Settings nodeSettings = cluster.client(node).admin().cluster().prepareNodesInfo(node).get().getNodes().get(0).getSettings();
-        assertEquals("node setting of node [" + node + "] has the wrong min_master_node setting: ["
-            + nodeSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]",
-            DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(nodeSettings).intValue(), minMasterNodes);
-    }
-
-    private void assertMMNinClusterSetting(InternalTestCluster cluster, int masterNodes) {
-        for (final String node : cluster.getNodeNames()) {
-            assertMMNinClusterSetting(node, cluster, masterNodes);
-        }
-    }
-
-    private void assertMMNinClusterSetting(String node, InternalTestCluster cluster, int masterNodes) {
-        final int minMasterNodes = masterNodes / 2 + 1;
-        Settings stateSettings = cluster.client(node).admin().cluster().prepareState().setLocal(true)
-            .get().getState().getMetaData().settings();
-
-        assertEquals("dynamic setting for node [" + node + "] has the wrong min_master_node setting : ["
-                + stateSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]",
-            DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(stateSettings).intValue(), minMasterNodes);
-    }
-
     public void testBeforeTest() throws Exception {
         final boolean autoManageMinMasterNodes = randomBoolean();
         long clusterSeed = randomLong();
@@ -204,7 +172,6 @@ public class InternalTestClusterTests extends ESTestCase {
                 if (autoManageMinMasterNodes == false) {
                     assert minNumDataNodes == maxNumDataNodes;
                     assert masterNodes == false;
-                    settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1);
                 }
                 return settings.build();
             }
@@ -257,8 +224,6 @@ public class InternalTestClusterTests extends ESTestCase {
                 Client other = iterator1.next();
                 assertSettings(client.settings(), other.settings(), false);
             }
-            assertMMNinNodeSetting(cluster0, cluster0.numMasterNodes());
-            assertMMNinNodeSetting(cluster1, cluster0.numMasterNodes());
             cluster0.afterTest();
             cluster1.afterTest();
         } finally {
@@ -320,7 +285,6 @@ public class InternalTestClusterTests extends ESTestCase {
         try {
             cluster.beforeTest(random(), 0.0);
             final int originalMasterCount = cluster.numMasterNodes();
-            assertMMNinNodeSetting(cluster, originalMasterCount);
             final Map<String,Path[]> shardNodePaths = new HashMap<>();
             for (String name: cluster.getNodeNames()) {
                 shardNodePaths.put(name, getNodePaths(cluster, name));
@@ -335,10 +299,6 @@ public class InternalTestClusterTests extends ESTestCase {
                 expectedMasterCount--;
             }
             cluster.stopRandomNode(InternalTestCluster.nameFilter(poorNode));
-            if (expectedMasterCount != originalMasterCount) {
-                // check for updated
-                assertMMNinClusterSetting(cluster, expectedMasterCount);
-            }
             assertFileExists(testMarker); // stopping a node half way shouldn't clean data
 
             final String stableNode = randomFrom(cluster.getNodeNames());
@@ -351,14 +311,8 @@ public class InternalTestClusterTests extends ESTestCase {
             expectedMasterCount++;
             assertThat(getNodePaths(cluster, newNode1)[0], equalTo(dataPath));
             assertFileExists(testMarker); // starting a node should re-use data folders and not clean it
-            if (expectedMasterCount > 1) { // this is the first master, it's in cluster state settings won't be updated
-                assertMMNinClusterSetting(cluster, expectedMasterCount);
-            }
-            assertMMNinNodeSetting(newNode1, cluster, expectedMasterCount);
-
             final String newNode2 =  cluster.startNode();
             expectedMasterCount++;
-            assertMMNinClusterSetting(cluster, expectedMasterCount);
             final Path newDataPath = getNodePaths(cluster, newNode2)[0];
             final Path newTestMarker = newDataPath.resolve("newTestMarker");
             assertThat(newDataPath, not(dataPath));
@@ -377,8 +331,6 @@ public class InternalTestClusterTests extends ESTestCase {
                 assertThat("data paths for " + name + " changed", getNodePaths(cluster, name),
                     equalTo(shardNodePaths.get(name)));
             }
-            assertMMNinNodeSetting(cluster, originalMasterCount);
-
         } finally {
             cluster.close();
         }
@@ -393,7 +345,6 @@ public class InternalTestClusterTests extends ESTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37462")
     public void testDifferentRolesMaintainPathOnRestart() throws Exception {
         final Path baseDir = createTempDir();
         final int numNodes = 5;
@@ -448,11 +399,6 @@ public class InternalTestClusterTests extends ESTestCase {
             roles.add(role);
         }
 
-        final long masterCount = roles.stream().filter(role -> role == MASTER).count();
-        final Settings minMasterNodes = Settings.builder()
-            .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), masterCount / 2 + 1)
-            .build();
-
         try {
             Map<DiscoveryNode.Role, Set<String>> pathsPerRole = new HashMap<>();
             for (int i = 0; i < numNodes; i++) {
@@ -460,13 +406,13 @@ public class InternalTestClusterTests extends ESTestCase {
                 final String node;
                 switch (role) {
                     case MASTER:
-                        node = cluster.startMasterOnlyNode(minMasterNodes);
+                        node = cluster.startMasterOnlyNode();
                         break;
                     case DATA:
-                        node = cluster.startDataOnlyNode(minMasterNodes);
+                        node = cluster.startDataOnlyNode();
                         break;
                     case INGEST:
-                        node = cluster.startCoordinatingOnlyNode(minMasterNodes);
+                        node = cluster.startCoordinatingOnlyNode(Settings.EMPTY);
                         break;
                     default:
                         throw new IllegalStateException("get your story straight");
@@ -476,6 +422,7 @@ public class InternalTestClusterTests extends ESTestCase {
                     assertTrue(rolePaths.add(path.toString()));
                 }
             }
+            cluster.validateClusterFormed();
             cluster.fullRestart();
 
             Map<DiscoveryNode.Role, Set<String>> result = new HashMap<>();
@@ -533,34 +480,18 @@ public class InternalTestClusterTests extends ESTestCase {
             plugins, Function.identity());
         try {
             cluster.beforeTest(random(), 0.0);
-            assertMMNinNodeSetting(cluster, 2);
             switch (randomInt(2)) {
                 case 0:
                     cluster.stopRandomDataNode();
-                    assertMMNinClusterSetting(cluster, 1);
                     cluster.startNode();
-                    assertMMNinClusterSetting(cluster, 2);
-                    assertMMNinNodeSetting(cluster, 2);
                     break;
                 case 1:
-                    cluster.rollingRestart(new InternalTestCluster.RestartCallback() {
+                    cluster.rollingRestart(InternalTestCluster.EMPTY_CALLBACK);
-                        @Override
-                        public Settings onNodeStopped(String nodeName) throws Exception {
-                            for (String name : cluster.getNodeNames()) {
-                                if (name.equals(nodeName) == false) {
-                                    assertMMNinClusterSetting(name, cluster, 1);
-                                }
-                            }
-                            return super.onNodeStopped(nodeName);
-                        }
-                    });
-                    assertMMNinClusterSetting(cluster, 2);
                     break;
                 case 2:
                     cluster.fullRestart();
                     break;
             }
-            assertMMNinNodeSetting(cluster, 2);
         } finally {
             cluster.close();
         }
@@ -585,6 +516,5 @@ public class InternalTestClusterTests extends ESTestCase {
             }
             return Settings.builder().put("node.attr.dummy", true).build();
         }
-
     }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
@@ -290,23 +291,25 @@ public class LicensingTests extends SecurityIntegTestCase {
         Path home = createTempDir();
         Path conf = home.resolve("config");
         Files.createDirectories(conf);
-        Settings nodeSettings = Settings.builder()
+        Settings.Builder nodeSettings = Settings.builder()
             .put(nodeSettings(maxNumberOfNodes() - 1).filter(s -> "xpack.security.enabled".equals(s) == false))
             .put("node.name", "my-test-node")
             .put("network.host", "localhost")
             .put("cluster.name", internalCluster().getClusterName())
-            .put("discovery.zen.minimum_master_nodes",
-                internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes"))
             .put("path.home", home)
             .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false)
             .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen")
             .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
             .putList(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey())
-            .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList)
+            .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList);
-            .build();
+        if (getUseZen2() == false) {
+            nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(),
+                ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class)));
+        }
+
         Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class,
             MockHttpTransport.TestPlugin.class);
-        try (Node node = new MockNode(nodeSettings, mockPlugins)) {
+        try (Node node = new MockNode(nodeSettings.build(), mockPlugins)) {
             node.start();
             ensureStableCluster(cluster().size() + 1);
         }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java

@@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
@@ -101,8 +102,6 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
             .put("network.host", "localhost")
             .put("cluster.name", internalCluster().getClusterName())
             .put("discovery.zen.ping.unicast.hosts", unicastHost)
-            .put("discovery.zen.minimum_master_nodes",
-                    internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes"))
             .put("xpack.security.enabled", true)
             .put("xpack.security.audit.enabled", false)
             .put("xpack.security.transport.ssl.enabled", true)
@@ -111,6 +110,10 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
             .put(Node.NODE_MASTER_SETTING.getKey(), false)
             .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
             .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false);
+        if (getUseZen2() == false) {
+            nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(),
+                ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class)));
+        }
         Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(
             LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class);
         addSSLSettingsForPEMFiles(
@@ -144,8 +147,6 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
             .put(SecurityField.USER_SETTING.getKey(), "test_user:" + SecuritySettingsSourceField.TEST_PASSWORD)
             .put("cluster.name", internalCluster().getClusterName())
             .put("discovery.zen.ping.unicast.hosts", unicastHost)
-            .put("discovery.zen.minimum_master_nodes",
-                internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes"))
             .put("xpack.security.enabled", true)
             .put("xpack.security.audit.enabled", false)
             .put("xpack.security.transport.ssl.enabled", true)
@@ -155,6 +156,10 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
             .put(Node.NODE_MASTER_SETTING.getKey(), false)
             .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
             .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false);
+        if (getUseZen2() == false) {
+            nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(),
+                ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class)));
+        }
         Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(
             LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class);
         addSSLSettingsForPEMFiles(