diff --git a/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java b/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java index 64c5bc409ae..972a42ea8de 100644 --- a/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java +++ b/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java @@ -58,7 +58,6 @@ public class ClusterDynamicSettingsModule extends AbstractModule { clusterDynamicSettings.addDynamicSetting(DisableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_DISABLE_NEW_ALLOCATION); clusterDynamicSettings.addDynamicSetting(DisableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_DISABLE_ALLOCATION); clusterDynamicSettings.addDynamicSetting(DisableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_DISABLE_REPLICA_ALLOCATION); - clusterDynamicSettings.addDynamicSetting(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, Validator.INTEGER); clusterDynamicSettings.addDynamicSetting(ZenDiscovery.SETTING_REJOIN_ON_MASTER_GONE, Validator.BOOLEAN); clusterDynamicSettings.addDynamicSetting(DiscoverySettings.NO_MASTER_BLOCK); clusterDynamicSettings.addDynamicSetting(FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP + "*"); diff --git a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index acc41d64667..ab34f8a006a 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -36,6 +36,9 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.service.InternalClusterService; +import org.elasticsearch.cluster.settings.ClusterDynamicSettings; +import org.elasticsearch.cluster.settings.DynamicSettings; +import org.elasticsearch.cluster.settings.Validator; import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -144,9 +147,9 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen @Inject public ZenDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, - TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, + TransportService transportService, final ClusterService clusterService, NodeSettingsService nodeSettingsService, DiscoveryNodeService discoveryNodeService, ZenPingService pingService, ElectMasterService electMasterService, - DiscoverySettings discoverySettings) { + DiscoverySettings discoverySettings, @ClusterDynamicSettings DynamicSettings dynamicSettings) { super(settings); this.clusterName = clusterName; this.clusterService = clusterService; @@ -196,6 +199,24 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen this.joinThreadControl = new JoinThreadControl(threadPool); transportService.registerHandler(DISCOVERY_REJOIN_ACTION_NAME, new RejoinClusterRequestHandler()); + + dynamicSettings.addDynamicSetting(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, new Validator() { + @Override + public String validate(String setting, String value) { + int intValue; + try { + intValue = Integer.parseInt(value); + } catch (NumberFormatException ex) { + return "cannot parse value [" + value + "] as an integer"; + } + int masterNodes = clusterService.state().nodes().masterNodes().size(); + if (intValue > masterNodes) { + return "cannot set " + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES + " to more than the current master nodes count [" + masterNodes + "]"; + } + return null; + } + }); + } @Override diff --git a/src/test/java/org/elasticsearch/cluster/ClusterServiceTests.java b/src/test/java/org/elasticsearch/cluster/ClusterServiceTests.java index ec37acaad77..4958ef166ee 100644 --- a/src/test/java/org/elasticsearch/cluster/ClusterServiceTests.java +++ b/src/test/java/org/elasticsearch/cluster/ClusterServiceTests.java @@ -608,7 +608,7 @@ public class ClusterServiceTests extends ElasticsearchIntegrationTest { .put("plugin.types", TestPlugin.class.getName()) .build(); - internalCluster().startNode(settings); + String node_0 = internalCluster().startNode(settings); ClusterService clusterService = internalCluster().getInstance(ClusterService.class); MasterAwareService testService = internalCluster().getInstance(MasterAwareService.class); @@ -635,15 +635,22 @@ public class ClusterServiceTests extends ElasticsearchIntegrationTest { clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("1").get(); assertThat(clusterHealth.isTimedOut(), equalTo(false)); - // now that node1 is closed, node2 should be elected as master + // now that node0 is closed, node1 should be elected as master assertThat(clusterService1.state().nodes().localNodeMaster(), is(true)); assertThat(testService1.master(), is(true)); + // start another node and set min_master_node + internalCluster().startNode(ImmutableSettings.builder().put(settings)); + assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut()); + Settings transientSettings = settingsBuilder() .put("discovery.zen.minimum_master_nodes", 2) .build(); client().admin().cluster().prepareUpdateSettings().setTransientSettings(transientSettings).get(); + // and shutdown the second node + internalCluster().stopRandomNonMasterNode(); + // there should not be any master as the minimum number of required eligible masters is not met awaitBusy(new Predicate() { public boolean apply(Object obj) { @@ -652,7 +659,7 @@ public class ClusterServiceTests extends ElasticsearchIntegrationTest { }); assertThat(testService1.master(), is(false)); - + // bring the node back up String node_2 = internalCluster().startNode(ImmutableSettings.builder().put(settings).put(transientSettings)); ClusterService clusterService2 = internalCluster().getInstance(ClusterService.class, node_2); MasterAwareService testService2 = internalCluster().getInstance(MasterAwareService.class, node_2); diff --git a/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesTests.java b/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesTests.java index 50090be578f..8d0f951ec4b 100644 --- a/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesTests.java +++ b/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Client; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.elect.ElectMasterService; @@ -35,14 +36,17 @@ import org.elasticsearch.test.junit.annotations.TestLogging; import org.junit.Test; import java.io.IOException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static org.elasticsearch.client.Requests.clusterHealthRequest; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.*; -@ClusterScope(scope = Scope.TEST, numDataNodes =0) +@ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class MinimumMasterNodesTests extends ElasticsearchIntegrationTest { @Test @@ -98,7 +102,7 @@ public class MinimumMasterNodesTests extends ElasticsearchIntegrationTest { internalCluster().stopCurrentMasterNode(); awaitBusy(new Predicate() { public boolean apply(Object obj) { - ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); + ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); return state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID); } }); @@ -187,7 +191,7 @@ public class MinimumMasterNodesTests extends ElasticsearchIntegrationTest { return state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID); } }); - + awaitBusy(new Predicate() { public boolean apply(Object obj) { ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); @@ -308,4 +312,40 @@ public class MinimumMasterNodesTests extends ElasticsearchIntegrationTest { } }, 20, TimeUnit.SECONDS), equalTo(true)); } + + @Test + public void testCanNotBringClusterDown() throws ExecutionException, InterruptedException { + int nodeCount = scaledRandomIntBetween(1, 10); + ImmutableSettings.Builder settings = settingsBuilder() + .put("discovery.type", "zen") + .put("discovery.zen.ping_timeout", "200ms") + .put("discovery.initial_state_timeout", "500ms") + .put("gateway.type", "local"); + + if (randomBoolean()) { + // sometime set an initial value + settings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, randomIntBetween(1, nodeCount)); + } + + internalCluster().startNodesAsync(nodeCount, settings.build()).get(); + + client().admin().cluster().prepareHealth().setWaitForNodes("=" + nodeCount); + + int updateCount = randomIntBetween(1, nodeCount); + + logger.info("--> updating [{}] to [{}]", ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, updateCount); + assertAcked(client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(settingsBuilder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, updateCount))); + + client().admin().cluster().prepareHealth().setWaitForNodes("=" + nodeCount); + + updateCount = nodeCount + randomIntBetween(1, 2000); + logger.info("--> trying to updating [{}] to [{}]", ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, updateCount); + assertThat(client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(settingsBuilder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, updateCount)) + .get().getPersistentSettings().getAsMap().keySet(), + empty()); + + client().admin().cluster().prepareHealth().setWaitForNodes("=" + nodeCount); + } }