[Test] FullRollingRestartTests - correctly setting minimum_master_nodes

Also added a ElasticsearchIntegrationTest.setMinimumMasterNodes() helper function.

src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionTests.java

@@ -34,7 +34,6 @@ import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
@@ -57,12 +56,6 @@ import static org.hamcrest.Matchers.*;
 @ClusterScope(scope = Scope.TEST, numNodes = 0)
 public class IndexLifecycleActionTests extends ElasticsearchIntegrationTest {
 
-    protected void setMinimumMasterNodes(int n) {
-        assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
-                settingsBuilder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, n))
-                .get().isAcknowledged());
-    }
-
     @Slow
     @Test
     public void testIndexLifecycleActions() throws Exception {

src/test/java/org/elasticsearch/recovery/FullRollingRestartTests.java

@@ -25,9 +25,9 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.junit.Test;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -80,6 +80,9 @@ public class FullRollingRestartTests extends ElasticsearchIntegrationTest {
         cluster().startNode();
         cluster().startNode();
 
+        // We now have 5 nodes
+        setMinimumMasterNodes(3);
+
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForNodes("5"));
 
@@ -92,6 +95,10 @@ public class FullRollingRestartTests extends ElasticsearchIntegrationTest {
         cluster().stopRandomNode();
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForNodes("4"));
+
+        // going down to 3 nodes. note that the min_master_node may not be in effect when we shutdown the 4th
+        // node, but that's OK as it is set to 3 before.
+        setMinimumMasterNodes(2);
         cluster().stopRandomNode();
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForNodes("3"));
@@ -101,9 +108,13 @@ public class FullRollingRestartTests extends ElasticsearchIntegrationTest {
             assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), 2000l);
         }
 
+        // closing the 3rd node
         cluster().stopRandomNode();
         // make sure the cluster state is green, and all has been recovered
         assertTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForNodes("2"));
+
+        // closing the 2nd node
+        setMinimumMasterNodes(1);
         cluster().stopRandomNode();
 
         // make sure the cluster state is green, and all has been recovered

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -58,6 +58,7 @@ import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.index.mapper.FieldMapper.Loading;
 import org.elasticsearch.index.merge.policy.*;
 import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
@@ -87,6 +88,7 @@ import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
 import static org.elasticsearch.test.TestCluster.clusterName;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
@@ -114,8 +116,7 @@ import static org.hamcrest.Matchers.equalTo;
  * <pre>
  *
  * @ClusterScope(scope=Scope.TEST) public class SomeIntegrationTest extends ElasticsearchIntegrationTest {
- * @Test
- * public void testMethod() {}
+ * @Test public void testMethod() {}
  * }
  * </pre>
  * <p/>
@@ -129,8 +130,7 @@ import static org.hamcrest.Matchers.equalTo;
  *  <pre>
  * @ClusterScope(scope=Scope.SUITE, numNodes=3)
  * public class SomeIntegrationTest extends ElasticsearchIntegrationTest {
- * @Test
- * public void testMethod() {}
+ * @Test public void testMethod() {}
  * }
  * </pre>
  * <p/>
@@ -186,7 +186,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
             GLOBAL_CLUSTER = new TestCluster(masterSeed, clusterName("shared", ElasticsearchTestCase.CHILD_VM_ID, masterSeed));
         }
     }
-    
+
     @Before
     public final void before() throws IOException {
         assert Thread.getDefaultUncaughtExceptionHandler() instanceof ElasticsearchUncaughtExceptionHandler;
@@ -613,6 +613,16 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
         return actionGet.getStatus();
     }
 
+    /**
+     * Sets the cluster's minimum master node and make sure the response is acknowledge.
+     * Note: this doesn't guaranty the new settings is in effect, just that it has been received bu all nodes.
+     */
+    public void setMinimumMasterNodes(int n) {
+        assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+                settingsBuilder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, n))
+                .get().isAcknowledged());
+    }
+
     /**
      * Ensures the cluster has a yellow state via the cluster health API.
      */