[TEST] Ensure cluster size reflected in the cluster state

We perform some management operations that require the cluster to be
consistent with respect to the number of nodes in the cluster state
/ visible to the master in order to rely on the ack mechanism. This
only applies to the test infrastructure when nodes are not explicitly
started / stopped as well as while tearing down the cluster and wiping
indices after the tests.

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -102,6 +102,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.TestCluster.clusterName;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -504,6 +505,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
                     assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().getAsMap(), metaData
                             .transientSettings().getAsMap().size(), equalTo(0));
                 }
+                ensureClusterSizeConsistency();
                 immutableCluster().wipe(); // wipe after to make sure we fail in the test that didn't ack the delete
                 immutableCluster().assertAfterTest();
             } finally {
@@ -837,6 +839,9 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
         return actionGet.getStatus();
     }
 
+    private void ensureClusterSizeConsistency() {
+        assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(immutableCluster().size())).get());
+    }
 
     /**
      * Ensures the cluster is in a searchable state for the given indices. This means a searchable copy of each

src/test/java/org/elasticsearch/test/TestCluster.java

@@ -94,6 +94,7 @@ import static org.apache.lucene.util.LuceneTestCase.rarely;
 import static org.apache.lucene.util.LuceneTestCase.usually;
 import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
 import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
 
 /**
  * TestCluster manages a set of JVM private nodes and allows convenient access to them.
@@ -397,6 +398,11 @@ public final class TestCluster extends ImmutableTestCluster {
         } catch (Exception e) {
             throw new ElasticsearchException("failed to start nodes", e);
         }
+        if (!futures.isEmpty()) {
+            synchronized (this) {
+                assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(nodes.size())).get());
+            }
+        }
     }
 
     /**
@@ -424,6 +430,9 @@ public final class TestCluster extends ImmutableTestCluster {
         for (NodeAndClient toRemove : nodesToRemove) {
             nodes.remove(toRemove.name);
         }
+        if (!nodesToRemove.isEmpty() && size() > 0) {
+            assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(nodes.size())).get());
+        }
     }
 
     private NodeAndClient buildNode(Settings settings, Version version) {

src/test/java/org/elasticsearch/tribe/TribeTests.java

@@ -346,4 +346,4 @@ public class TribeTests extends ElasticsearchIntegrationTest {
         }
         return count;
     }
-}
+}