[TEST] Fix testMultipleNodesShutdownNonMasterNodes to wait for the right nodes to rejoin the cluster

This test was sporadically failing for the following reason:
- 4 nodes (nodes 0, 1, 2, and 3) running with `minimum_master_nodes` set to 3
- we stop 2 nodes (node 0 and 3)
- wait for cluster block to be in place on all nodes
- start 2 nodes (node 4 and node 5) and do a `prepareHealth().setWaitForNodes("4")`
- then do a search request

The search request runs into the `ClusterBlockException` as the `prepareHealth().setWaitForNodes("4")` check succeeds on a cluster state that has
nodes 1, 2, 3, and 4, i.e., only one of the two new nodes has joined the cluster and only one of the two dead nodes was removed by the master
(removing the dead nodes only happens after there are again `minimum_master_nodes` nodes in the cluster).

This commit fixes the issue by reusing a method from InternalTestCluster that checks that the right nodes have rejoined the cluster.
This commit is contained in:
Yannick Welsch 2017-04-07 15:23:04 +02:00
parent 0821fa23ff
commit a3cceb8a00
2 changed files with 3 additions and 4 deletions

View File

@ -249,8 +249,7 @@ public class MinimumMasterNodesIT extends ESIntegTestCase {
String[] newNodes = internalCluster().startNodes(2, settings).stream().toArray(String[]::new); String[] newNodes = internalCluster().startNodes(2, settings).stream().toArray(String[]::new);
ensureGreen(); ensureGreen();
clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("4").execute().actionGet(); internalCluster().validateClusterFormed();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
state = client().admin().cluster().prepareState().execute().actionGet().getState(); state = client().admin().cluster().prepareState().execute().actionGet().getState();
assertThat(state.nodes().getSize(), equalTo(4)); assertThat(state.nodes().getSize(), equalTo(4));

View File

@ -1059,13 +1059,13 @@ public final class InternalTestCluster extends TestCluster {
} }
/** ensure a cluster is formed with all published nodes. */ /** ensure a cluster is formed with all published nodes. */
private void validateClusterFormed() { public synchronized void validateClusterFormed() {
String name = randomFrom(random, getNodeNames()); String name = randomFrom(random, getNodeNames());
validateClusterFormed(name); validateClusterFormed(name);
} }
/** ensure a cluster is formed with all published nodes, but do so by using the client of the specified node */ /** ensure a cluster is formed with all published nodes, but do so by using the client of the specified node */
private void validateClusterFormed(String viaNode) { public synchronized void validateClusterFormed(String viaNode) {
Set<DiscoveryNode> expectedNodes = new HashSet<>(); Set<DiscoveryNode> expectedNodes = new HashSet<>();
for (NodeAndClient nodeAndClient : nodes.values()) { for (NodeAndClient nodeAndClient : nodes.values()) {
expectedNodes.add(getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode()); expectedNodes.add(getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode());