Fixes the DiscoveryWithServiceDisruptionsIT#testIndicesDeleted test

In particular, this test ensures we don't restart the master node until
we know the index deletion has taken effect on master and the master
eligible nodes.

Closes #16917
Closes #16890
This commit is contained in:
Ali Beyad 2016-03-01 18:10:46 -05:00
parent 06929f8ed4
commit e411cbb060
1 changed files with 26 additions and 8 deletions

View File

@ -177,13 +177,17 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
}
private void configureUnicastCluster(int numberOfNodes, @Nullable int[] unicastHostsOrdinals, int minimumMasterNode) throws ExecutionException, InterruptedException {
configureUnicastCluster(DEFAULT_SETTINGS, numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
}
private void configureUnicastCluster(Settings settings, int numberOfNodes, @Nullable int[] unicastHostsOrdinals, int minimumMasterNode) throws ExecutionException, InterruptedException {
if (minimumMasterNode < 0) {
minimumMasterNode = numberOfNodes / 2 + 1;
}
logger.info("---> configured unicast");
// TODO: Rarely use default settings form some of these
Settings nodeSettings = Settings.builder()
.put(DEFAULT_SETTINGS)
.put(settings)
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNode)
.build();
@ -196,7 +200,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
}
}
/**
* Test that no split brain occurs under partial network partition. See https://github.com/elastic/elasticsearch/issues/2488
*/
@ -1075,25 +1078,40 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
* Tests that indices are properly deleted even if there is a master transition in between.
* Test for https://github.com/elastic/elasticsearch/issues/11665
*/
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/16890")
public void testIndicesDeleted() throws Exception {
configureUnicastCluster(3, null, 2);
final Settings settings = Settings.builder()
.put(DEFAULT_SETTINGS)
.put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0s") // don't wait on isolated data node
.put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // wait till cluster state is committed
.build();
final String idxName = "test";
configureUnicastCluster(settings, 3, null, 2);
InternalTestCluster.Async<List<String>> masterNodes = internalCluster().startMasterOnlyNodesAsync(2);
InternalTestCluster.Async<String> dataNode = internalCluster().startDataOnlyNodeAsync();
dataNode.get();
masterNodes.get();
final List<String> allMasterEligibleNodes = masterNodes.get();
ensureStableCluster(3);
assertAcked(prepareCreate("test"));
ensureYellow();
String masterNode1 = internalCluster().getMasterName();
final String masterNode1 = internalCluster().getMasterName();
NetworkPartition networkPartition = new NetworkUnresponsivePartition(masterNode1, dataNode.get(), getRandom());
internalCluster().setDisruptionScheme(networkPartition);
networkPartition.startDisrupting();
internalCluster().client(masterNode1).admin().indices().prepareDelete("test").setTimeout("1s").get();
// We know this will time out due to the partition, we check manually below to not proceed until
// the delete has been applied to the master node and the master eligible node.
internalCluster().client(masterNode1).admin().indices().prepareDelete(idxName).setTimeout("0s").get();
// Don't restart the master node until we know the index deletion has taken effect on master and the master eligible node.
assertBusy(() -> {
for (String masterNode : allMasterEligibleNodes) {
final ClusterState masterState = internalCluster().clusterService(masterNode).state();
assertTrue("index not deleted on " + masterNode, masterState.metaData().hasIndex(idxName) == false &&
masterState.status() == ClusterState.ClusterStateStatus.APPLIED);
}
});
internalCluster().restartNode(masterNode1, InternalTestCluster.EMPTY_CALLBACK);
ensureYellow();
assertFalse(client().admin().indices().prepareExists("test").get().isExists());
assertFalse(client().admin().indices().prepareExists(idxName).get().isExists());
}
protected NetworkPartition addRandomPartition() {