honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix NodeJoinTests again (#36133) 

In #36033 we removed a catch block because we thought we were preventing
exceptions by avoiding concurrent elections, missing the obvious fact that some
joins are supposed to be failing.

As a quick fix the catch was reinstated in 3a5dab6d8e
but this change adds finesse by only catching exceptions from the joins that we
expect to fail. It also inlines an always-false parameter to `initialState()`.
This commit is contained in:
David Turner 2018-12-01 09:54:01 +00:00 committed by GitHub
parent 9cc416bc46
commit 8bb1952975
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 31 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
View File

@ -101,13 +101,12 @@ public class NodeJoinTests extends ESTestCase {
masterService.close(); masterService.close();
} }
private static ClusterState initialState(boolean withMaster, DiscoveryNode localNode, long term, long version, private static ClusterState initialState(DiscoveryNode localNode, long term, long version,
VotingConfiguration config) { VotingConfiguration config) {
ClusterState initialClusterState = ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName())) return ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName()))
.nodes(DiscoveryNodes.builder() .nodes(DiscoveryNodes.builder()
.add(localNode) .add(localNode)
.localNodeId(localNode.getId()) .localNodeId(localNode.getId()))
.masterNodeId(withMaster ? localNode.getId() : null))
.metaData(MetaData.builder() .metaData(MetaData.builder()
.coordinationMetaData( .coordinationMetaData(
CoordinationMetaData.builder() CoordinationMetaData.builder()
@ -117,7 +116,6 @@ public class NodeJoinTests extends ESTestCase {
.build())) .build()))
.version(version) .version(version)
.blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build(); .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build();
return initialClusterState;
} }
private void setupFakeMasterServiceAndCoordinator(long term, ClusterState initialState) { private void setupFakeMasterServiceAndCoordinator(long term, ClusterState initialState) {
@ -267,7 +265,7 @@ public class NodeJoinTests extends ESTestCase {
DiscoveryNode node1 = newNode(1, true); DiscoveryNode node1 = newNode(1, true);
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupFakeMasterServiceAndCoordinator(initialTerm, initialState(false, node0, initialTerm, initialVersion, setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
new VotingConfiguration(Collections.singleton(randomFrom(node0, node1).getId())))); new VotingConfiguration(Collections.singleton(randomFrom(node0, node1).getId()))));
assertFalse(isLocalNodeElectedMaster()); assertFalse(isLocalNodeElectedMaster());
assertNull(coordinator.getStateForMasterService().nodes().getMasterNodeId()); assertNull(coordinator.getStateForMasterService().nodes().getMasterNodeId());
@ -286,7 +284,7 @@ public class NodeJoinTests extends ESTestCase {
DiscoveryNode node1 = newNode(1, true); DiscoveryNode node1 = newNode(1, true);
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupFakeMasterServiceAndCoordinator(initialTerm, initialState(false, node0, initialTerm, initialVersion, setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
new VotingConfiguration(Collections.singleton(node1.getId())))); new VotingConfiguration(Collections.singleton(node1.getId()))));
assertFalse(isLocalNodeElectedMaster()); assertFalse(isLocalNodeElectedMaster());
long newTerm = initialTerm + randomLongBetween(1, 10); long newTerm = initialTerm + randomLongBetween(1, 10);
@ -301,7 +299,7 @@ public class NodeJoinTests extends ESTestCase {
DiscoveryNode node1 = newNode(1, true); DiscoveryNode node1 = newNode(1, true);
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupFakeMasterServiceAndCoordinator(initialTerm, initialState(false, node0, initialTerm, initialVersion, setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
new VotingConfiguration(Collections.singleton(node0.getId())))); new VotingConfiguration(Collections.singleton(node0.getId()))));
assertFalse(isLocalNodeElectedMaster()); assertFalse(isLocalNodeElectedMaster());
long newTerm = initialTerm + randomLongBetween(1, 10); long newTerm = initialTerm + randomLongBetween(1, 10);
@ -315,7 +313,7 @@ public class NodeJoinTests extends ESTestCase {
DiscoveryNode node1 = newNode(1, true); DiscoveryNode node1 = newNode(1, true);
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupFakeMasterServiceAndCoordinator(initialTerm, initialState(false, node0, initialTerm, initialVersion, setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
new VotingConfiguration(Collections.singleton(node0.getId())))); new VotingConfiguration(Collections.singleton(node0.getId()))));
assertFalse(isLocalNodeElectedMaster()); assertFalse(isLocalNodeElectedMaster());
long newTerm = initialTerm + randomLongBetween(1, 10); long newTerm = initialTerm + randomLongBetween(1, 10);
@ -333,7 +331,7 @@ public class NodeJoinTests extends ESTestCase {
DiscoveryNode node2 = newNode(2, true); DiscoveryNode node2 = newNode(2, true);
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupFakeMasterServiceAndCoordinator(initialTerm, initialState(false, node0, initialTerm, initialVersion, setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
new VotingConfiguration(Collections.singleton(node2.getId())))); new VotingConfiguration(Collections.singleton(node2.getId()))));
assertFalse(isLocalNodeElectedMaster()); assertFalse(isLocalNodeElectedMaster());
long newTerm = initialTerm + randomLongBetween(1, 10); long newTerm = initialTerm + randomLongBetween(1, 10);
@ -360,7 +358,7 @@ public class NodeJoinTests extends ESTestCase {
DiscoveryNode node1 = newNode(1, true); DiscoveryNode node1 = newNode(1, true);
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupFakeMasterServiceAndCoordinator(initialTerm, initialState(false, node0, initialTerm, initialVersion, setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
new VotingConfiguration(Collections.singleton(node0.getId())))); new VotingConfiguration(Collections.singleton(node0.getId()))));
long newTerm = initialTerm + randomLongBetween(1, 10); long newTerm = initialTerm + randomLongBetween(1, 10);
coordinator.coordinationState.get().handleStartJoin(new StartJoinRequest(node1, newTerm)); coordinator.coordinationState.get().handleStartJoin(new StartJoinRequest(node1, newTerm));
@ -379,7 +377,7 @@ public class NodeJoinTests extends ESTestCase {
DiscoveryNode node1 = newNode(1, true); DiscoveryNode node1 = newNode(1, true);
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupFakeMasterServiceAndCoordinator(initialTerm, initialState(false, node0, initialTerm, initialVersion, setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
new VotingConfiguration(Collections.singleton(node0.getId())))); new VotingConfiguration(Collections.singleton(node0.getId()))));
long newTerm = initialTerm + randomLongBetween(1, 10); long newTerm = initialTerm + randomLongBetween(1, 10);
coordinator.coordinationState.get().handleStartJoin(new StartJoinRequest(node1, newTerm)); coordinator.coordinationState.get().handleStartJoin(new StartJoinRequest(node1, newTerm));
@ -398,7 +396,7 @@ public class NodeJoinTests extends ESTestCase {
DiscoveryNode node1 = newNode(1, true); DiscoveryNode node1 = newNode(1, true);
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupFakeMasterServiceAndCoordinator(initialTerm, initialState(false, node0, initialTerm, initialVersion, setupFakeMasterServiceAndCoordinator(initialTerm, initialState(node0, initialTerm, initialVersion,
new VotingConfiguration(Collections.singleton(node1.getId())))); new VotingConfiguration(Collections.singleton(node1.getId()))));
long newTerm = initialTerm + randomLongBetween(1, 10); long newTerm = initialTerm + randomLongBetween(1, 10);
SimpleFuture fut = joinNodeAsync(new JoinRequest(node0, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion)))); SimpleFuture fut = joinNodeAsync(new JoinRequest(node0, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion))));
@ -427,7 +425,7 @@ public class NodeJoinTests extends ESTestCase {
long initialTerm = randomLongBetween(1, 10); long initialTerm = randomLongBetween(1, 10);
long initialVersion = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10);
setupRealMasterServiceAndCoordinator(initialTerm, initialState(false, localNode, initialTerm, initialVersion, votingConfiguration)); setupRealMasterServiceAndCoordinator(initialTerm, initialState(localNode, initialTerm, initialVersion, votingConfiguration));
long newTerm = initialTerm + randomLongBetween(1, 10); long newTerm = initialTerm + randomLongBetween(1, 10);
// we need at least a quorum of voting nodes with a correct term and worse state // we need at least a quorum of voting nodes with a correct term and worse state
@ -468,33 +466,35 @@ public class NodeJoinTests extends ESTestCase {
possiblyFailingJoinRequests.addAll(randomSubsetOf(possiblyFailingJoinRequests)); possiblyFailingJoinRequests.addAll(randomSubsetOf(possiblyFailingJoinRequests));
CyclicBarrier barrier = new CyclicBarrier(correctJoinRequests.size() + possiblyFailingJoinRequests.size() + 1); CyclicBarrier barrier = new CyclicBarrier(correctJoinRequests.size() + possiblyFailingJoinRequests.size() + 1);
final Runnable awaitBarrier = () -> {
final AtomicBoolean stopAsserting = new AtomicBoolean();
final Thread assertionThread = new Thread(() -> {
try { try {
barrier.await(); barrier.await();
} catch (InterruptedException | BrokenBarrierException e) { } catch (InterruptedException | BrokenBarrierException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
};
final AtomicBoolean stopAsserting = new AtomicBoolean();
final Thread assertionThread = new Thread(() -> {
awaitBarrier.run();
while (stopAsserting.get() == false) { while (stopAsserting.get() == false) {
coordinator.invariant(); coordinator.invariant();
} }
}, "assert invariants"); }, "assert invariants");
final List<Thread> joinThreads = Stream.concat(correctJoinRequests.stream(), possiblyFailingJoinRequests.stream()) final List<Thread> joinThreads = Stream.concat(correctJoinRequests.stream().map(joinRequest ->
.map(joinRequest ->
new Thread(() -> { new Thread(() -> {
try { awaitBarrier.run();
barrier.await(); joinNode(joinRequest);
} catch (InterruptedException | BrokenBarrierException e) { }, "process " + joinRequest)), possiblyFailingJoinRequests.stream().map(joinRequest ->
throw new RuntimeException(e); new Thread(() -> {
} awaitBarrier.run();
try { try {
joinNode(joinRequest); joinNode(joinRequest);
} catch (CoordinationStateRejectedException e) { } catch (CoordinationStateRejectedException e) {
// ignore // ignore - these requests are expected to fail
} }
}, "process " + joinRequest)).collect(Collectors.toList()); }, "process " + joinRequest))).collect(Collectors.toList());
assertionThread.start(); assertionThread.start();
joinThreads.forEach(Thread::start); joinThreads.forEach(Thread::start);