HADOOP-12186. ActiveStandbyElector shouldn't call monitorLockNodeAsync multiple times (Contributed by zhihai xu)
This commit is contained in:
parent
bff67dfe2f
commit
233cab89ad
|
@ -924,6 +924,9 @@ Release 2.8.0 - UNRELEASED
|
||||||
HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command
|
HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command
|
||||||
name using reflection. (Lei (Eddy) Xu)
|
name using reflection. (Lei (Eddy) Xu)
|
||||||
|
|
||||||
|
HADOOP-12186. ActiveStandbyElector shouldn't call monitorLockNodeAsync
|
||||||
|
multiple times (zhihai xu via vinayakumarb)
|
||||||
|
|
||||||
Release 2.7.2 - UNRELEASED
|
Release 2.7.2 - UNRELEASED
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
|
@ -173,7 +173,9 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
|
||||||
|
|
||||||
private Lock sessionReestablishLockForTests = new ReentrantLock();
|
private Lock sessionReestablishLockForTests = new ReentrantLock();
|
||||||
private boolean wantToBeInElection;
|
private boolean wantToBeInElection;
|
||||||
|
private boolean monitorLockNodePending = false;
|
||||||
|
private ZooKeeper monitorLockNodeClient;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a new ActiveStandbyElector object <br/>
|
* Create a new ActiveStandbyElector object <br/>
|
||||||
* The elector is created by providing to it the Zookeeper configuration, the
|
* The elector is created by providing to it the Zookeeper configuration, the
|
||||||
|
@ -468,7 +470,8 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
|
||||||
public synchronized void processResult(int rc, String path, Object ctx,
|
public synchronized void processResult(int rc, String path, Object ctx,
|
||||||
Stat stat) {
|
Stat stat) {
|
||||||
if (isStaleClient(ctx)) return;
|
if (isStaleClient(ctx)) return;
|
||||||
|
monitorLockNodePending = false;
|
||||||
|
|
||||||
assert wantToBeInElection :
|
assert wantToBeInElection :
|
||||||
"Got a StatNode result after quitting election";
|
"Got a StatNode result after quitting election";
|
||||||
|
|
||||||
|
@ -744,6 +747,11 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
|
||||||
return state;
|
return state;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
synchronized boolean isMonitorLockNodePending() {
|
||||||
|
return monitorLockNodePending;
|
||||||
|
}
|
||||||
|
|
||||||
private boolean reEstablishSession() {
|
private boolean reEstablishSession() {
|
||||||
int connectionRetryCount = 0;
|
int connectionRetryCount = 0;
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
|
@ -949,7 +957,13 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void monitorLockNodeAsync() {
|
private void monitorLockNodeAsync() {
|
||||||
zkClient.exists(zkLockFilePath,
|
if (monitorLockNodePending && monitorLockNodeClient == zkClient) {
|
||||||
|
LOG.info("Ignore duplicate monitor lock-node request.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
monitorLockNodePending = true;
|
||||||
|
monitorLockNodeClient = zkClient;
|
||||||
|
zkClient.exists(zkLockFilePath,
|
||||||
watcher, this,
|
watcher, this,
|
||||||
zkClient);
|
zkClient);
|
||||||
}
|
}
|
||||||
|
|
|
@ -452,6 +452,10 @@ public class TestActiveStandbyElector {
|
||||||
Event.KeeperState.SyncConnected);
|
Event.KeeperState.SyncConnected);
|
||||||
elector.processWatchEvent(mockZK, mockEvent);
|
elector.processWatchEvent(mockZK, mockEvent);
|
||||||
verifyExistCall(1);
|
verifyExistCall(1);
|
||||||
|
Assert.assertTrue(elector.isMonitorLockNodePending());
|
||||||
|
elector.processResult(Code.SESSIONEXPIRED.intValue(), ZK_LOCK_NAME,
|
||||||
|
mockZK, new Stat());
|
||||||
|
Assert.assertFalse(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
// session expired should enter safe mode and initiate re-election
|
// session expired should enter safe mode and initiate re-election
|
||||||
// re-election checked via checking re-creation of new zookeeper and
|
// re-election checked via checking re-creation of new zookeeper and
|
||||||
|
@ -495,6 +499,13 @@ public class TestActiveStandbyElector {
|
||||||
ZK_LOCK_NAME);
|
ZK_LOCK_NAME);
|
||||||
Mockito.verify(mockApp, Mockito.times(1)).becomeStandby();
|
Mockito.verify(mockApp, Mockito.times(1)).becomeStandby();
|
||||||
verifyExistCall(1);
|
verifyExistCall(1);
|
||||||
|
Assert.assertTrue(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
|
Stat stat = new Stat();
|
||||||
|
stat.setEphemeralOwner(0L);
|
||||||
|
Mockito.when(mockZK.getSessionId()).thenReturn(1L);
|
||||||
|
elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
|
||||||
|
Assert.assertFalse(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
WatchedEvent mockEvent = Mockito.mock(WatchedEvent.class);
|
WatchedEvent mockEvent = Mockito.mock(WatchedEvent.class);
|
||||||
Mockito.when(mockEvent.getPath()).thenReturn(ZK_LOCK_NAME);
|
Mockito.when(mockEvent.getPath()).thenReturn(ZK_LOCK_NAME);
|
||||||
|
@ -504,12 +515,18 @@ public class TestActiveStandbyElector {
|
||||||
Event.EventType.NodeDataChanged);
|
Event.EventType.NodeDataChanged);
|
||||||
elector.processWatchEvent(mockZK, mockEvent);
|
elector.processWatchEvent(mockZK, mockEvent);
|
||||||
verifyExistCall(2);
|
verifyExistCall(2);
|
||||||
|
Assert.assertTrue(elector.isMonitorLockNodePending());
|
||||||
|
elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
|
||||||
|
Assert.assertFalse(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
// monitoring should be setup again after event is received
|
// monitoring should be setup again after event is received
|
||||||
Mockito.when(mockEvent.getType()).thenReturn(
|
Mockito.when(mockEvent.getType()).thenReturn(
|
||||||
Event.EventType.NodeChildrenChanged);
|
Event.EventType.NodeChildrenChanged);
|
||||||
elector.processWatchEvent(mockZK, mockEvent);
|
elector.processWatchEvent(mockZK, mockEvent);
|
||||||
verifyExistCall(3);
|
verifyExistCall(3);
|
||||||
|
Assert.assertTrue(elector.isMonitorLockNodePending());
|
||||||
|
elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
|
||||||
|
Assert.assertFalse(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
// lock node deletion when in standby mode should create znode again
|
// lock node deletion when in standby mode should create znode again
|
||||||
// successful znode creation enters active state and sets monitor
|
// successful znode creation enters active state and sets monitor
|
||||||
|
@ -524,6 +541,10 @@ public class TestActiveStandbyElector {
|
||||||
ZK_LOCK_NAME);
|
ZK_LOCK_NAME);
|
||||||
Mockito.verify(mockApp, Mockito.times(1)).becomeActive();
|
Mockito.verify(mockApp, Mockito.times(1)).becomeActive();
|
||||||
verifyExistCall(4);
|
verifyExistCall(4);
|
||||||
|
Assert.assertTrue(elector.isMonitorLockNodePending());
|
||||||
|
stat.setEphemeralOwner(1L);
|
||||||
|
elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
|
||||||
|
Assert.assertFalse(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
// lock node deletion in active mode should enter neutral mode and create
|
// lock node deletion in active mode should enter neutral mode and create
|
||||||
// znode again successful znode creation enters active state and sets
|
// znode again successful znode creation enters active state and sets
|
||||||
|
@ -538,6 +559,9 @@ public class TestActiveStandbyElector {
|
||||||
ZK_LOCK_NAME);
|
ZK_LOCK_NAME);
|
||||||
Mockito.verify(mockApp, Mockito.times(2)).becomeActive();
|
Mockito.verify(mockApp, Mockito.times(2)).becomeActive();
|
||||||
verifyExistCall(5);
|
verifyExistCall(5);
|
||||||
|
Assert.assertTrue(elector.isMonitorLockNodePending());
|
||||||
|
elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
|
||||||
|
Assert.assertFalse(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
// bad path name results in fatal error
|
// bad path name results in fatal error
|
||||||
Mockito.when(mockEvent.getPath()).thenReturn(null);
|
Mockito.when(mockEvent.getPath()).thenReturn(null);
|
||||||
|
@ -570,6 +594,13 @@ public class TestActiveStandbyElector {
|
||||||
ZK_LOCK_NAME);
|
ZK_LOCK_NAME);
|
||||||
Mockito.verify(mockApp, Mockito.times(1)).becomeStandby();
|
Mockito.verify(mockApp, Mockito.times(1)).becomeStandby();
|
||||||
verifyExistCall(1);
|
verifyExistCall(1);
|
||||||
|
Assert.assertTrue(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
|
Stat stat = new Stat();
|
||||||
|
stat.setEphemeralOwner(0L);
|
||||||
|
Mockito.when(mockZK.getSessionId()).thenReturn(1L);
|
||||||
|
elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
|
||||||
|
Assert.assertFalse(elector.isMonitorLockNodePending());
|
||||||
|
|
||||||
WatchedEvent mockEvent = Mockito.mock(WatchedEvent.class);
|
WatchedEvent mockEvent = Mockito.mock(WatchedEvent.class);
|
||||||
Mockito.when(mockEvent.getPath()).thenReturn(ZK_LOCK_NAME);
|
Mockito.when(mockEvent.getPath()).thenReturn(ZK_LOCK_NAME);
|
||||||
|
|
Loading…
Reference in New Issue