HBASE-14498 Master stuck in infinite loop when all Zookeeper servers are unreachable
Signed-off-by: Michael Stack <stack@apache.org>
This commit is contained in:
parent
c606a565c1
commit
12a7d2bace
|
@ -26,6 +26,9 @@ import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.CopyOnWriteArrayList;
|
import java.util.concurrent.CopyOnWriteArrayList;
|
||||||
import java.util.concurrent.CountDownLatch;
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
@ -39,6 +42,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
|
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
|
||||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.hbase.security.Superusers;
|
import org.apache.hadoop.hbase.security.Superusers;
|
||||||
|
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
import org.apache.zookeeper.WatchedEvent;
|
import org.apache.zookeeper.WatchedEvent;
|
||||||
|
@ -76,7 +80,7 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
|
||||||
private final RecoverableZooKeeper recoverableZooKeeper;
|
private final RecoverableZooKeeper recoverableZooKeeper;
|
||||||
|
|
||||||
// abortable in case of zk failure
|
// abortable in case of zk failure
|
||||||
protected Abortable abortable;
|
protected final Abortable abortable;
|
||||||
// Used if abortable is null
|
// Used if abortable is null
|
||||||
private boolean aborted = false;
|
private boolean aborted = false;
|
||||||
|
|
||||||
|
@ -89,6 +93,13 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
|
||||||
// negotiation to complete
|
// negotiation to complete
|
||||||
public CountDownLatch saslLatch = new CountDownLatch(1);
|
public CountDownLatch saslLatch = new CountDownLatch(1);
|
||||||
|
|
||||||
|
// Connection timeout on disconnect event
|
||||||
|
private long connWaitTimeOut;
|
||||||
|
private AtomicBoolean connected = new AtomicBoolean(false);
|
||||||
|
private boolean forceAbortOnZKDisconnect;
|
||||||
|
|
||||||
|
// Execute service for zookeeper disconnect event watcher
|
||||||
|
private ExecutorService zkEventWatcherExecService = null;
|
||||||
|
|
||||||
|
|
||||||
private final Configuration conf;
|
private final Configuration conf;
|
||||||
|
@ -122,6 +133,24 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
|
||||||
public ZooKeeperWatcher(Configuration conf, String identifier,
|
public ZooKeeperWatcher(Configuration conf, String identifier,
|
||||||
Abortable abortable, boolean canCreateBaseZNode)
|
Abortable abortable, boolean canCreateBaseZNode)
|
||||||
throws IOException, ZooKeeperConnectionException {
|
throws IOException, ZooKeeperConnectionException {
|
||||||
|
this(conf, identifier, abortable, canCreateBaseZNode, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiate a ZooKeeper connection and watcher.
|
||||||
|
* @param conf Configuration
|
||||||
|
* @param identifier string that is passed to RecoverableZookeeper to be used as identifier for
|
||||||
|
* this instance. Use null for default.
|
||||||
|
* @param abortable Can be null if there is on error there is no host to abort: e.g. client
|
||||||
|
* context.
|
||||||
|
* @param canCreateBaseZNode whether create base node.
|
||||||
|
* @param forceAbortOnZKDisconnect abort the watcher if true.
|
||||||
|
* @throws IOException when any IO exception
|
||||||
|
* @throws ZooKeeperConnectionException when any zookeeper connection exception
|
||||||
|
*/
|
||||||
|
public ZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable,
|
||||||
|
boolean canCreateBaseZNode, boolean forceAbortOnZKDisconnect) throws IOException,
|
||||||
|
ZooKeeperConnectionException {
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.quorum = ZKConfig.getZKQuorumServersString(conf);
|
this.quorum = ZKConfig.getZKQuorumServersString(conf);
|
||||||
this.prefix = identifier;
|
this.prefix = identifier;
|
||||||
|
@ -130,6 +159,9 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
|
||||||
this.identifier = identifier + "0x0";
|
this.identifier = identifier + "0x0";
|
||||||
this.abortable = abortable;
|
this.abortable = abortable;
|
||||||
this.znodePaths = new ZNodePaths(conf);
|
this.znodePaths = new ZNodePaths(conf);
|
||||||
|
// On Disconnected event a thread will wait for sometime (2/3 of zookeeper.session.timeout),
|
||||||
|
// it will abort the process if no SyncConnected event reported by the time.
|
||||||
|
connWaitTimeOut = this.conf.getLong("zookeeper.session.timeout", 90000) * 2 / 3;
|
||||||
PendingWatcher pendingWatcher = new PendingWatcher();
|
PendingWatcher pendingWatcher = new PendingWatcher();
|
||||||
this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, pendingWatcher, identifier);
|
this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, pendingWatcher, identifier);
|
||||||
pendingWatcher.prepare(this);
|
pendingWatcher.prepare(this);
|
||||||
|
@ -146,6 +178,10 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
|
||||||
throw zce;
|
throw zce;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
this.forceAbortOnZKDisconnect = forceAbortOnZKDisconnect;
|
||||||
|
if (this.forceAbortOnZKDisconnect) {
|
||||||
|
this.zkEventWatcherExecService = Executors.newSingleThreadExecutor();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void createBaseZNodes() throws ZooKeeperConnectionException {
|
private void createBaseZNodes() throws ZooKeeperConnectionException {
|
||||||
|
@ -526,11 +562,19 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
|
||||||
Long.toHexString(this.recoverableZooKeeper.getSessionId());
|
Long.toHexString(this.recoverableZooKeeper.getSessionId());
|
||||||
// Update our identifier. Otherwise ignore.
|
// Update our identifier. Otherwise ignore.
|
||||||
LOG.debug(this.identifier + " connected");
|
LOG.debug(this.identifier + " connected");
|
||||||
|
connected.set(true);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
// Abort the server if Disconnected or Expired
|
// Abort the server if Disconnected or Expired
|
||||||
case Disconnected:
|
case Disconnected:
|
||||||
LOG.debug(prefix("Received Disconnected from ZooKeeper, ignoring"));
|
LOG.debug(prefix("Received Disconnected from ZooKeeper."));
|
||||||
|
if (forceAbortOnZKDisconnect) {
|
||||||
|
connected.set(false);
|
||||||
|
ZKDisconnectEventWatcher task = new ZKDisconnectEventWatcher();
|
||||||
|
zkEventWatcherExecService.execute(task);
|
||||||
|
} else {
|
||||||
|
LOG.debug("Received Disconnected from ZooKeeper, ignoring.");
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Expired:
|
case Expired:
|
||||||
|
@ -553,6 +597,39 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Task to watch zookeper disconnect event.
|
||||||
|
*/
|
||||||
|
class ZKDisconnectEventWatcher implements Runnable {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
if (connected.get()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
long startTime = EnvironmentEdgeManager.currentTime();
|
||||||
|
while (EnvironmentEdgeManager.currentTime() - startTime < connWaitTimeOut) {
|
||||||
|
if (connected.get()) {
|
||||||
|
LOG.debug("Client got reconnected to zookeeper.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Thread.sleep(100);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!connected.get() && abortable != null) {
|
||||||
|
String msg =
|
||||||
|
prefix("Couldn't connect to ZooKeeper after waiting " + connWaitTimeOut
|
||||||
|
+ " ms, aborting");
|
||||||
|
abortable.abort(msg, new KeeperException.ConnectionLossException());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Forces a synchronization of this ZooKeeper client connection.
|
* Forces a synchronization of this ZooKeeper client connection.
|
||||||
* <p>
|
* <p>
|
||||||
|
@ -617,6 +694,10 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
|
||||||
public void close() {
|
public void close() {
|
||||||
try {
|
try {
|
||||||
recoverableZooKeeper.close();
|
recoverableZooKeeper.close();
|
||||||
|
if (zkEventWatcherExecService != null) {
|
||||||
|
zkEventWatcherExecService.shutdown();
|
||||||
|
zkEventWatcherExecService = null;
|
||||||
|
}
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
Thread.currentThread().interrupt();
|
Thread.currentThread().interrupt();
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,14 +23,22 @@ import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.hbase.Abortable;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
|
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
|
||||||
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||||
|
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||||
|
import org.apache.zookeeper.WatchedEvent;
|
||||||
|
import org.apache.zookeeper.Watcher;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.experimental.categories.Category;
|
import org.junit.experimental.categories.Category;
|
||||||
|
|
||||||
@Category({ SmallTests.class })
|
@Category({ SmallTests.class })
|
||||||
public class TestZooKeeperWatcher {
|
public class TestZooKeeperWatcher {
|
||||||
|
private final static Log LOG = LogFactory.getLog(TestZooKeeperWatcher.class);
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testIsClientReadable() throws ZooKeeperConnectionException, IOException {
|
public void testIsClientReadable() throws ZooKeeperConnectionException, IOException {
|
||||||
|
@ -57,4 +65,43 @@ public class TestZooKeeperWatcher {
|
||||||
watcher.close();
|
watcher.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testConnectionEvent() throws ZooKeeperConnectionException, IOException {
|
||||||
|
long zkSessionTimeout = 15000l;
|
||||||
|
Configuration conf = HBaseConfiguration.create();
|
||||||
|
conf.set("zookeeper.session.timeout", "15000");
|
||||||
|
|
||||||
|
Abortable abortable = new Abortable() {
|
||||||
|
boolean aborted = false;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void abort(String why, Throwable e) {
|
||||||
|
aborted = true;
|
||||||
|
LOG.error(why, e);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAborted() {
|
||||||
|
return aborted;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
ZooKeeperWatcher watcher = new ZooKeeperWatcher(conf, "testConnectionEvent", abortable, false, true);
|
||||||
|
|
||||||
|
WatchedEvent event =
|
||||||
|
new WatchedEvent(Watcher.Event.EventType.None, Watcher.Event.KeeperState.Disconnected, null);
|
||||||
|
|
||||||
|
long startTime = EnvironmentEdgeManager.currentTime();
|
||||||
|
while (!abortable.isAborted()
|
||||||
|
&& (EnvironmentEdgeManager.currentTime() - startTime < zkSessionTimeout)) {
|
||||||
|
watcher.process(event);
|
||||||
|
try {
|
||||||
|
Thread.sleep(1000);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assertTrue(abortable.isAborted());
|
||||||
|
watcher.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -642,7 +642,7 @@ public class HRegionServer extends HasThread implements
|
||||||
if (!conf.getBoolean("hbase.testing.nocluster", false)) {
|
if (!conf.getBoolean("hbase.testing.nocluster", false)) {
|
||||||
// Open connection to zookeeper and set primary watcher
|
// Open connection to zookeeper and set primary watcher
|
||||||
zooKeeper = new ZooKeeperWatcher(conf, getProcessName() + ":" +
|
zooKeeper = new ZooKeeperWatcher(conf, getProcessName() + ":" +
|
||||||
rpcServices.isa.getPort(), this, canCreateBaseZNode());
|
rpcServices.isa.getPort(), this, canCreateBaseZNode(), true);
|
||||||
|
|
||||||
this.csm = (BaseCoordinatedStateManager) csm;
|
this.csm = (BaseCoordinatedStateManager) csm;
|
||||||
this.csm.initialize(this);
|
this.csm.initialize(this);
|
||||||
|
|
Loading…
Reference in New Issue