Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-3362 If .META. offline between OPENING and OPENED, then wrong server location in .META. is possible 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1050276 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2010-12-17 05:38:18 +00:00
parent 28012b50ce
commit d154629ef9
2 changed files with 225 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -785,6 +785,8 @@ Release 0.90.0 - Unreleased
HBASE-3360 ReplicationLogCleaner is enabled by default in 0.90 -- causes NPE
HBASE-3363 ReplicationSink should batch delete
HBASE-3365 EOFE contacting crashed RS causes Master abort
HBASE-3362 If .META. offline between OPENING and OPENED, then wrong server
location in .META. is possible
IMPROVEMENTS

338
src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
View File

@ -20,7 +20,7 @@
package org.apache.hadoop.hbase.regionserver.handler;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.util.Progressable;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.Code;
/**
* Handles opening of a region on a region server.
@ -46,6 +45,11 @@ public class OpenRegionHandler extends EventHandler {
private final HRegionInfo regionInfo;
// We get version of our znode at start of open process and monitor it across
// the total open. We'll fail the open if someone hijacks our znode; we can
// tell this has happened if version is not as expected.
private volatile int version = -1;
public OpenRegionHandler(final Server server,
final RegionServerServices rsServices, HRegionInfo regionInfo) {
this(server, rsServices, regionInfo, EventType.M_RS_OPEN_REGION);
@ -73,120 +77,32 @@ public class OpenRegionHandler extends EventHandler {
}
final String encodedName = regionInfo.getEncodedName();
// TODO: Previously we would check for root region availability (but only that it
// was initially available, does not check if it later went away)
// Do we need to wait on both root and meta to be available to open a region
// now since we edit meta?
// Check that this region is not already online
HRegion region = this.rsServices.getFromOnlineRegions(encodedName);
if (region != null) {
LOG.warn("Attempting open of " + name +
" but it's already online on this server");
LOG.warn("Attempted open of " + name +
" but already online on this server");
return;
}
int openingVersion = transitionZookeeperOfflineToOpening(encodedName);
if (openingVersion == -1) return;
// If fails, just return. Someone stole the region from under us.
// Calling transitionZookeeperOfflineToOpening initalizes this.version.
if (!transitionZookeeperOfflineToOpening(encodedName)) return;
// Open the region
final AtomicInteger openingInteger = new AtomicInteger(openingVersion);
try {
// Instantiate the region. This also periodically updates OPENING.
region = HRegion.openHRegion(regionInfo, this.rsServices.getWAL(),
server.getConfiguration(), this.rsServices,
new Progressable() {
public void progress() {
try {
int vsn = ZKAssign.retransitionNodeOpening(
server.getZooKeeper(), regionInfo, server.getServerName(),
openingInteger.get());
if (vsn == -1) {
throw KeeperException.create(Code.BADVERSION);
}
openingInteger.set(vsn);
} catch (KeeperException e) {
server.abort("ZK exception refreshing OPENING node; " + name, e);
}
}
});
} catch (IOException e) {
LOG.error("Failed open of " + regionInfo +
"; resetting state of transition node from OPENING to OFFLINE", e);
try {
// TODO: We should rely on the master timing out OPENING instead of this
// TODO: What if this was a split open? The RS made the OFFLINE
// znode, not the master.
ZKAssign.forceNodeOffline(server.getZooKeeper(), regionInfo,
server.getServerName());
} catch (KeeperException e1) {
LOG.error("Error forcing node back to OFFLINE from OPENING; " + name);
}
return;
// Open region. After a successful open, failures in subsequent processing
// needs to do a close as part of cleanup.
region = openRegion();
if (region == null) return;
boolean failed = true;
if (tickleOpening("post_region_open")) {
if (updateMeta(region)) failed = false;
}
// Region is now open. Close it if error.
// Re-transition node to OPENING again to verify no one has stomped on us
openingVersion = openingInteger.get();
try {
if ((openingVersion = ZKAssign.retransitionNodeOpening(
server.getZooKeeper(), regionInfo, server.getServerName(),
openingVersion)) == -1) {
LOG.warn("Completed the OPEN of region " + name +
" but when transitioning from " +
" OPENING to OPENING got a version mismatch, someone else clashed " +
"-- closing region");
cleanupFailedOpen(region);
return;
}
} catch (KeeperException e) {
LOG.error("Failed transitioning node " + name +
" from OPENING to OPENED -- closing region", e);
cleanupFailedOpen(region);
return;
} catch (IOException e) {
LOG.error("Failed to close region " + name +
" after failing to transition -- closing region", e);
if (failed) {
cleanupFailedOpen(region);
return;
}
// Update ZK, ROOT or META
try {
this.rsServices.postOpenDeployTasks(region,
this.server.getCatalogTracker(), false);
} catch (IOException e) {
LOG.error("Error updating " + name + " location in catalog table -- " +
"closing region", e);
cleanupFailedOpen(region);
return;
} catch (KeeperException e) {
// TODO: rollback the open?
LOG.error("ZK Error updating " + name + " location in catalog " +
"table -- closing region", e);
cleanupFailedOpen(region);
return;
}
// Finally, Transition ZK node to OPENED
try {
if (ZKAssign.transitionNodeOpened(server.getZooKeeper(), regionInfo,
server.getServerName(), openingVersion) == -1) {
LOG.warn("Completed the OPEN of region " + name +
" but when transitioning from " +
" OPENING to OPENED got a version mismatch, someone else clashed " +
"so now unassigning -- closing region");
cleanupFailedOpen(region);
return;
}
} catch (KeeperException e) {
LOG.error("Failed transitioning node " + name +
" from OPENING to OPENED -- closing region", e);
cleanupFailedOpen(region);
return;
} catch (IOException e) {
LOG.error("Failed to close " + name +
" after failing to transition -- closing region", e);
if (!transitionToOpened(region)) {
cleanupFailedOpen(region);
return;
}
@ -195,25 +111,217 @@ public class OpenRegionHandler extends EventHandler {
LOG.debug("Opened " + name);
}
/**
* Update ZK, ROOT or META. This can take a while if for example the
* .META. is not available -- if server hosting .META. crashed and we are
* waiting on it to come back -- so run in a thread and keep updating znode
* state meantime so master doesn't timeout our region-in-transition.
* Caller must cleanup region if this fails.
*/
private boolean updateMeta(final HRegion r) {
// Object we do wait/notify on. Make it boolean. If set, we're done.
// Else, wait.
final AtomicBoolean signaller = new AtomicBoolean(false);
PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r,
this.server, this.rsServices, signaller);
t.start();
int assignmentTimeout = this.server.getConfiguration().
getInt("hbase.master.assignment.timeoutmonitor.period", 10000);
// Total timeout for meta edit. If we fail adding the edit then close out
// the region and let it be assigned elsewhere.
long timeout = assignmentTimeout * 10;
long now = System.currentTimeMillis();
long endTime = now + timeout;
// Let our period at which we update OPENING state to be be 1/3rd of the
// regions-in-transition timeout period.
long period = Math.max(1, assignmentTimeout/ 3);
long lastUpdate = now;
while (!signaller.get() && t.isAlive() && !this.server.isStopped() &&
!this.rsServices.isStopping() && (endTime > now)) {
long elapsed = now - lastUpdate;
if (elapsed > period) {
// Only tickle OPENING if postOpenDeployTasks is taking some time.
lastUpdate = now;
tickleOpening("post_open_deploy");
}
synchronized (signaller) {
try {
signaller.wait(period);
} catch (InterruptedException e) {
// Go to the loop check.
}
}
now = System.currentTimeMillis();
}
// Is thread still alive? We may have left above loop because server is
// stopping or we timed out the edit. Is so, interrupt it.
if (t.isAlive()) {
LOG.debug("Interrupting thread " + t);
t.interrupt();
}
// Was there an exception opening the region? This should trigger on
// InterruptedException too. If so, we failed.
return t.getException() == null;
}
/**
* Thread to run region post open tasks. Call {@link #getException()} after
* the thread finishes to check for exceptions running
* {@link RegionServerServices#postOpenDeployTasks(HRegion, org.apache.hadoop.hbase.catalog.CatalogTracker, boolean)}.
*/
static class PostOpenDeployTasksThread extends Thread {
private Exception exception = null;
private final Server server;
private final RegionServerServices services;
private final HRegion region;
private final AtomicBoolean signaller;
PostOpenDeployTasksThread(final HRegion region, final Server server,
final RegionServerServices services, final AtomicBoolean signaller) {
super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName());
this.setDaemon(true);
this.server = server;
this.services = services;
this.region = region;
this.signaller = signaller;
}
public void run() {
try {
this.services.postOpenDeployTasks(this.region,
this.server.getCatalogTracker(), false);
} catch (Exception e) {
LOG.warn("Exception running postOpenDeployTasks; region=" +
this.region.getRegionInfo().getEncodedName(), e);
this.exception = e;
}
// We're done. Set flag then wake up anyone waiting on thread to complete.
this.signaller.set(true);
synchronized (this.signaller) {
this.signaller.notify();
}
}
/**
* @return Null or the run exception; call this method after thread is done.
*/
Exception getException() {
return this.exception;
}
}
/**
* @param r Region we're working on.
* @return Transition znode to OPENED state.
* @throws IOException
*/
private boolean transitionToOpened(final HRegion r) throws IOException {
boolean result = false;
HRegionInfo hri = r.getRegionInfo();
final String name = hri.getRegionNameAsString();
// Finally, Transition ZK node to OPENED
try {
if (ZKAssign.transitionNodeOpened(this.server.getZooKeeper(), hri,
this.server.getServerName(), this.version) == -1) {
LOG.warn("Completed the OPEN of region " + name +
" but when transitioning from " +
" OPENING to OPENED got a version mismatch, someone else clashed " +
"so now unassigning -- closing region");
} else {
result = true;
}
} catch (KeeperException e) {
LOG.error("Failed transitioning node " + name +
" from OPENING to OPENED -- closing region", e);
}
return result;
}
/**
* @return Instance of HRegion if successful open else null.
*/
private HRegion openRegion() {
HRegion region = null;
try {
// Instantiate the region. This also periodically tickles our zk OPENING
// state so master doesn't timeout this region in transition.
region = HRegion.openHRegion(this.regionInfo, this.rsServices.getWAL(),
this.server.getConfiguration(), this.rsServices,
new Progressable() {
public void progress() {
// We may lose the znode ownership during the open. Currently its
// too hard interrupting ongoing region open. Just let it complete
// and check we still have the znode after region open.
tickleOpening("open_region_progress");
}
});
} catch (IOException e) {
// We failed open. Let our znode expire in regions-in-transition and
// Master will assign elsewhere. Presumes nothing to close.
LOG.error("Failed open of region=" +
this.regionInfo.getRegionNameAsString(), e);
}
return region;
}
private void cleanupFailedOpen(final HRegion region) throws IOException {
if (region != null) region.close();
this.rsServices.removeFromOnlineRegions(regionInfo.getEncodedName());
}
int transitionZookeeperOfflineToOpening(final String encodedName) {
// Transition ZK node from OFFLINE to OPENING
/**
* Transition ZK node from OFFLINE to OPENING.
* @param encodedName Name of the znode file (Region encodedName is the znode
* name).
* @return True if successful transition.
*/
boolean transitionZookeeperOfflineToOpening(final String encodedName) {
// TODO: should also handle transition from CLOSED?
int openingVersion = -1;
try {
if ((openingVersion = ZKAssign.transitionNodeOpening(server.getZooKeeper(),
regionInfo, server.getServerName())) == -1) {
LOG.warn("Error transitioning node from OFFLINE to OPENING, " +
"aborting open");
}
// Initialize the znode version.
this.version =
ZKAssign.transitionNodeOpening(server.getZooKeeper(),
regionInfo, server.getServerName());
} catch (KeeperException e) {
LOG.error("Error transitioning node from OFFLINE to OPENING for region " +
LOG.error("Error transition from OFFLINE to OPENING for region=" +
encodedName, e);
}
return openingVersion;
boolean b = isGoodVersion();
if (!b) {
LOG.warn("Failed transition from OFFLINE to OPENING for region=" +
encodedName);
}
return b;
}
}
/**
* Update our OPENING state in zookeeper.
* Do this so master doesn't timeout this region-in-transition.
* @param context Some context to add to logs if failure
* @return True if successful transition.
*/
boolean tickleOpening(final String context) {
// If previous checks failed... do not try again.
if (!isGoodVersion()) return false;
String encodedName = this.regionInfo.getEncodedName();
try {
this.version =
ZKAssign.retransitionNodeOpening(server.getZooKeeper(),
this.regionInfo, this.server.getServerName(), this.version);
} catch (KeeperException e) {
server.abort("Exception refreshing OPENING; region=" + encodedName +
", context=" + context, e);
}
boolean b = isGoodVersion();
if (!b) {
LOG.warn("Failed refreshing OPENING; region=" + encodedName +
", context=" + context);
}
return b;
}
private boolean isGoodVersion() {
return this.version != -1;
}
}