Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-1302 When a new master comes up, regionservers should continue with 

their region assignments from the last master



git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@778819 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jean-Daniel Cryans 2009-05-26 18:01:56 +00:00
parent 8018305551
commit 0f0e4579a9
10 changed files with 182 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -154,6 +154,8 @@ Release 0.20.0 - Unreleased
localhost_1237525439599_56094" <- You'd have to be perverse
to recognize that as a hostname, startcode, and port
HBASE-1395 InfoServers no longer put up a UI
HBASE-1302 When a new master comes up, regionservers should continue with
their region assignments from the last master
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage

11
src/java/org/apache/hadoop/hbase/client/HConnection.java
View File

@ -133,6 +133,17 @@ public interface HConnection {
public HRegionInterface getHRegionConnection(HServerAddress regionServer)
throws IOException;
/**
* Establishes a connection to the region server at the specified address.
* @param regionServer - the server to connect to
* @param getMaster - do we check if master is alive
* @return proxy for HRegionServer
* @throws IOException
*/
public HRegionInterface getHRegionConnection(
HServerAddress regionServer, boolean getMaster)
throws IOException;
/**
* Find region location hosting passed row
* @param tableName

12
src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
View File

@ -116,6 +116,7 @@ public class HConnectionManager implements HConstants {
}
}
/* Encapsulates finding the servers for an HBase instance */
private static class TableServers implements ServerConnection, HConstants, Watcher {
private static final Log LOG = LogFactory.getLog(TableServers.class);
@ -766,9 +767,12 @@ public class HConnectionManager implements HConstants {
tableLocations.put(startKey, location);
}
public HRegionInterface getHRegionConnection(HServerAddress regionServer)
public HRegionInterface getHRegionConnection(
HServerAddress regionServer, boolean getMaster)
throws IOException {
if(getMaster) {
getMaster();
}
HRegionInterface server;
synchronized (this.servers) {
// See if we already have a connection
@ -788,6 +792,12 @@ public class HConnectionManager implements HConstants {
return server;
}
public HRegionInterface getHRegionConnection(
HServerAddress regionServer)
throws IOException {
return getHRegionConnection(regionServer, true);
}
public synchronized ZooKeeperWrapper getZooKeeperWrapper() throws IOException {
if (zooKeeperWrapper == null) {
zooKeeperWrapper = new ZooKeeperWrapper(conf, this);

3
src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
View File

@ -69,7 +69,8 @@ public interface HBaseRPCProtocolVersion extends VersionedProtocol {
* HMasterInterface.findRootRegion. We use ZooKeeper to store root region
* location instead.</li>
* <li>Version 17: Added incrementColumnValue.</li>
* <li>Version 18: HBASE-1302.</li>
* </ul>
*/
public static final long versionID = 17L;
public static final long versionID = 18L;
}

15
src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.NotServingRegionException;
/**
@ -306,4 +307,18 @@ public interface HRegionInterface extends HBaseRPCProtocolVersion {
*/
public long incrementColumnValue(byte [] regionName, byte [] row,
byte [] column, long amount) throws IOException;
/**
* Method used when a master is taking the place of another failed one.
* @return All regions assigned on this region server
* @throws IOException
*/
public HRegionInfo[] getRegionsAssignment() throws IOException;
/**
* Method used when a master is taking the place of another failed one.
* @return The HSI
* @throws IOException
*/
public HServerInfo getHServerInfo() throws IOException;
}

58
src/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -25,6 +25,7 @@ import java.lang.management.RuntimeMXBean;
import java.lang.reflect.Constructor;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
@ -44,6 +45,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HMsg;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.HServerLoad;
@ -374,6 +376,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
public void run() {
final String threadName = "HMaster";
Thread.currentThread().setName(threadName);
verifyClusterState();
startServiceThreads();
/* Main processing loop */
try {
@ -503,6 +506,61 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
return true;
}
/*
* Verifies if this instance of HBase is fresh or the master was started
* following a failover. In the second case, it inspects the region server
* directory and gets their regions assignment.
*/
private void verifyClusterState() {
try {
LOG.debug("Checking cluster state...");
HServerAddress rootLocation = zooKeeperWrapper.readRootRegionLocation();
List<HServerAddress> addresses = zooKeeperWrapper.scanRSDirectory();
// Check if this is a fresh start of the cluster
if(addresses.size() == 0) {
LOG.debug("This is a fresh start, proceeding with normal startup");
return;
}
LOG.info("This is a failover, ZK inspection begins...");
boolean isRootRegionAssigned = false;
Map<byte[], HRegionInfo> assignedRegions =
new HashMap<byte[], HRegionInfo>();
// This is a failover case. We must:
// - contact every region server to add them to the regionservers list
// - get their current regions assignment
for (HServerAddress address : addresses) {
HRegionInterface hri =
this.connection.getHRegionConnection(address, false);
HServerInfo info = hri.getHServerInfo();
LOG.debug("Inspection found server " + info.getName());
serverManager.recordNewServer(info);
HRegionInfo[] regions = hri.getRegionsAssignment();
for (HRegionInfo region : regions) {
if(region.isRootRegion()) {
connection.setRootRegionLocation(
new HRegionLocation(region, rootLocation));
regionManager.setRootRegionLocation(rootLocation);
// Undo the unassign work in the RegionManager constructor
regionManager.removeRegion(region);
isRootRegionAssigned = true;
}
else if(region.isMetaRegion()) {
MetaRegion m =
new MetaRegion(new HServerAddress(address),
region.getRegionName(), region.getStartKey());
regionManager.addMetaRegionToScan(m);
}
assignedRegions.put(region.getRegionName(), region);
}
}
LOG.info("Inspection found " + assignedRegions.size() + " regions, " +
(isRootRegionAssigned ? "with -ROOT-" : "but -ROOT- was MIA"));
} catch(IOException ex) {
ex.printStackTrace();
}
}
/*
* Start up all services. If any of these threads gets an unhandled exception
* then they just die with a logged message. This should be fine because

1
src/java/org/apache/hadoop/hbase/master/RegionManager.java
View File

@ -554,6 +554,7 @@ class RegionManager implements HConstants {
} catch(Exception iex) {
LOG.warn("meta scanner", iex);
}
zooKeeperWrapper.clearRSDirectory();
zooKeeperWrapper.close();
}

18
src/java/org/apache/hadoop/hbase/master/ServerManager.java
View File

@ -161,8 +161,6 @@ class ServerManager implements HConstants {
LOG.debug("deadServers.contains: " + deadServers.contains(serverName));
throw new Leases.LeaseStillHeldException(serverName);
}
Watcher watcher = new ServerExpirer(serverName, info.getServerAddress());
zooKeeperWrapper.updateRSLocationGetWatch(info, watcher);
LOG.info("Received start message from: " + serverName);
// Go on to process the regionserver registration.
@ -198,9 +196,21 @@ class ServerManager implements HConstants {
LOG.error("Insertion into toDoQueue was interrupted", e);
}
}
// record new server
load = new HServerLoad();
recordNewServer(info);
}
/**
* Adds the HSI to the RS list
* @param info The region server informations
*/
public void recordNewServer(HServerInfo info) {
HServerLoad load = new HServerLoad();
String serverName = HServerInfo.getServerName(info);
info.setLoad(load);
// We must set this watcher here because it can be set on a fresh start
// or on a failover
Watcher watcher = new ServerExpirer(serverName, info.getServerAddress());
zooKeeperWrapper.updateRSLocationGetWatch(info, watcher);
serversToServerInfo.put(serverName, info);
serverAddressToServerInfo.put(info.getServerAddress(), info);
serversToLoad.put(serverName, load);

32
src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -323,15 +323,6 @@ public class HRegionServer implements HConstants, HRegionInterface,
private void reinitializeZooKeeper() throws IOException {
zooKeeperWrapper = new ZooKeeperWrapper(conf);
watchMasterAddress();
boolean startCodeOk = false;
while(!startCodeOk) {
serverInfo.setStartCode(System.currentTimeMillis());
startCodeOk = zooKeeperWrapper.writeRSLocation(serverInfo);
if(!startCodeOk) {
LOG.debug("Start code already taken, trying another one");
}
}
}
private void reinitializeThreads() {
@ -384,6 +375,8 @@ public class HRegionServer implements HConstants, HRegionInterface,
if (state == KeeperState.Expired) {
LOG.error("ZooKeeper session expired");
restart();
} else if (type == EventType.NodeDeleted) {
watchMasterAddress();
} else if (type == EventType.NodeCreated) {
getMaster();
@ -1330,6 +1323,14 @@ public class HRegionServer implements HConstants, HRegionInterface,
if (LOG.isDebugEnabled())
LOG.debug("sending initial server load: " + hsl);
lastMsg = System.currentTimeMillis();
boolean startCodeOk = false;
while(!startCodeOk) {
serverInfo.setStartCode(System.currentTimeMillis());
startCodeOk = zooKeeperWrapper.writeRSLocation(serverInfo);
if(!startCodeOk) {
LOG.debug("Start code already taken, trying another one");
}
}
result = this.hbaseMaster.regionServerStartup(serverInfo);
break;
} catch (Leases.LeaseStillHeldException e) {
@ -2451,7 +2452,20 @@ public class HRegionServer implements HConstants, HRegionInterface,
checkFileSystem();
throw e;
}
}
/** {@inheritDoc} */
public HRegionInfo[] getRegionsAssignment() throws IOException {
HRegionInfo[] regions = new HRegionInfo[onlineRegions.size()];
Iterator<HRegion> ite = onlineRegions.values().iterator();
for(int i = 0; ite.hasNext(); i++) {
regions[i] = ite.next().getRegionInfo();
}
return regions;
}
/** {@inheritDoc} */
public HServerInfo getHServerInfo() throws IOException {
return serverInfo;
}
}

47
src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
View File

@ -462,12 +462,12 @@ public class ZooKeeperWrapper implements HConstants {
*/
public boolean writeRSLocation(HServerInfo info) {
ensureExists(rsZNode);
byte[] data = Bytes.toBytes(info.getServerAddress().getBindAddress());
byte[] data = Bytes.toBytes(info.getServerAddress().toString());
String znode = joinPath(rsZNode, Long.toString(info.getStartCode()));
try {
zooKeeper.create(znode, data, Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
LOG.debug("Created ZNode " + znode
+ " with data " + info.getServerAddress().getBindAddress());
+ " with data " + info.getServerAddress().toString());
return true;
} catch (KeeperException e) {
LOG.warn("Failed to create " + znode + " znode in ZooKeeper: " + e);
@ -484,12 +484,12 @@ public class ZooKeeperWrapper implements HConstants {
* @return true if the update is done, false if it failed
*/
public boolean updateRSLocationGetWatch(HServerInfo info, Watcher watcher) {
byte[] data = Bytes.toBytes(info.getServerAddress().getBindAddress());
String znode = rsZNode + "/" + info.getStartCode();
byte[] data = Bytes.toBytes(info.getServerAddress().toString());
String znode = rsZNode + ZNODE_PATH_SEPARATOR + info.getStartCode();
try {
zooKeeper.setData(znode, data, -1);
LOG.debug("Updated ZNode " + znode
+ " with data " + info.getServerAddress().getBindAddress());
+ " with data " + info.getServerAddress().toString());
zooKeeper.getData(znode, watcher, null);
return true;
} catch (KeeperException e) {
@ -501,6 +501,43 @@ public class ZooKeeperWrapper implements HConstants {
return false;
}
/**
* Scans the regions servers directory
* @return A list of server addresses
*/
public List<HServerAddress> scanRSDirectory() {
List<HServerAddress> addresses = new ArrayList<HServerAddress>();
try {
List<String> nodes = zooKeeper.getChildren(rsZNode, false);
for (String node : nodes) {
addresses.add(readAddress(rsZNode + ZNODE_PATH_SEPARATOR + node, null));
}
} catch (KeeperException e) {
LOG.warn("Failed to read " + rsZNode + " znode in ZooKeeper: " + e);
} catch (InterruptedException e) {
LOG.warn("Failed to read " + rsZNode + " znode in ZooKeeper: " + e);
}
return addresses;
}
/**
* Method used to make sure the region server directory is empty.
*
*/
public void clearRSDirectory() {
try {
List<String> nodes = zooKeeper.getChildren(rsZNode, false);
for (String node : nodes) {
LOG.debug("Deleting node: " + node);
zooKeeper.delete(node, -1);
}
} catch (KeeperException e) {
LOG.warn("Failed to delete " + rsZNode + " znode in ZooKeeper: " + e);
} catch (InterruptedException e) {
LOG.warn("Failed to delete " + rsZNode + " znode in ZooKeeper: " + e);
}
}
private boolean checkExistenceOf(String path) {
Stat stat = null;
try {