diff --git a/CHANGES.txt b/CHANGES.txt index efb6138b2bc..12e86f2c6af 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -154,6 +154,8 @@ Release 0.20.0 - Unreleased localhost_1237525439599_56094" <- You'd have to be perverse to recognize that as a hostname, startcode, and port HBASE-1395 InfoServers no longer put up a UI + HBASE-1302 When a new master comes up, regionservers should continue with + their region assignments from the last master IMPROVEMENTS HBASE-1089 Add count of regions on filesystem to master UI; add percentage diff --git a/src/java/org/apache/hadoop/hbase/client/HConnection.java b/src/java/org/apache/hadoop/hbase/client/HConnection.java index 421afe0e257..44033ad6d20 100644 --- a/src/java/org/apache/hadoop/hbase/client/HConnection.java +++ b/src/java/org/apache/hadoop/hbase/client/HConnection.java @@ -133,6 +133,17 @@ public interface HConnection { public HRegionInterface getHRegionConnection(HServerAddress regionServer) throws IOException; + /** + * Establishes a connection to the region server at the specified address. + * @param regionServer - the server to connect to + * @param getMaster - do we check if master is alive + * @return proxy for HRegionServer + * @throws IOException + */ + public HRegionInterface getHRegionConnection( + HServerAddress regionServer, boolean getMaster) + throws IOException; + /** * Find region location hosting passed row * @param tableName diff --git a/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java index aff8644a31c..6a15b557495 100644 --- a/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ b/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -116,6 +116,7 @@ public class HConnectionManager implements HConstants { } } + /* Encapsulates finding the servers for an HBase instance */ private static class TableServers implements ServerConnection, HConstants, Watcher { private static final Log LOG = LogFactory.getLog(TableServers.class); @@ -766,9 +767,12 @@ public class HConnectionManager implements HConstants { tableLocations.put(startKey, location); } - public HRegionInterface getHRegionConnection(HServerAddress regionServer) + public HRegionInterface getHRegionConnection( + HServerAddress regionServer, boolean getMaster) throws IOException { - getMaster(); + if(getMaster) { + getMaster(); + } HRegionInterface server; synchronized (this.servers) { // See if we already have a connection @@ -787,6 +791,12 @@ public class HConnectionManager implements HConstants { } return server; } + + public HRegionInterface getHRegionConnection( + HServerAddress regionServer) + throws IOException { + return getHRegionConnection(regionServer, true); + } public synchronized ZooKeeperWrapper getZooKeeperWrapper() throws IOException { if (zooKeeperWrapper == null) { diff --git a/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java b/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java index 904d859b2c2..9578fc1093b 100644 --- a/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java +++ b/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java @@ -69,7 +69,8 @@ public interface HBaseRPCProtocolVersion extends VersionedProtocol { * HMasterInterface.findRootRegion. We use ZooKeeper to store root region * location instead. *
  • Version 17: Added incrementColumnValue.
    • + *
  • Version 18: HBASE-1302.
    • * */ - public static final long versionID = 17L; + public static final long versionID = 18L; } diff --git a/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java index f26dee89f08..66885cfd267 100644 --- a/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java +++ b/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.io.RowResult; import org.apache.hadoop.hbase.io.HbaseMapWritable; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerInfo; import org.apache.hadoop.hbase.NotServingRegionException; /** @@ -306,4 +307,18 @@ public interface HRegionInterface extends HBaseRPCProtocolVersion { */ public long incrementColumnValue(byte [] regionName, byte [] row, byte [] column, long amount) throws IOException; + + /** + * Method used when a master is taking the place of another failed one. + * @return All regions assigned on this region server + * @throws IOException + */ + public HRegionInfo[] getRegionsAssignment() throws IOException; + + /** + * Method used when a master is taking the place of another failed one. + * @return The HSI + * @throws IOException + */ + public HServerInfo getHServerInfo() throws IOException; } diff --git a/src/java/org/apache/hadoop/hbase/master/HMaster.java b/src/java/org/apache/hadoop/hbase/master/HMaster.java index f3cc1db60dd..027793d68c1 100644 --- a/src/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/java/org/apache/hadoop/hbase/master/HMaster.java @@ -25,6 +25,7 @@ import java.lang.management.RuntimeMXBean; import java.lang.reflect.Constructor; import java.net.InetAddress; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -44,6 +45,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HMsg; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HServerInfo; import org.apache.hadoop.hbase.HServerLoad; @@ -374,6 +376,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface, public void run() { final String threadName = "HMaster"; Thread.currentThread().setName(threadName); + verifyClusterState(); startServiceThreads(); /* Main processing loop */ try { @@ -503,6 +506,61 @@ public class HMaster extends Thread implements HConstants, HMasterInterface, return true; } + /* + * Verifies if this instance of HBase is fresh or the master was started + * following a failover. In the second case, it inspects the region server + * directory and gets their regions assignment. + */ + private void verifyClusterState() { + try { + LOG.debug("Checking cluster state..."); + HServerAddress rootLocation = zooKeeperWrapper.readRootRegionLocation(); + List addresses = zooKeeperWrapper.scanRSDirectory(); + + // Check if this is a fresh start of the cluster + if(addresses.size() == 0) { + LOG.debug("This is a fresh start, proceeding with normal startup"); + return; + } + LOG.info("This is a failover, ZK inspection begins..."); + boolean isRootRegionAssigned = false; + Map assignedRegions = + new HashMap(); + // This is a failover case. We must: + // - contact every region server to add them to the regionservers list + // - get their current regions assignment + for (HServerAddress address : addresses) { + HRegionInterface hri = + this.connection.getHRegionConnection(address, false); + HServerInfo info = hri.getHServerInfo(); + LOG.debug("Inspection found server " + info.getName()); + serverManager.recordNewServer(info); + HRegionInfo[] regions = hri.getRegionsAssignment(); + for (HRegionInfo region : regions) { + if(region.isRootRegion()) { + connection.setRootRegionLocation( + new HRegionLocation(region, rootLocation)); + regionManager.setRootRegionLocation(rootLocation); + // Undo the unassign work in the RegionManager constructor + regionManager.removeRegion(region); + isRootRegionAssigned = true; + } + else if(region.isMetaRegion()) { + MetaRegion m = + new MetaRegion(new HServerAddress(address), + region.getRegionName(), region.getStartKey()); + regionManager.addMetaRegionToScan(m); + } + assignedRegions.put(region.getRegionName(), region); + } + } + LOG.info("Inspection found " + assignedRegions.size() + " regions, " + + (isRootRegionAssigned ? "with -ROOT-" : "but -ROOT- was MIA")); + } catch(IOException ex) { + ex.printStackTrace(); + } + } + /* * Start up all services. If any of these threads gets an unhandled exception * then they just die with a logged message. This should be fine because diff --git a/src/java/org/apache/hadoop/hbase/master/RegionManager.java b/src/java/org/apache/hadoop/hbase/master/RegionManager.java index 8977060a094..0881d570c86 100644 --- a/src/java/org/apache/hadoop/hbase/master/RegionManager.java +++ b/src/java/org/apache/hadoop/hbase/master/RegionManager.java @@ -554,6 +554,7 @@ class RegionManager implements HConstants { } catch(Exception iex) { LOG.warn("meta scanner", iex); } + zooKeeperWrapper.clearRSDirectory(); zooKeeperWrapper.close(); } diff --git a/src/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/java/org/apache/hadoop/hbase/master/ServerManager.java index 3c8803d58b9..7e4e996f3a5 100644 --- a/src/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/src/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -161,8 +161,6 @@ class ServerManager implements HConstants { LOG.debug("deadServers.contains: " + deadServers.contains(serverName)); throw new Leases.LeaseStillHeldException(serverName); } - Watcher watcher = new ServerExpirer(serverName, info.getServerAddress()); - zooKeeperWrapper.updateRSLocationGetWatch(info, watcher); LOG.info("Received start message from: " + serverName); // Go on to process the regionserver registration. @@ -198,9 +196,21 @@ class ServerManager implements HConstants { LOG.error("Insertion into toDoQueue was interrupted", e); } } - // record new server - load = new HServerLoad(); + recordNewServer(info); + } + + /** + * Adds the HSI to the RS list + * @param info The region server informations + */ + public void recordNewServer(HServerInfo info) { + HServerLoad load = new HServerLoad(); + String serverName = HServerInfo.getServerName(info); info.setLoad(load); + // We must set this watcher here because it can be set on a fresh start + // or on a failover + Watcher watcher = new ServerExpirer(serverName, info.getServerAddress()); + zooKeeperWrapper.updateRSLocationGetWatch(info, watcher); serversToServerInfo.put(serverName, info); serverAddressToServerInfo.put(info.getServerAddress(), info); serversToLoad.put(serverName, load); diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 43435f87b17..b08e56d1447 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -323,15 +323,6 @@ public class HRegionServer implements HConstants, HRegionInterface, private void reinitializeZooKeeper() throws IOException { zooKeeperWrapper = new ZooKeeperWrapper(conf); watchMasterAddress(); - - boolean startCodeOk = false; - while(!startCodeOk) { - serverInfo.setStartCode(System.currentTimeMillis()); - startCodeOk = zooKeeperWrapper.writeRSLocation(serverInfo); - if(!startCodeOk) { - LOG.debug("Start code already taken, trying another one"); - } - } } private void reinitializeThreads() { @@ -384,6 +375,8 @@ public class HRegionServer implements HConstants, HRegionInterface, if (state == KeeperState.Expired) { LOG.error("ZooKeeper session expired"); restart(); + } else if (type == EventType.NodeDeleted) { + watchMasterAddress(); } else if (type == EventType.NodeCreated) { getMaster(); @@ -1330,6 +1323,14 @@ public class HRegionServer implements HConstants, HRegionInterface, if (LOG.isDebugEnabled()) LOG.debug("sending initial server load: " + hsl); lastMsg = System.currentTimeMillis(); + boolean startCodeOk = false; + while(!startCodeOk) { + serverInfo.setStartCode(System.currentTimeMillis()); + startCodeOk = zooKeeperWrapper.writeRSLocation(serverInfo); + if(!startCodeOk) { + LOG.debug("Start code already taken, trying another one"); + } + } result = this.hbaseMaster.regionServerStartup(serverInfo); break; } catch (Leases.LeaseStillHeldException e) { @@ -2451,7 +2452,20 @@ public class HRegionServer implements HConstants, HRegionInterface, checkFileSystem(); throw e; } - - + } + + /** {@inheritDoc} */ + public HRegionInfo[] getRegionsAssignment() throws IOException { + HRegionInfo[] regions = new HRegionInfo[onlineRegions.size()]; + Iterator ite = onlineRegions.values().iterator(); + for(int i = 0; ite.hasNext(); i++) { + regions[i] = ite.next().getRegionInfo(); + } + return regions; + } + + /** {@inheritDoc} */ + public HServerInfo getHServerInfo() throws IOException { + return serverInfo; } } diff --git a/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java b/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java index a8e1f5e3141..9f686bcf04b 100644 --- a/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java +++ b/src/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java @@ -462,12 +462,12 @@ public class ZooKeeperWrapper implements HConstants { */ public boolean writeRSLocation(HServerInfo info) { ensureExists(rsZNode); - byte[] data = Bytes.toBytes(info.getServerAddress().getBindAddress()); + byte[] data = Bytes.toBytes(info.getServerAddress().toString()); String znode = joinPath(rsZNode, Long.toString(info.getStartCode())); try { zooKeeper.create(znode, data, Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); LOG.debug("Created ZNode " + znode - + " with data " + info.getServerAddress().getBindAddress()); + + " with data " + info.getServerAddress().toString()); return true; } catch (KeeperException e) { LOG.warn("Failed to create " + znode + " znode in ZooKeeper: " + e); @@ -484,12 +484,12 @@ public class ZooKeeperWrapper implements HConstants { * @return true if the update is done, false if it failed */ public boolean updateRSLocationGetWatch(HServerInfo info, Watcher watcher) { - byte[] data = Bytes.toBytes(info.getServerAddress().getBindAddress()); - String znode = rsZNode + "/" + info.getStartCode(); + byte[] data = Bytes.toBytes(info.getServerAddress().toString()); + String znode = rsZNode + ZNODE_PATH_SEPARATOR + info.getStartCode(); try { zooKeeper.setData(znode, data, -1); LOG.debug("Updated ZNode " + znode - + " with data " + info.getServerAddress().getBindAddress()); + + " with data " + info.getServerAddress().toString()); zooKeeper.getData(znode, watcher, null); return true; } catch (KeeperException e) { @@ -501,6 +501,43 @@ public class ZooKeeperWrapper implements HConstants { return false; } + /** + * Scans the regions servers directory + * @return A list of server addresses + */ + public List scanRSDirectory() { + List addresses = new ArrayList(); + try { + List nodes = zooKeeper.getChildren(rsZNode, false); + for (String node : nodes) { + addresses.add(readAddress(rsZNode + ZNODE_PATH_SEPARATOR + node, null)); + } + } catch (KeeperException e) { + LOG.warn("Failed to read " + rsZNode + " znode in ZooKeeper: " + e); + } catch (InterruptedException e) { + LOG.warn("Failed to read " + rsZNode + " znode in ZooKeeper: " + e); + } + return addresses; + } + + /** + * Method used to make sure the region server directory is empty. + * + */ + public void clearRSDirectory() { + try { + List nodes = zooKeeper.getChildren(rsZNode, false); + for (String node : nodes) { + LOG.debug("Deleting node: " + node); + zooKeeper.delete(node, -1); + } + } catch (KeeperException e) { + LOG.warn("Failed to delete " + rsZNode + " znode in ZooKeeper: " + e); + } catch (InterruptedException e) { + LOG.warn("Failed to delete " + rsZNode + " znode in ZooKeeper: " + e); + } + } + private boolean checkExistenceOf(String path) { Stat stat = null; try {