Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-10815 Master regionserver should be rolling-upgradable 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1583373 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
jxiang 2014-03-31 16:42:04 +00:00
parent ea78f39854
commit bc42dc04ac
10 changed files with 216 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
hbase-common/src/main/resources/hbase-default.xml
View File

@ -96,11 +96,6 @@ possible configurations would overwhelm and obscure the important.
</property>
<!--Master configurations-->
<property >
<name>hbase.master.port</name>
<value>16000</value>
<description>The port the HBase Master should bind to.</description>
</property>
<property>
<name>hbase.master.info.port</name>
<value>16010</value>
@ -147,23 +142,11 @@ possible configurations would overwhelm and obscure the important.
META.</description>
</property>
<property>
<name>fail.fast.expired.active.master</name>
<value>false</value>
<description>If abort immediately for the expired master without trying
to recover its zk session.</description>
</property>
<property>
<name>hbase.master.dns.interface</name>
<value>default</value>
<description>The name of the Network Interface from which a master
should report its IP address.</description>
</property>
<property>
<name>hbase.master.dns.nameserver</name>
<value>default</value>
<description>The host name or IP address of the name server (DNS)
which a master should use to determine the host name used
for communication and display purposes.</description>
<name>hbase.master.infoserver.redirect</name>
<value>true</value>
<description>Whether or not the Master listens to the Master web
UI port (hbase.master.info.port) and redirects requests to the web
UI server shared by the Master and RegionServer.</description>
</property>
<!--RegionServer configurations-->
@ -344,7 +327,7 @@ possible configurations would overwhelm and obscure the important.
<name>hbase.zookeeper.peerport</name>
<value>2888</value>
<description>Port used by ZooKeeper peers to talk to each other.
Seehttp://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper
See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper
for more information.</description>
</property>
<property>
@ -513,6 +496,12 @@ possible configurations would overwhelm and obscure the important.
<value>300000</value>
<description>Period at which the region balancer runs in the Master.</description>
</property>
<property>
<name>hbase.balancer.use-backupmaster</name>
<value>true</value>
<description>Whether or not the region balancer uses the backup Masters
as regionservers, and assigns regions to them.</description>
</property>
<property>
<name>hbase.regions.slop</name>
<value>0.2</value>

15
hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
View File

@ -148,13 +148,13 @@ public class ActiveMasterManager extends ZooKeeperListener {
*/
boolean blockUntilBecomingActiveMaster(
int checkInterval, MonitoredTask startupStatus) {
String backupZNode = ZKUtil.joinZNode(
this.watcher.backupMasterAddressesZNode, this.sn.toString());
while (!(master.isAborted() || master.isStopped())) {
startupStatus.setStatus("Trying to register in ZK as active master");
// Try to become the active master, watch if there is another master.
// Write out our ServerName as versioned bytes.
try {
String backupZNode =
ZKUtil.joinZNode(this.watcher.backupMasterAddressesZNode, this.sn.toString());
if (MasterAddressTracker.setMasterAddress(this.watcher,
this.watcher.getMasterAddressZNode(), this.sn)) {
@ -178,17 +178,6 @@ public class ActiveMasterManager extends ZooKeeperListener {
// and the master ephemeral node has not expired yet.
this.clusterHasActiveMaster.set(true);
/*
* Add a ZNode for ourselves in the backup master directory since we are
* not the active master.
*
* If we become the active master later, ActiveMasterManager will delete
* this node explicitly. If we crash before then, ZooKeeper will delete
* this node for us since it is ephemeral.
*/
LOG.info("Adding ZNode for " + backupZNode + " in backup master directory");
MasterAddressTracker.setMasterAddress(this.watcher, backupZNode, this.sn);
String msg;
byte[] bytes =
ZKUtil.getDataAndWatch(this.watcher, this.watcher.getMasterAddressZNode());

87
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -34,7 +34,10 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -107,12 +110,16 @@ import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.nio.SelectChannelConnector;
import org.mortbay.jetty.servlet.Context;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@ -214,6 +221,23 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
/** flag used in test cases in order to simulate RS failures during master initialization */
private volatile boolean initializationBeforeMetaAssignment = false;
/** jetty server for master to redirect requests to regionserver infoServer */
private org.mortbay.jetty.Server masterJettyServer;
public static class RedirectServlet extends HttpServlet {
private static final long serialVersionUID = 2894774810058302472L;
private static int regionServerInfoPort;
@Override
public void doGet(HttpServletRequest request,
HttpServletResponse response) throws ServletException, IOException {
String redirectUrl = request.getScheme() + "://"
+ request.getServerName() + ":" + regionServerInfoPort
+ request.getRequestURI();
response.sendRedirect(redirectUrl);
}
}
/**
* Initializes the HMaster. The steps are as follows:
* <p>
@ -271,6 +295,34 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
}
}
startActiveMasterManager();
putUpJettyServer();
}
private void putUpJettyServer() throws IOException {
if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) {
return;
}
int infoPort = conf.getInt("hbase.master.info.port.orig",
HConstants.DEFAULT_MASTER_INFOPORT);
// -1 is for disabling info server, so no redirecting
if (infoPort < 0 || infoServer == null) {
return;
}
RedirectServlet.regionServerInfoPort = infoServer.getPort();
masterJettyServer = new org.mortbay.jetty.Server();
Connector connector = new SelectChannelConnector();
connector.setHost(conf.get("hbase.master.info.bindAddress", "0.0.0.0"));
connector.setPort(infoPort);
masterJettyServer.addConnector(connector);
masterJettyServer.setStopAtShutdown(true);
Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS);
context.addServlet(RedirectServlet.class, "/*");
try {
masterJettyServer.start();
} catch (Exception e) {
throw new IOException("Failed to start redirecting jetty server", e);
}
}
/**
@ -479,11 +531,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
this.initializationBeforeMetaAssignment = true;
//initialize load balancer
this.balancer.setClusterStatus(getClusterStatus());
this.balancer.setMasterServices(this);
this.balancer.initialize();
// Wait for regionserver to finish initialization.
while (!isOnline()) {
synchronized (online) {
@ -491,6 +538,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
}
}
//initialize load balancer
this.balancer.setClusterStatus(getClusterStatus());
this.balancer.setMasterServices(this);
this.balancer.initialize();
// Make sure meta assigned before proceeding.
status.setStatus("Assigning Meta Region");
assignMeta(status, previouslyFailedMetaRSs);
@ -785,6 +837,14 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
}
protected void stopServiceThreads() {
if (masterJettyServer != null) {
LOG.info("Stopping master jetty server");
try {
masterJettyServer.stop();
} catch (Exception e) {
LOG.error("Failed to stop master jetty server", e);
}
}
super.stopServiceThreads();
stopChores();
// Wait for all the remaining region servers to report in IFF we were
@ -1136,7 +1196,22 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
}
}
private void startActiveMasterManager() {
private void startActiveMasterManager() throws KeeperException {
String backupZNode = ZKUtil.joinZNode(
zooKeeper.backupMasterAddressesZNode, serverName.toString());
/*
* Add a ZNode for ourselves in the backup master directory since we
* may not become the active master. If so, we want the actual active
* master to know we are backup masters, so that it won't assign
* regions to us if so configured.
*
* If we become the active master later, ActiveMasterManager will delete
* this node explicitly. If we crash before then, ZooKeeper will delete
* this node for us since it is ephemeral.
*/
LOG.info("Adding ZNode for " + backupZNode + " in backup master directory");
MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName);
activeMasterManager = new ActiveMasterManager(zooKeeper, serverName, this);
// Start a thread to try to become the active master, so we won't block here
Threads.setDaemonThreadRunning(new Thread(new Runnable() {

2
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
View File

@ -178,7 +178,7 @@ public class HMasterCommandLine extends ServerCommandLine {
// Need to have the zk cluster shutdown when master is shutdown.
// Run a subclass that does the zk cluster shutdown on its way out.
LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1),
conf.getInt("hbase.regionservers", 1), LocalHMaster.class, HRegionServer.class);
conf.getInt("hbase.regionservers", 0), LocalHMaster.class, HRegionServer.class);
((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
cluster.startup();
waitOnMasterThreads(cluster);

23
hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
View File

@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@ -139,6 +140,8 @@ public class ServerManager {
private final long maxSkew;
private final long warningSkew;
private final boolean checkingBackupMaster;
private BaseLoadBalancer balancer;
/**
* Set of region servers which are dead but not processed immediately. If one
@ -194,6 +197,14 @@ public class ServerManager {
maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
this.connection = connect ? HConnectionManager.getConnection(c) : null;
// Put this in constructor so we don't cast it every time
checkingBackupMaster = (master instanceof HMaster)
&& !c.getBoolean("hbase.balancer.use-backupmaster", true)
&& ((HMaster)master).balancer instanceof BaseLoadBalancer;
if (checkingBackupMaster) {
balancer = (BaseLoadBalancer)((HMaster)master).balancer;
}
}
/**
@ -375,6 +386,18 @@ public class ServerManager {
@VisibleForTesting
void recordNewServerWithLock(final ServerName serverName, final ServerLoad sl) {
LOG.info("Registering server=" + serverName);
if (checkingBackupMaster) {
ZooKeeperWatcher zooKeeper = master.getZooKeeper();
String backupZNode = ZKUtil.joinZNode(
zooKeeper.backupMasterAddressesZNode, serverName.toString());
try {
if (ZKUtil.checkExists(zooKeeper, backupZNode) != -1) {
balancer.excludeServer(serverName);
}
} catch (KeeperException e) {
master.abort("Failed to check if a new server a backup master", e);
}
}
this.onlineServers.put(serverName, sl);
this.rsAdmins.remove(serverName);
}

45
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
View File

@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.balancer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashMap;
@ -406,6 +407,11 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
private static final Random RANDOM = new Random(System.currentTimeMillis());
private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class);
// a flag to indicate if assigning regions to backup masters
protected boolean usingBackupMasters = false;
protected final Set<ServerName> excludedServers =
Collections.synchronizedSet(new HashSet<ServerName>());
protected final MetricsBalancer metricsBalancer = new MetricsBalancer();
protected ServerName masterServerName;
protected MasterServices services;
@ -417,12 +423,29 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
else if (slop > 1) slop = 1;
this.config = conf;
usingBackupMasters = conf.getBoolean("hbase.balancer.use-backupmaster", true);
}
protected void setSlop(Configuration conf) {
this.slop = conf.getFloat("hbase.regions.slop", (float) 0.2);
}
/**
* If there is any server excluded, filter it out from the cluster map so
* we won't assign any region to it, assuming none's already assigned there.
*/
protected void filterExcludedServers(Map<ServerName, List<HRegionInfo>> clusterMap) {
if (excludedServers.isEmpty()) { // No server to filter out
return;
}
Iterator<Map.Entry<ServerName, List<HRegionInfo>>> it = clusterMap.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<ServerName, List<HRegionInfo>> en = it.next();
if (excludedServers.contains(en.getKey()) && en.getValue().isEmpty()) {
it.remove();
}
}
}
/**
* Balance the regions that should be on master regionserver.
*/
@ -469,6 +492,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
return plans;
}
public void excludeServer(ServerName serverName) {
if (!usingBackupMasters) excludedServers.add(serverName);
}
@Override
public Configuration getConf() {
return this.config;
@ -476,12 +503,19 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
@Override
public void setClusterStatus(ClusterStatus st) {
// Not used except for the StocasticBalancer
if (st == null || usingBackupMasters) return;
// Not assign any region to backup masters.
// Put them on the excluded server list.
// Assume there won't be too much backup masters
// re/starting, so this won't leak much memory.
excludedServers.addAll(st.getBackupMasters());
}
@Override
public void setMasterServices(MasterServices masterServices) {
masterServerName = masterServices.getServerName();
excludedServers.remove(masterServerName);
this.services = masterServices;
}
@ -535,6 +569,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
List<ServerName> servers) {
metricsBalancer.incrMiscInvocations();
if (!excludedServers.isEmpty() && servers != null) {
servers.removeAll(excludedServers);
}
if (regions.isEmpty() || servers.isEmpty()) {
return null;
}
@ -619,6 +656,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
metricsBalancer.incrMiscInvocations();
if (!excludedServers.isEmpty() && servers != null) {
servers.removeAll(excludedServers);
}
if (servers == null || servers.isEmpty()) {
LOG.warn("Wanted to do random assignment but no servers to assign to");
return null;
@ -660,6 +700,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
// Update metrics
metricsBalancer.incrMiscInvocations();
if (!excludedServers.isEmpty() && servers != null) {
servers.removeAll(excludedServers);
}
if (regions.isEmpty() || servers.isEmpty()) {
return null;
}

1
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
View File

@ -184,6 +184,7 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
if (regionsToReturn != null) {
return regionsToReturn;
}
filterExcludedServers(clusterMap);
boolean emptyRegionServerPresent = false;
long startTime = System.currentTimeMillis();

1
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View File

@ -196,6 +196,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
if (plans != null) {
return plans;
}
filterExcludedServers(clusterState);
if (!needsBalance(new ClusterLoadState(masterServerName, clusterState))) {
return null;
}

5
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -777,7 +777,7 @@ public class HRegionServer extends HasThread implements
try {
this.infoServer.stop();
} catch (Exception e) {
e.printStackTrace();
LOG.error("Failed to stop infoServer", e);
}
}
// Send cache a shutdown.
@ -1534,6 +1534,9 @@ public class HRegionServer extends HasThread implements
}
port = this.infoServer.getPort();
conf.setInt(HConstants.REGIONSERVER_INFO_PORT, port);
int masterInfoPort = conf.getInt(HConstants.MASTER_INFO_PORT,
HConstants.DEFAULT_MASTER_INFOPORT);
conf.setInt("hbase.master.info.port.orig", masterInfoPort);
conf.setInt(HConstants.MASTER_INFO_PORT, port);
return port;
}

47
hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java
View File

@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.balancer;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -35,7 +36,9 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.ServerName;
@ -365,4 +368,48 @@ public class TestBaseLoadBalancer extends BalancerTestBase {
assertEquals(-1, cluster.regionLocations[r43][0]);
}
@Test
public void testBackupMastersExcluded() throws HBaseIOException {
ClusterStatus st = Mockito.mock(ClusterStatus.class);
ArrayList<ServerName> backupMasters = new ArrayList<ServerName>();
ServerName backupMaster = ServerName.valueOf("fake-backupmaster", 0, 1L);
backupMasters.add(backupMaster);
BaseLoadBalancer balancer = (BaseLoadBalancer)loadBalancer;
balancer.usingBackupMasters = false;
Mockito.when(st.getBackupMasters()).thenReturn(backupMasters);
loadBalancer.setClusterStatus(st);
assertEquals(1, balancer.excludedServers.size());
assertTrue(balancer.excludedServers.contains(backupMaster));
// Round robin assignment
List<HRegionInfo> regions = randomRegions(1);
HRegionInfo region = regions.get(0);
assertNull(loadBalancer.randomAssignment(region, backupMasters));
assertNull(loadBalancer.roundRobinAssignment(regions, backupMasters));
HashMap<HRegionInfo, ServerName> assignments = new HashMap<HRegionInfo, ServerName>();
assignments.put(region, backupMaster);
assertNull(loadBalancer.retainAssignment(assignments, backupMasters));
ArrayList<ServerName> servers = new ArrayList<ServerName>(backupMasters);
ServerName sn = ServerName.valueOf("fake-rs", 0, 1L);
servers.add(sn);
assertEquals(sn, loadBalancer.randomAssignment(region, servers));
Map<ServerName, List<HRegionInfo>> plans =
loadBalancer.roundRobinAssignment(regions, servers);
assertEquals(1, plans.size());
assertTrue(plans.get(sn).contains(region));
// Retain assignment
plans = loadBalancer.retainAssignment(assignments, servers);
assertEquals(1, plans.size());
assertTrue(plans.get(sn).contains(region));
// Filter backup masters for balance cluster
Map<ServerName, List<HRegionInfo>> clusterMap =
new HashMap<ServerName, List<HRegionInfo>>();
clusterMap.put(backupMaster, new ArrayList<HRegionInfo>());
clusterMap.put(sn, new ArrayList<HRegionInfo>());
balancer.filterExcludedServers(clusterMap);
assertTrue(clusterMap.containsKey(sn));
assertEquals(1, clusterMap.size());
}
}