From bc42dc04ac674d804b52c7c932655e9de71c6a4b Mon Sep 17 00:00:00 2001 From: jxiang Date: Mon, 31 Mar 2014 16:42:04 +0000 Subject: [PATCH] HBASE-10815 Master regionserver should be rolling-upgradable git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1583373 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/main/resources/hbase-default.xml | 35 +++----- .../hbase/master/ActiveMasterManager.java | 15 +--- .../apache/hadoop/hbase/master/HMaster.java | 87 +++++++++++++++++-- .../hbase/master/HMasterCommandLine.java | 2 +- .../hadoop/hbase/master/ServerManager.java | 23 +++++ .../master/balancer/BaseLoadBalancer.java | 45 +++++++++- .../master/balancer/SimpleLoadBalancer.java | 1 + .../balancer/StochasticLoadBalancer.java | 1 + .../hbase/regionserver/HRegionServer.java | 5 +- .../master/balancer/TestBaseLoadBalancer.java | 47 ++++++++++ 10 files changed, 216 insertions(+), 45 deletions(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index da485673905..5be15e267fa 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -96,11 +96,6 @@ possible configurations would overwhelm and obscure the important. - - hbase.master.port - 16000 - The port the HBase Master should bind to. - hbase.master.info.port 16010 @@ -147,23 +142,11 @@ possible configurations would overwhelm and obscure the important. META. - fail.fast.expired.active.master - false - If abort immediately for the expired master without trying - to recover its zk session. - - - hbase.master.dns.interface - default - The name of the Network Interface from which a master - should report its IP address. - - - hbase.master.dns.nameserver - default - The host name or IP address of the name server (DNS) - which a master should use to determine the host name used - for communication and display purposes. + hbase.master.infoserver.redirect + true + Whether or not the Master listens to the Master web + UI port (hbase.master.info.port) and redirects requests to the web + UI server shared by the Master and RegionServer. @@ -344,7 +327,7 @@ possible configurations would overwhelm and obscure the important. hbase.zookeeper.peerport 2888 Port used by ZooKeeper peers to talk to each other. - Seehttp://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper + See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper for more information. @@ -513,6 +496,12 @@ possible configurations would overwhelm and obscure the important. 300000 Period at which the region balancer runs in the Master. + + hbase.balancer.use-backupmaster + true + Whether or not the region balancer uses the backup Masters + as regionservers, and assigns regions to them. + hbase.regions.slop 0.2 diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java index d3011272e06..596998690a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java @@ -148,13 +148,13 @@ public class ActiveMasterManager extends ZooKeeperListener { */ boolean blockUntilBecomingActiveMaster( int checkInterval, MonitoredTask startupStatus) { + String backupZNode = ZKUtil.joinZNode( + this.watcher.backupMasterAddressesZNode, this.sn.toString()); while (!(master.isAborted() || master.isStopped())) { startupStatus.setStatus("Trying to register in ZK as active master"); // Try to become the active master, watch if there is another master. // Write out our ServerName as versioned bytes. try { - String backupZNode = - ZKUtil.joinZNode(this.watcher.backupMasterAddressesZNode, this.sn.toString()); if (MasterAddressTracker.setMasterAddress(this.watcher, this.watcher.getMasterAddressZNode(), this.sn)) { @@ -178,17 +178,6 @@ public class ActiveMasterManager extends ZooKeeperListener { // and the master ephemeral node has not expired yet. this.clusterHasActiveMaster.set(true); - /* - * Add a ZNode for ourselves in the backup master directory since we are - * not the active master. - * - * If we become the active master later, ActiveMasterManager will delete - * this node explicitly. If we crash before then, ZooKeeper will delete - * this node for us since it is ephemeral. - */ - LOG.info("Adding ZNode for " + backupZNode + " in backup master directory"); - MasterAddressTracker.setMasterAddress(this.watcher, backupZNode, this.sn); - String msg; byte[] bytes = ZKUtil.getDataAndWatch(this.watcher, this.watcher.getMasterAddressZNode()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 46d4223cbcf..82bae87ce9f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -34,7 +34,10 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -107,12 +110,16 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.VersionInfo; import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker; import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; +import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; import org.apache.hadoop.hbase.zookeeper.RegionServerTracker; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Watcher; +import org.mortbay.jetty.Connector; +import org.mortbay.jetty.nio.SelectChannelConnector; +import org.mortbay.jetty.servlet.Context; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -214,6 +221,23 @@ public class HMaster extends HRegionServer implements MasterServices, Server { /** flag used in test cases in order to simulate RS failures during master initialization */ private volatile boolean initializationBeforeMetaAssignment = false; + /** jetty server for master to redirect requests to regionserver infoServer */ + private org.mortbay.jetty.Server masterJettyServer; + + public static class RedirectServlet extends HttpServlet { + private static final long serialVersionUID = 2894774810058302472L; + private static int regionServerInfoPort; + + @Override + public void doGet(HttpServletRequest request, + HttpServletResponse response) throws ServletException, IOException { + String redirectUrl = request.getScheme() + "://" + + request.getServerName() + ":" + regionServerInfoPort + + request.getRequestURI(); + response.sendRedirect(redirectUrl); + } + } + /** * Initializes the HMaster. The steps are as follows: *

@@ -271,6 +295,34 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } startActiveMasterManager(); + putUpJettyServer(); + } + + private void putUpJettyServer() throws IOException { + if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) { + return; + } + int infoPort = conf.getInt("hbase.master.info.port.orig", + HConstants.DEFAULT_MASTER_INFOPORT); + // -1 is for disabling info server, so no redirecting + if (infoPort < 0 || infoServer == null) { + return; + } + + RedirectServlet.regionServerInfoPort = infoServer.getPort(); + masterJettyServer = new org.mortbay.jetty.Server(); + Connector connector = new SelectChannelConnector(); + connector.setHost(conf.get("hbase.master.info.bindAddress", "0.0.0.0")); + connector.setPort(infoPort); + masterJettyServer.addConnector(connector); + masterJettyServer.setStopAtShutdown(true); + Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS); + context.addServlet(RedirectServlet.class, "/*"); + try { + masterJettyServer.start(); + } catch (Exception e) { + throw new IOException("Failed to start redirecting jetty server", e); + } } /** @@ -479,11 +531,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { this.initializationBeforeMetaAssignment = true; - //initialize load balancer - this.balancer.setClusterStatus(getClusterStatus()); - this.balancer.setMasterServices(this); - this.balancer.initialize(); - // Wait for regionserver to finish initialization. while (!isOnline()) { synchronized (online) { @@ -491,6 +538,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } + //initialize load balancer + this.balancer.setClusterStatus(getClusterStatus()); + this.balancer.setMasterServices(this); + this.balancer.initialize(); + // Make sure meta assigned before proceeding. status.setStatus("Assigning Meta Region"); assignMeta(status, previouslyFailedMetaRSs); @@ -785,6 +837,14 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } protected void stopServiceThreads() { + if (masterJettyServer != null) { + LOG.info("Stopping master jetty server"); + try { + masterJettyServer.stop(); + } catch (Exception e) { + LOG.error("Failed to stop master jetty server", e); + } + } super.stopServiceThreads(); stopChores(); // Wait for all the remaining region servers to report in IFF we were @@ -1136,7 +1196,22 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } - private void startActiveMasterManager() { + private void startActiveMasterManager() throws KeeperException { + String backupZNode = ZKUtil.joinZNode( + zooKeeper.backupMasterAddressesZNode, serverName.toString()); + /* + * Add a ZNode for ourselves in the backup master directory since we + * may not become the active master. If so, we want the actual active + * master to know we are backup masters, so that it won't assign + * regions to us if so configured. + * + * If we become the active master later, ActiveMasterManager will delete + * this node explicitly. If we crash before then, ZooKeeper will delete + * this node for us since it is ephemeral. + */ + LOG.info("Adding ZNode for " + backupZNode + " in backup master directory"); + MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName); + activeMasterManager = new ActiveMasterManager(zooKeeper, serverName, this); // Start a thread to try to become the active master, so we won't block here Threads.setDaemonThreadRunning(new Thread(new Runnable() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java index 7a2015d1e37..55aaa7cf17e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java @@ -178,7 +178,7 @@ public class HMasterCommandLine extends ServerCommandLine { // Need to have the zk cluster shutdown when master is shutdown. // Run a subclass that does the zk cluster shutdown on its way out. LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1), - conf.getInt("hbase.regionservers", 1), LocalHMaster.class, HRegionServer.class); + conf.getInt("hbase.regionservers", 0), LocalHMaster.class, HRegionServer.class); ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster); cluster.startup(); waitOnMasterThreads(cluster); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 63232edc28a..391e0b84237 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.RetriesExhaustedException; +import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler; import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler; import org.apache.hadoop.hbase.monitoring.MonitoredTask; @@ -139,6 +140,8 @@ public class ServerManager { private final long maxSkew; private final long warningSkew; + private final boolean checkingBackupMaster; + private BaseLoadBalancer balancer; /** * Set of region servers which are dead but not processed immediately. If one @@ -194,6 +197,14 @@ public class ServerManager { maxSkew = c.getLong("hbase.master.maxclockskew", 30000); warningSkew = c.getLong("hbase.master.warningclockskew", 10000); this.connection = connect ? HConnectionManager.getConnection(c) : null; + + // Put this in constructor so we don't cast it every time + checkingBackupMaster = (master instanceof HMaster) + && !c.getBoolean("hbase.balancer.use-backupmaster", true) + && ((HMaster)master).balancer instanceof BaseLoadBalancer; + if (checkingBackupMaster) { + balancer = (BaseLoadBalancer)((HMaster)master).balancer; + } } /** @@ -375,6 +386,18 @@ public class ServerManager { @VisibleForTesting void recordNewServerWithLock(final ServerName serverName, final ServerLoad sl) { LOG.info("Registering server=" + serverName); + if (checkingBackupMaster) { + ZooKeeperWatcher zooKeeper = master.getZooKeeper(); + String backupZNode = ZKUtil.joinZNode( + zooKeeper.backupMasterAddressesZNode, serverName.toString()); + try { + if (ZKUtil.checkExists(zooKeeper, backupZNode) != -1) { + balancer.excludeServer(serverName); + } + } catch (KeeperException e) { + master.abort("Failed to check if a new server a backup master", e); + } + } this.onlineServers.put(serverName, sl); this.rsAdmins.remove(serverName); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index b39f50f1511..e7002c39745 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.balancer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.Deque; import java.util.HashMap; @@ -406,6 +407,11 @@ public abstract class BaseLoadBalancer implements LoadBalancer { private static final Random RANDOM = new Random(System.currentTimeMillis()); private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class); + // a flag to indicate if assigning regions to backup masters + protected boolean usingBackupMasters = false; + protected final Set excludedServers = + Collections.synchronizedSet(new HashSet()); + protected final MetricsBalancer metricsBalancer = new MetricsBalancer(); protected ServerName masterServerName; protected MasterServices services; @@ -417,12 +423,29 @@ public abstract class BaseLoadBalancer implements LoadBalancer { else if (slop > 1) slop = 1; this.config = conf; + usingBackupMasters = conf.getBoolean("hbase.balancer.use-backupmaster", true); } protected void setSlop(Configuration conf) { this.slop = conf.getFloat("hbase.regions.slop", (float) 0.2); } + /** + * If there is any server excluded, filter it out from the cluster map so + * we won't assign any region to it, assuming none's already assigned there. + */ + protected void filterExcludedServers(Map> clusterMap) { + if (excludedServers.isEmpty()) { // No server to filter out + return; + } + Iterator>> it = clusterMap.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry> en = it.next(); + if (excludedServers.contains(en.getKey()) && en.getValue().isEmpty()) { + it.remove(); + } + } + } /** * Balance the regions that should be on master regionserver. */ @@ -469,6 +492,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer { return plans; } + public void excludeServer(ServerName serverName) { + if (!usingBackupMasters) excludedServers.add(serverName); + } + @Override public Configuration getConf() { return this.config; @@ -476,12 +503,19 @@ public abstract class BaseLoadBalancer implements LoadBalancer { @Override public void setClusterStatus(ClusterStatus st) { - // Not used except for the StocasticBalancer + if (st == null || usingBackupMasters) return; + + // Not assign any region to backup masters. + // Put them on the excluded server list. + // Assume there won't be too much backup masters + // re/starting, so this won't leak much memory. + excludedServers.addAll(st.getBackupMasters()); } @Override public void setMasterServices(MasterServices masterServices) { masterServerName = masterServices.getServerName(); + excludedServers.remove(masterServerName); this.services = masterServices; } @@ -535,6 +569,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer { List servers) { metricsBalancer.incrMiscInvocations(); + if (!excludedServers.isEmpty() && servers != null) { + servers.removeAll(excludedServers); + } if (regions.isEmpty() || servers.isEmpty()) { return null; } @@ -619,6 +656,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer { public ServerName randomAssignment(HRegionInfo regionInfo, List servers) { metricsBalancer.incrMiscInvocations(); + if (!excludedServers.isEmpty() && servers != null) { + servers.removeAll(excludedServers); + } if (servers == null || servers.isEmpty()) { LOG.warn("Wanted to do random assignment but no servers to assign to"); return null; @@ -660,6 +700,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer { // Update metrics metricsBalancer.incrMiscInvocations(); + if (!excludedServers.isEmpty() && servers != null) { + servers.removeAll(excludedServers); + } if (regions.isEmpty() || servers.isEmpty()) { return null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java index fc88212ca24..110bd2b1718 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java @@ -184,6 +184,7 @@ public class SimpleLoadBalancer extends BaseLoadBalancer { if (regionsToReturn != null) { return regionsToReturn; } + filterExcludedServers(clusterMap); boolean emptyRegionServerPresent = false; long startTime = System.currentTimeMillis(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index b5fa4e73b12..e281deef0fb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -196,6 +196,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { if (plans != null) { return plans; } + filterExcludedServers(clusterState); if (!needsBalance(new ClusterLoadState(masterServerName, clusterState))) { return null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 46f2fa0ed51..f514524fb6b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -777,7 +777,7 @@ public class HRegionServer extends HasThread implements try { this.infoServer.stop(); } catch (Exception e) { - e.printStackTrace(); + LOG.error("Failed to stop infoServer", e); } } // Send cache a shutdown. @@ -1534,6 +1534,9 @@ public class HRegionServer extends HasThread implements } port = this.infoServer.getPort(); conf.setInt(HConstants.REGIONSERVER_INFO_PORT, port); + int masterInfoPort = conf.getInt(HConstants.MASTER_INFO_PORT, + HConstants.DEFAULT_MASTER_INFOPORT); + conf.setInt("hbase.master.info.port.orig", masterInfoPort); conf.setInt(HConstants.MASTER_INFO_PORT, port); return port; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java index ad9528750f4..b31f81669e0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.balancer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -35,7 +36,9 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.ServerName; @@ -365,4 +368,48 @@ public class TestBaseLoadBalancer extends BalancerTestBase { assertEquals(-1, cluster.regionLocations[r43][0]); } + @Test + public void testBackupMastersExcluded() throws HBaseIOException { + ClusterStatus st = Mockito.mock(ClusterStatus.class); + ArrayList backupMasters = new ArrayList(); + ServerName backupMaster = ServerName.valueOf("fake-backupmaster", 0, 1L); + backupMasters.add(backupMaster); + BaseLoadBalancer balancer = (BaseLoadBalancer)loadBalancer; + balancer.usingBackupMasters = false; + Mockito.when(st.getBackupMasters()).thenReturn(backupMasters); + loadBalancer.setClusterStatus(st); + assertEquals(1, balancer.excludedServers.size()); + assertTrue(balancer.excludedServers.contains(backupMaster)); + + // Round robin assignment + List regions = randomRegions(1); + HRegionInfo region = regions.get(0); + assertNull(loadBalancer.randomAssignment(region, backupMasters)); + assertNull(loadBalancer.roundRobinAssignment(regions, backupMasters)); + HashMap assignments = new HashMap(); + assignments.put(region, backupMaster); + assertNull(loadBalancer.retainAssignment(assignments, backupMasters)); + ArrayList servers = new ArrayList(backupMasters); + ServerName sn = ServerName.valueOf("fake-rs", 0, 1L); + servers.add(sn); + assertEquals(sn, loadBalancer.randomAssignment(region, servers)); + Map> plans = + loadBalancer.roundRobinAssignment(regions, servers); + assertEquals(1, plans.size()); + assertTrue(plans.get(sn).contains(region)); + + // Retain assignment + plans = loadBalancer.retainAssignment(assignments, servers); + assertEquals(1, plans.size()); + assertTrue(plans.get(sn).contains(region)); + + // Filter backup masters for balance cluster + Map> clusterMap = + new HashMap>(); + clusterMap.put(backupMaster, new ArrayList()); + clusterMap.put(sn, new ArrayList()); + balancer.filterExcludedServers(clusterMap); + assertTrue(clusterMap.containsKey(sn)); + assertEquals(1, clusterMap.size()); + } }