Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-3037 When new master joins running cluster does "Received report from unknown server -- telling it to STOP_REGIONSERVER..." 

M src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
 On regionServerReport, if we get a report from an 'unknown' regionserver,
 we used to tell it stop itself.  Now, if 'unknown' server AND
 this master did not start the cluster, its joining the cluster, then
 treat the report as a regionServerStart and register the incoming
 server rather than tell it shutdown.
M src/main/java/org/apache/hadoop/hbase/master/HMaster.java
 Pass the freshClusterStartup flag to ServerManager.
 Add more executors for opening and closing.  On cluster startup a
 master shouldn't be bottleneck clearning the server opens.
 Expose the run-balancer method so can make it available in
 ServeAdmin.
M src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 Minor formatting and javadoc
M src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
 Upped rpc version number because of new balancer addition (and because
 we didn't do it when we put in new master).
M src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
 Added balance method.
M src/main/resources/hbase-default.xml
 Change how ofter we check in from every 3 seconds to every 5 seconds.


git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1001140 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2010-09-25 06:24:42 +00:00
parent e58ebc4830
commit 1339f395d6
8 changed files with 86 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -540,6 +540,8 @@ Release 0.21.0 - Unreleased
HBASE-3028 No basescanner means no GC'ing of split, offlined parent regions
HBASE-2989 [replication] RSM won't cleanup after locking if 0 peers
HBASE-2992 [replication] MalformedObjectNameException in ReplicationMetrics
HBASE-3037 When new master joins running cluster does "Received report from
unknown server -- telling it to STOP_REGIONSERVER.
IMPROVEMENTS
HBASE-1760 Cleanup TODOs in HTable

18
src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
View File

@ -879,14 +879,24 @@ public class HBaseAdmin implements Abortable {
}
/**
* Turn the load balancer on or off.
* @param b If true, enable balancer. If false, disable balancer.
* @return Previous balancer value
* @throws ZooKeeperConnectionException
* @throws MasterNotRunningException
*/
public boolean balance(final boolean b)
public boolean balanceSwitch(final boolean b)
throws MasterNotRunningException, ZooKeeperConnectionException {
return getMaster().balance(b);
return getMaster().balanceSwitch(b);
}
/**
* Invoke the balancer. Will run the balancer and if regions to move, it will
* go ahead and do the reassignments. Can NOT run for various reasons. Check
* logs.
* @return True if balancer ran, false otherwise.
*/
public boolean balancer()
throws MasterNotRunningException, ZooKeeperConnectionException {
return getMaster().balance();
}
/**

3
src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
View File

@ -77,7 +77,8 @@ public interface HBaseRPCProtocolVersion extends VersionedProtocol {
* <li>Version 23: HBASE-2066, multi-put.</li>
* <li>Version 24: HBASE-2473, create table with regions.</li>
* <li>Version 25: Added openRegion and Stoppable/Abortable to API.</li>
* <li>Version 26: New master.</li>
* </ul>
*/
public static final long versionID = 25L;
public static final long versionID = 26L;
}

11
src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
View File

@ -144,8 +144,17 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion {
throws UnknownRegionException;
/**
* Run the balancer. Will run the balancer and if regions to move, it will
* go ahead and do the reassignments. Can NOT run for various reasons. Check
* logs.
* @return True if balancer ran, false otherwise.
*/
public boolean balance();
/**
* Turn the load balancer on or off.
* @param b If true, enable balancer. If false, disable balancer.
* @return Previous balancer value
*/
public boolean balance(final boolean b);
public boolean balanceSwitch(final boolean b);
}

13
src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
View File

@ -177,13 +177,13 @@ public class AssignmentManager extends ZooKeeperListener {
// Check existing regions in transition
List<String> nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
watcher.assignmentZNode);
if(nodes.isEmpty()) {
if (nodes.isEmpty()) {
LOG.info("No regions in transition in ZK to process on failover");
return;
}
LOG.info("Failed-over master needs to process " + nodes.size() +
" regions in transition");
for(String regionName : nodes) {
for (String regionName: nodes) {
RegionTransitionData data = ZKAssign.getData(watcher, regionName);
HRegionInfo regionInfo =
MetaReader.getRegion(catalogTracker, data.getRegionName()).getFirst();
@ -738,10 +738,10 @@ public class AssignmentManager extends ZooKeeperListener {
private void rebuildUserRegions() throws IOException {
Map<HRegionInfo,HServerAddress> allRegions =
MetaReader.fullScan(catalogTracker);
for(Map.Entry<HRegionInfo,HServerAddress> region : allRegions.entrySet()) {
for (Map.Entry<HRegionInfo,HServerAddress> region : allRegions.entrySet()) {
HServerAddress regionLocation = region.getValue();
HRegionInfo regionInfo = region.getKey();
if(regionLocation == null) {
if (regionLocation == null) {
regions.put(regionInfo, null);
continue;
}
@ -1020,6 +1020,7 @@ public class AssignmentManager extends ZooKeeperListener {
synchronized (this.regions) {
checkRegion(hsi, parent, true);
checkRegion(hsi, a, false);
checkRegion(hsi, b, false);
this.regions.put(a, hsi);
this.regions.put(b, hsi);
removeFromServers(hsi, parent, true);
@ -1031,10 +1032,10 @@ public class AssignmentManager extends ZooKeeperListener {
}
/*
* Caller must hold locks on regions Map.
* Caller must hold locks on this.regions Map.
* @param hsi
* @param hri
* @param expected
* @param expected True if we expect <code>hri</code> to be in this.regions.
*/
private void checkRegion(final HServerInfo hsi, final HRegionInfo hri,
final boolean expected) {

44
src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -160,7 +160,8 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
private LoadBalancer balancer = new LoadBalancer();
private Thread balancerChore;
private volatile boolean balance = true;
// If 'true', the balancer is 'on'. If 'false', the balancer will not run.
private volatile boolean balanceSwitch = true;
private Thread catalogJanitorChore;
@ -241,7 +242,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
this.connection = HConnectionManager.getConnection(conf);
this.executorService = new ExecutorService(getServerName());
this.serverManager = new ServerManager(this, this);
this.serverManager = new ServerManager(this, this, this.freshClusterStartup);
this.catalogTracker = new CatalogTracker(this.zooKeeper, this.connection,
this, conf.getInt("hbase.master.catalog.timeout", Integer.MAX_VALUE));
@ -304,15 +305,13 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
this.serverManager.waitForRegionServers();
// Start assignment of user regions, startup or failure
if (!this.stopped) {
if (this.freshClusterStartup) {
clusterStarterInitializations(this.fileSystemManager,
this.serverManager, this.catalogTracker, this.assignmentManager);
} else {
// Process existing unassigned nodes in ZK, read all regions from META,
// rebuild in-memory state.
this.assignmentManager.processFailover();
}
if (this.freshClusterStartup) {
clusterStarterInitializations(this.fileSystemManager,
this.serverManager, this.catalogTracker, this.assignmentManager);
} else {
// Process existing unassigned nodes in ZK, read all regions from META,
// rebuild in-memory state.
this.assignmentManager.processFailover();
}
// Start balancer and meta catalog janitor after meta and regions have
@ -320,6 +319,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
this.balancerChore = getAndStartBalancerChore(this);
this.catalogJanitorChore =
Threads.setDaemonThreadRunning(new CatalogJanitor(this, this));
// Check if we should stop every second.
Sleeper sleeper = new Sleeper(1000, this);
while (!this.stopped) sleeper.sleep();
@ -442,9 +442,9 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
try {
// Start the executor service pools
this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
conf.getInt("hbase.master.executor.openregion.threads", 5));
conf.getInt("hbase.master.executor.openregion.threads", 10));
this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
conf.getInt("hbase.master.executor.closeregion.threads", 5));
conf.getInt("hbase.master.executor.closeregion.threads", 10));
this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
conf.getInt("hbase.master.executor.serverops.threads", 5));
this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS,
@ -496,9 +496,8 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
}
private static Thread getAndStartBalancerChore(final HMaster master) {
String name = master.getServerName() + "-balancerChore";
int period = master.getConfiguration().
getInt("hbase.balancer.period", 3000000);
String name = master.getServerName() + "-BalancerChore";
int period = master.getConfiguration().getInt("hbase.balancer.period", 300000);
// Start up the load balancer chore
Chore chore = new Chore(name, period, master) {
@Override
@ -566,13 +565,10 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
return !isStopped();
}
/**
* Run the balancer.
* @return True if balancer ran, false otherwise.
*/
@Override
public boolean balance() {
// If balance not true, don't run balancer.
if (!this.balance) return false;
if (!this.balanceSwitch) return false;
synchronized (this.balancer) {
// Only allow one balance run at at time.
if (this.assignmentManager.isRegionsInTransition()) {
@ -606,9 +602,9 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
}
@Override
public boolean balance(final boolean b) {
boolean oldValue = this.balance;
this.balance = b;
public boolean balanceSwitch(final boolean b) {
boolean oldValue = this.balanceSwitch;
this.balanceSwitch = b;
LOG.info("Balance=" + b);
return oldValue;
}

33
src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
View File

@ -84,6 +84,7 @@ public class ServerManager {
private final Server master;
private final MasterServices services;
private final boolean freshClusterStartup;
private final ServerMonitor serverMonitorThread;
@ -119,10 +120,14 @@ public class ServerManager {
* Constructor.
* @param master
* @param services
* @param freshClusterStartup True if we are original master on a fresh
* cluster startup else if false, we are joining an already running cluster.
*/
public ServerManager(final Server master, final MasterServices services) {
public ServerManager(final Server master, final MasterServices services,
final boolean freshClusterStartup) {
this.master = master;
this.services = services;
this.freshClusterStartup = freshClusterStartup;
Configuration c = master.getConfiguration();
int monitorInterval = c.getInt("hbase.master.monitor.interval", 60 * 1000);
this.metrics = new MasterMetrics(master.getServerName());
@ -249,9 +254,29 @@ public class ServerManager {
// If we don't know this server, tell it shutdown.
HServerInfo storedInfo = this.onlineServers.get(info.getServerName());
if (storedInfo == null) {
LOG.warn("Received report from unknown server -- telling it " +
"to " + HMsg.Type.STOP_REGIONSERVER + ": " + info.getServerName());
return HMsg.STOP_REGIONSERVER_ARRAY;
if (!this.freshClusterStartup) {
// If we are joining an existing cluster, then soon as we come up we'll
// be getting reports from already running regionservers.
LOG.info("Registering new server: " + info.getServerName());
// recordNewServer is what happens at the end of reportServerStartup.
// The only thing we are skipping is passing back to the regionserver
// the HServerInfo to use. Here we presume a master has already done
// that so we'll press on with whatever it gave us for HSI.
recordNewServer(info);
// If msgs, put off their processing but this is not enough because
// its possible that the next time the server reports in, we'll still
// not be up and serving. For example, if a split, we'll need the
// regions and servers setup in the master before the below
// handleSplitReport will work. TODO: FIx!!
if (msgs.length > 0) throw new PleaseHoldException("FIX! Putting off " +
"message processing because not yet rwady but possible we won't be " +
"ready next on next report");
} else {
LOG.warn("Received report from unknown server, a server calling " +
" regionServerReport w/o having first called regionServerStartup; " +
"telling it " + HMsg.Type.STOP_REGIONSERVER + ": " + info.getServerName());
return HMsg.STOP_REGIONSERVER_ARRAY;
}
}
// Check startcodes

5
src/main/resources/hbase-default.xml
View File

@ -157,10 +157,9 @@
</property>
<property>
<name>hbase.regionserver.msginterval</name>
<value>3000</value>
<value>5000</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Use a high value for clusters with more than 100
nodes. Default is 3 seconds.
in milliseconds.
</description>
</property>
<property>