Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25032 Wait for region server to become online before adding it to online servers in Master (#2769) 

Signed-off-by: Bharath Vissapragada <bharathv@apache.org>
Signed-off-by: Michael Stack <stack@apache.org>
This commit is contained in:
caroliney14 2021-03-25 10:39:07 -07:00 committed by GitHub
parent f6bb4bb93e
commit 1e4639d2eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 33 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
View File

@ -204,7 +204,8 @@ public class ServerManager {
}
/**
* Let the server manager know a new regionserver has come online
* Let the server manager know a regionserver is requesting configurations.
* Regionserver will not be added here, but in its first report.
* @param request the startup request
* @param versionNumber the version number of the new regionserver
* @param version the version of the new regionserver, could contain strings like "SNAPSHOT"
@ -227,10 +228,6 @@ public class ServerManager {
ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode());
checkClockSkew(sn, request.getServerCurrentTime());
checkIsDead(sn, "STARTUP");
if (!checkAndRecordNewServer(sn, ServerMetricsBuilder.of(sn, versionNumber, version))) {
LOG.warn(
"THIS SHOULD NOT HAPPEN, RegionServerStartup" + " could not record the server: " + sn);
}
return sn;
}
@ -281,7 +278,6 @@ public class ServerManager {
if (null == this.onlineServers.replace(sn, sl)) {
// Already have this host+port combo and its just different start code?
// Just let the server in. Presume master joining a running cluster.
// recordNewServer is what happens at the end of reportServerStartup.
// The only thing we are skipping is passing back to the regionserver
// the ServerName to use. Here we presume a master has already done
// that so we'll press on with whatever it gave us for ServerName.

34
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -1014,10 +1014,9 @@ public class HRegionServer extends Thread implements
// node was created, in case any coprocessors want to use ZooKeeper
this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
// Try and register with the Master; tell it we are here. Break if server is stopped or
// the clusterup flag is down or hdfs went wacky. Once registered successfully, go ahead and
// start up all Services. Use RetryCounter to get backoff in case Master is struggling to
// come up.
// Get configurations from the Master. Break if server is stopped or
// the clusterup flag is down or hdfs went wacky. Then start up all Services.
// Use RetryCounter to get backoff in case Master is struggling to come up.
LOG.debug("About to register with Master.");
RetryCounterFactory rcf =
new RetryCounterFactory(Integer.MAX_VALUE, this.sleeper.getPeriod(), 1000 * 60 * 5);
@ -1050,7 +1049,7 @@ public class HRegionServer extends Thread implements
}
}
// We registered with the Master. Go into run mode.
// Run mode.
long lastMsg = System.currentTimeMillis();
long oldRequestCount = -1;
// The main run loop.
@ -1084,7 +1083,14 @@ public class HRegionServer extends Thread implements
}
long now = System.currentTimeMillis();
if ((now - lastMsg) >= msgInterval) {
tryRegionServerReport(lastMsg, now);
// Register with the Master now that our setup is complete.
if (tryRegionServerReport(lastMsg, now) && !online.get()) {
// Wake up anyone waiting for this server to online
synchronized (online) {
online.set(true);
online.notifyAll();
}
}
lastMsg = System.currentTimeMillis();
}
if (!isStopped() && !isAborted()) {
@ -1253,12 +1259,12 @@ public class HRegionServer extends Thread implements
}
@InterfaceAudience.Private
protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
protected boolean tryRegionServerReport(long reportStartTime, long reportEndTime)
throws IOException {
RegionServerStatusService.BlockingInterface rss = rssStub;
if (rss == null) {
// the current server could be stopping.
return;
return false;
}
ClusterStatusProtos.ServerLoad sl = buildServerLoad(reportStartTime, reportEndTime);
try {
@ -1278,7 +1284,9 @@ public class HRegionServer extends Thread implements
// Couldn't connect to the master, get location from zk and reconnect
// Method blocks until new master is found or we are stopped
createRegionServerStatusStub(true);
return false;
}
return true;
}
/**
@ -1653,11 +1661,6 @@ public class HRegionServer extends Thread implements
", sessionid=0x" +
Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
// Wake up anyone waiting for this server to online
synchronized (online) {
online.set(true);
online.notifyAll();
}
} catch (Throwable e) {
stop("Failed initialization");
throw convertThrowableToIOE(cleanup(e, "Failed init"),
@ -2836,10 +2839,9 @@ public class HRegionServer extends Thread implements
}
/*
* Let the master know we're here Run initialization using parameters passed
* us by the master.
* Run initialization using parameters passed us by the master.
* @return A Map of key/value configurations we got from the Master else
* null if we failed to register.
* null if we failed during report.
* @throws IOException
*/
private RegionServerStartupResponse reportForDuty() throws IOException {

3
hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
View File

@ -446,8 +446,9 @@ public class MiniHBaseCluster extends HBaseCluster {
ServerName rsServerName = t.getRegionServer().getServerName();
long start = System.currentTimeMillis();
ClusterMetrics clusterStatus = getClusterMetrics();
ClusterMetrics clusterStatus;
while ((System.currentTimeMillis() - start) < timeout) {
clusterStatus = getClusterMetrics();
if (clusterStatus != null && clusterStatus.getLiveServerMetrics().containsKey(rsServerName)) {
return t;
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestGetReplicationLoad.java
View File

@ -58,8 +58,9 @@ public class TestGetReplicationLoad {
}
@Override
protected void tryRegionServerReport(long reportStartTime, long reportEndTime) {
protected boolean tryRegionServerReport(long reportStartTime, long reportEndTime) {
// do nothing
return true;
}
}

15
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
View File

@ -61,11 +61,6 @@ public class TestMasterMetrics {
public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
super(conf);
}
@Override
protected void tryRegionServerReport(long reportStartTime, long reportEndTime) {
// do nothing
}
}
public static class MyRegionServer extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
@ -73,11 +68,6 @@ public class TestMasterMetrics {
public MyRegionServer(Configuration conf) throws IOException, InterruptedException {
super(conf);
}
@Override
protected void tryRegionServerReport(long reportStartTime, long reportEndTime) {
// do nothing
}
}
@BeforeClass
@ -108,11 +98,14 @@ public class TestMasterMetrics {
request.setServer(ProtobufUtil.toServerName(serverName));
long expectedRequestNumber = 10000;
MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
ClusterStatusProtos.ServerLoad sl = ClusterStatusProtos.ServerLoad.newBuilder()
.setTotalNumberOfRequests(expectedRequestNumber).build();
request.setLoad(sl);
MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
// Init master source again to reset cluster requests counter
masterSource.init();
master.getMasterRpcServices().regionServerReport(null, request.build());
metricsHelper.assertCounter("cluster_requests", expectedRequestNumber, masterSource);

3
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionInDeadRegionServer.java
View File

@ -84,13 +84,14 @@ public class TestCompactionInDeadRegionServer {
}
@Override
protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
protected boolean tryRegionServerReport(long reportStartTime, long reportEndTime)
throws IOException {
try {
super.tryRegionServerReport(reportStartTime, reportEndTime);
} catch (YouAreDeadException e) {
// ignore, do not abort
}
return true;
}
}

4
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestShutdownWhileWALBroken.java
View File

@ -85,13 +85,15 @@ public class TestShutdownWhileWALBroken {
}
@Override
protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
protected boolean tryRegionServerReport(long reportStartTime, long reportEndTime)
throws IOException {
try {
super.tryRegionServerReport(reportStartTime, reportEndTime);
} catch (YouAreDeadException e) {
LOG.info("Caught YouAreDeadException, ignore", e);
return false;
}
return true;
}
@Override