Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26468 Region Server doesn't exit cleanly incase it crashes. (#3865) (#3862) 

Signed-off-by: Duo Zhang <zhangduo@apache.org>
Signed-off-by: Geoffrey Jacoby <gjacoby@apache.org>
Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
Rushabh Shah 2021-12-01 00:36:07 -05:00 committed by Viraj Jasani
parent f5af7611c0
commit 1408c044b1
No known key found for this signature in database
GPG Key ID: 08E70F70AB71C5A1
2 changed files with 50 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java
View File

@ -25,10 +25,12 @@ import java.lang.Thread.UncaughtExceptionHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
@ -266,4 +268,33 @@ public class Threads {
Preconditions.checkNotNull(PrintThreadInfoLazyHolder.HELPER,
"Cannot find method. Check hadoop jars linked").printThreadInfo(stream, title);
}
/**
* Checks whether any non-daemon thread is running.
* @return true if there are non daemon threads running, otherwise false
*/
public static boolean isNonDaemonThreadRunning() {
AtomicInteger nonDaemonThreadCount = new AtomicInteger();
Set<Thread> threads = Thread.getAllStackTraces().keySet();
threads.forEach(t -> {
// Exclude current thread
if (t.getId() != Thread.currentThread().getId() && !t.isDaemon()) {
nonDaemonThreadCount.getAndIncrement();
LOG.info("Non daemon thread {} is still alive", t.getName());
LOG.info(printStackTrace(t));
}
});
return nonDaemonThreadCount.get() > 0;
}
/*
Print stack trace of the passed thread
*/
public static String printStackTrace(Thread t) {
StringBuilder sb = new StringBuilder();
for (StackTraceElement frame: t.getStackTrace()) {
sb.append("\n").append(" ").append(frame.toString());
}
return sb.toString();
}
}

21
hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java
View File

@ -36,6 +36,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hbase.util.Threads.isNonDaemonThreadRunning;
/**
* Base class for command lines that start up various HBase daemons.
@ -141,8 +142,10 @@ public abstract class ServerCommandLine extends Configured implements Tool {
}
/**
* Parse and run the given command line. This may exit the JVM if
* a nonzero exit code is returned from <code>run()</code>.
* Parse and run the given command line. This will exit the JVM with
* the exit code returned from <code>run()</code>.
* If return code is 0, wait for atmost 30 seconds for all non-daemon threads to quit,
* otherwise exit the jvm
*/
public void doMain(String args[]) {
try {
@ -150,6 +153,20 @@ public abstract class ServerCommandLine extends Configured implements Tool {
if (ret != 0) {
System.exit(ret);
}
// Return code is 0 here.
boolean forceStop = false;
long startTime = EnvironmentEdgeManager.currentTime();
while (isNonDaemonThreadRunning()) {
if (EnvironmentEdgeManager.currentTime() - startTime > 30 * 1000) {
forceStop = true;
break;
}
Thread.sleep(1000);
}
if (forceStop) {
LOG.error("Failed to stop all non-daemon threads, so terminating JVM");
System.exit(-1);
}
} catch (Exception e) {
LOG.error("Failed to run", e);
System.exit(-1);