From ae77f81e7eaf07e9943688f10f7f7e14211151c2 Mon Sep 17 00:00:00 2001
From: Bo Cui <cuibo0108@163.com>
Date: Mon, 4 Jan 2021 23:34:38 +0800
Subject: [PATCH] HBASE-25447 remoteProc is suspended due to OOM ERROR (#2824)

Some OMME can not cause the JVM to exit, like "java.lang.OutOfMemoryError: Direct buffer memory", "java.lang.OutOfMemoryError: unable to create new native thread", as they dont call vmError#next_OnError_command. So abort HMaster when uncaught exception occurs in TimeoutExecutor, the new active Hmaster will resume the suspended procedure.

Signed-off-by: Duo Zhang <zhangduo@apache.org>
Signed-off-by: stack <stack@apache.com>
Signed-off-by: Pankaj Kumar<pankajkumar@apache.org>
(cherry picked from commit 600be60a4bd4d3b3e9652027a0cb8bdd32016c6b)
---
 .../hbase/procedure2/RemoteProcedureDispatcher.java       | 4 ++++
 .../hbase/master/procedure/RSProcedureDispatcher.java     | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 71d55ea9afb..8a5a19ea2db 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -106,6 +106,10 @@ public abstract class RemoteProcedureDispatcher<TEnv, TRemote extends Comparable
     return true;
   }
 
+  protected void setTimeoutExecutorUncaughtExceptionHandler(UncaughtExceptionHandler eh) {
+    timeoutExecutor.setUncaughtExceptionHandler(eh);
+  }
+
   public boolean stop() {
     if (!running.getAndSet(false)) {
       return false;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 17e5522efd7..3c554c41859 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -94,6 +94,7 @@ public class RSProcedureDispatcher
     if (!super.start()) {
       return false;
     }
+    setTimeoutExecutorUncaughtExceptionHandler(this::abort);
     if (master.isStopped()) {
       LOG.debug("Stopped");
       return false;
@@ -126,6 +127,13 @@ public class RSProcedureDispatcher
     return true;
   }
 
+  private void abort(Thread t, Throwable e) {
+    LOG.error("Caught error", e);
+    if (!master.isStopped() && !master.isStopping() && !master.isAborted()) {
+      master.abort("Aborting master", e);
+    }
+  }
+
   @Override
   public boolean stop() {
     if (!super.stop()) {