From 4ebbfa3d60c86fa7cac41ad582ab1b6edc990582 Mon Sep 17 00:00:00 2001
From: Peter Somogyi <psomogyi@apache.org>
Date: Thu, 29 Nov 2018 18:04:53 +0100
Subject: [PATCH] HBASE-21518 TestMasterFailoverWithProcedures is flaky

Signed-off-by: Sean Busbey <busbey@apache.org>
---
 .../hadoop/hbase/util/JVMClusterUtil.java     | 24 +++++++++++++------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
index 8c92f6660c0..7518d65a198 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
@@ -249,14 +249,24 @@ public class JVMClusterUtil {
       // Do backups first.
       JVMClusterUtil.MasterThread activeMaster = null;
       for (JVMClusterUtil.MasterThread t : masters) {
-        if (!t.master.isActiveMaster()) {
-          try {
-            t.master.stopMaster();
-          } catch (IOException e) {
-            LOG.error("Exception occurred while stopping master", e);
+        // Master was killed but could be still considered as active. Check first if it is stopped.
+        if (!t.master.isStopped()) {
+          if (!t.master.isActiveMaster()) {
+            try {
+              t.master.stopMaster();
+            } catch (IOException e) {
+              LOG.error("Exception occurred while stopping master", e);
+            }
+            LOG.info("Stopped backup Master {} is stopped: {}",
+                t.master.hashCode(), t.master.isStopped());
+          } else {
+            if (activeMaster != null) {
+              LOG.warn("Found more than 1 active master, hash {}", activeMaster.master.hashCode());
+            }
+            activeMaster = t;
+            LOG.debug("Found active master hash={}, stopped={}",
+                t.master.hashCode(), t.master.isStopped());
           }
-        } else {
-          activeMaster = t;
         }
       }
       // Do active after.