From 1ade4d2f440a629968d41487bf27062a7e22f08b Mon Sep 17 00:00:00 2001
From: Nihal Jain <nihaljain.cs@gmail.com>
Date: Sun, 1 Jul 2018 13:08:14 +0530
Subject: [PATCH] HBASE-20808 Wrong shutdown order between Chores and
 ChoreService

Signed-off-by: Reid Chan <reidchan@apache.org>
---
 .../apache/hadoop/hbase/master/HMaster.java   | 40 +++++++++----------
 .../hbase/regionserver/HRegionServer.java     | 19 +++++----
 2 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index dc627523569..b05f3e11c0a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -58,6 +58,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.ChoreService;
 import org.apache.hadoop.hbase.ClusterId;
 import org.apache.hadoop.hbase.ClusterMetrics;
 import org.apache.hadoop.hbase.ClusterMetrics.Option;
@@ -72,7 +73,6 @@ import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.PleaseHoldException;
 import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.TableName;
@@ -1269,12 +1269,6 @@ public class HMaster extends HRegionServer implements MasterServices {
     }
   }
 
-  private void cancelChore(ScheduledChore chore) {
-    if (chore != null) {
-      chore.cancel();
-    }
-  }
-
   @Override
   protected void stopServiceThreads() {
     if (masterJettyServer != null) {
@@ -1285,8 +1279,11 @@ public class HMaster extends HRegionServer implements MasterServices {
         LOG.error("Failed to stop master jetty server", e);
       }
     }
-    super.stopServiceThreads();
     stopChores();
+    if (this.mobCompactThread != null) {
+      this.mobCompactThread.close();
+    }
+    super.stopServiceThreads();
     CleanerChore.shutDownChorePool();
 
     LOG.debug("Stopping service threads");
@@ -1364,21 +1361,20 @@ public class HMaster extends HRegionServer implements MasterServices {
   }
 
   private void stopChores() {
-    cancelChore(this.expiredMobFileCleanerChore);
-    cancelChore(this.mobCompactChore);
-    cancelChore(this.balancerChore);
-    cancelChore(this.normalizerChore);
-    cancelChore(this.clusterStatusChore);
-    cancelChore(this.catalogJanitorChore);
-    cancelChore(this.clusterStatusPublisherChore);
-    if (this.mobCompactThread != null) {
-      this.mobCompactThread.close();
+    ChoreService choreService = getChoreService();
+    if (choreService != null) {
+      choreService.cancelChore(this.expiredMobFileCleanerChore);
+      choreService.cancelChore(this.mobCompactChore);
+      choreService.cancelChore(this.balancerChore);
+      choreService.cancelChore(this.normalizerChore);
+      choreService.cancelChore(this.clusterStatusChore);
+      choreService.cancelChore(this.catalogJanitorChore);
+      choreService.cancelChore(this.clusterStatusPublisherChore);
+      choreService.cancelChore(this.snapshotQuotaChore);
+      choreService.cancelChore(this.logCleaner);
+      choreService.cancelChore(this.hfileCleaner);
+      choreService.cancelChore(this.replicationBarrierCleaner);
     }
-    cancelChore(this.clusterStatusPublisherChore);
-    cancelChore(this.snapshotQuotaChore);
-    cancelChore(this.logCleaner);
-    cancelChore(this.hfileCleaner);
-    cancelChore(this.replicationBarrierCleaner);
   }
 
   /**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1614cf5a52f..3fe386d86fb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2436,14 +2436,17 @@ public class HRegionServer extends HasThread implements
    */
   protected void stopServiceThreads() {
     // clean up the scheduled chores
-    if (this.choreService != null) choreService.shutdown();
-    if (this.nonceManagerChore != null) nonceManagerChore.cancel(true);
-    if (this.compactionChecker != null) compactionChecker.cancel(true);
-    if (this.periodicFlusher != null) periodicFlusher.cancel(true);
-    if (this.healthCheckChore != null) healthCheckChore.cancel(true);
-    if (this.storefileRefresher != null) storefileRefresher.cancel(true);
-    if (this.movedRegionsCleaner != null) movedRegionsCleaner.cancel(true);
-    if (this.fsUtilizationChore != null) fsUtilizationChore.cancel(true);
+    if (this.choreService != null) {
+      choreService.cancelChore(nonceManagerChore);
+      choreService.cancelChore(compactionChecker);
+      choreService.cancelChore(periodicFlusher);
+      choreService.cancelChore(healthCheckChore);
+      choreService.cancelChore(storefileRefresher);
+      choreService.cancelChore(movedRegionsCleaner);
+      choreService.cancelChore(fsUtilizationChore);
+      // clean up the remaining scheduled chores (in case we missed out any)
+      choreService.shutdown();
+    }
 
     if (this.cacheFlusher != null) {
       this.cacheFlusher.join();