From 141d4e8b032a7fd2f9bd4e66eaa3330154f02b67 Mon Sep 17 00:00:00 2001
From: Allan Yang <allan163@apache.org>
Date: Thu, 25 Oct 2018 12:03:20 +0800
Subject: [PATCH] HBASE-21364 Procedure holds the lock should put to front of
 the queue after restart

---
 .../procedure2/AbstractProcedureScheduler.java     | 10 ++++++++++
 .../apache/hadoop/hbase/procedure2/Procedure.java  |  4 ++++
 .../hadoop/hbase/procedure2/ProcedureExecutor.java | 11 ++++++++++-
 .../hbase/procedure2/ProcedureScheduler.java       | 14 ++++++++++++++
 4 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java
index 5645f89e14d..7ab1329b32b 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java
@@ -85,6 +85,11 @@ public abstract class AbstractProcedureScheduler implements ProcedureScheduler {
     push(procedure, true, true);
   }
 
+  @Override
+  public void addFront(final Procedure procedure, boolean notify) {
+    push(procedure, true, notify);
+  }
+
   @Override
   public void addFront(Iterator<Procedure> procedureIterator) {
     schedLock();
@@ -109,6 +114,11 @@ public abstract class AbstractProcedureScheduler implements ProcedureScheduler {
     push(procedure, false, true);
   }
 
+  @Override
+  public void addBack(final Procedure procedure, boolean notify) {
+    push(procedure, false, notify);
+  }
+
   protected void push(final Procedure procedure, final boolean addFront, final boolean notify) {
     schedLock();
     try {
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 089caa06ff7..cf81b8a9b44 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -720,6 +720,10 @@ public abstract class Procedure<TEnvironment> implements Comparable<Procedure<TE
     this.lockedWhenLoading = true;
   }
 
+  public boolean isLockedWhenLoading() {
+    return lockedWhenLoading;
+  }
+
   // ==============================================================================================
   //  Runtime state, updated every operation by the ProcedureExecutor
   //
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index d67b420143c..916c4aff659 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -652,8 +652,17 @@ public class ProcedureExecutor<TEnvironment> {
       if (!p.hasParent()) {
         sendProcedureLoadedNotification(p.getProcId());
       }
-      scheduler.addBack(p);
+      // If the procedure holds the lock, put the procedure in front
+      if (p.isLockedWhenLoading()) {
+        scheduler.addFront(p, false);
+      } else {
+        // if it was not, it can wait.
+        scheduler.addBack(p, false);
+      }
     });
+    // After all procedures put into the queue, signal the worker threads.
+    // Otherwise, there is a race condition. See HBASE-21364.
+    scheduler.signalAll();
   }
 
   /**
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java
index e7e1cdbe407..9489f52f3fa 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java
@@ -52,6 +52,13 @@ public interface ProcedureScheduler {
    */
   void addFront(Procedure proc);
 
+  /**
+   * Inserts the specified element at the front of this queue.
+   * @param proc the Procedure to add
+   * @param notify whether need to notify worker
+   */
+  void addFront(Procedure proc, boolean notify);
+
   /**
    * Inserts all elements in the iterator at the front of this queue.
    */
@@ -63,6 +70,13 @@ public interface ProcedureScheduler {
    */
   void addBack(Procedure proc);
 
+  /**
+   * Inserts the specified element at the end of this queue.
+   * @param proc the Procedure to add
+   * @param notify whether need to notify worker
+   */
+  void addBack(Procedure proc, boolean notify);
+
   /**
    * The procedure can't run at the moment.
    * add it back to the queue, giving priority to someone else.