HBASE-21291 Add a test for bypassing stuck state-machine procedures

Signed-off-by: Michael Stack <stack@apache.org>

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java

@@ -1054,6 +1054,7 @@ public class ProcedureExecutor<TEnvironment> {
 
   boolean bypassProcedure(long pid, long lockWait, boolean override, boolean recursive)
       throws IOException {
+    Preconditions.checkArgument(lockWait > 0, "lockWait should be positive");
     final Procedure<TEnvironment> procedure = getProcedure(pid);
     if (procedure == null) {
       LOG.debug("Procedure pid={} does not exist, skipping bypass", pid);

hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java

@@ -389,6 +389,46 @@ public class ProcedureTestingUtility {
     }
   }
 
+  public static class NoopStateMachineProcedure<TEnv, TState>
+      extends StateMachineProcedure<TEnv, TState> {
+    private TState initialState;
+    private TEnv env;
+
+    public NoopStateMachineProcedure() {
+    }
+
+    public NoopStateMachineProcedure(TEnv env, TState initialState) {
+      this.env = env;
+      this.initialState = initialState;
+    }
+
+    @Override
+    protected Flow executeFromState(TEnv env, TState tState)
+        throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+      return null;
+    }
+
+    @Override
+    protected void rollbackState(TEnv env, TState tState) throws IOException, InterruptedException {
+
+    }
+
+    @Override
+    protected TState getState(int stateId) {
+      return null;
+    }
+
+    @Override
+    protected int getStateId(TState tState) {
+      return 0;
+    }
+
+    @Override
+    protected TState getInitialState() {
+      return initialState;
+    }
+  }
+
   public static class TestProcedure extends NoopProcedure<Void> {
     private byte[] data = null;
 

hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java

@@ -17,8 +17,10 @@
  */
 package org.apache.hadoop.hbase.procedure2;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 import org.apache.hadoop.fs.FileSystem;
@@ -133,6 +135,20 @@ public class TestProcedureBypass {
     LOG.info("{} finished", proc);
   }
 
+  @Test
+  public void testBypassingStuckStateMachineProcedure() throws Exception {
+    final StuckStateMachineProcedure proc =
+        new StuckStateMachineProcedure(procEnv, StuckStateMachineState.START);
+    long id = procExecutor.submitProcedure(proc);
+    Thread.sleep(500);
+    // bypass the procedure
+    assertFalse(procExecutor.bypassProcedure(id, 1000, false, false));
+    assertTrue(procExecutor.bypassProcedure(id, 1000, true, false));
+
+    htu.waitFor(5000, () -> proc.isSuccess() && proc.isBypass());
+    LOG.info("{} finished", proc);
+  }
+
   @AfterClass
   public static void tearDown() throws Exception {
     procExecutor.stop();
@@ -191,4 +207,51 @@ public class TestProcedureBypass {
       }
     }
   }
+
+  public enum StuckStateMachineState {
+    START, THEN, END
+  }
+
+  public static class StuckStateMachineProcedure extends
+      ProcedureTestingUtility.NoopStateMachineProcedure<TestProcEnv, StuckStateMachineState> {
+    private AtomicBoolean stop = new AtomicBoolean(false);
+
+    public StuckStateMachineProcedure() {
+      super();
+    }
+
+    public StuckStateMachineProcedure(TestProcEnv env, StuckStateMachineState initialState) {
+      super(env, initialState);
+    }
+
+    @Override
+    protected Flow executeFromState(TestProcEnv env, StuckStateMachineState tState)
+            throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+      switch (tState) {
+        case START:
+          LOG.info("PHASE 1: START");
+          setNextState(StuckStateMachineState.THEN);
+          return Flow.HAS_MORE_STATE;
+        case THEN:
+          if (stop.get()) {
+            setNextState(StuckStateMachineState.END);
+          }
+          return Flow.HAS_MORE_STATE;
+        case END:
+          return Flow.NO_MORE_STATE;
+        default:
+          throw new UnsupportedOperationException("unhandled state=" + tState);
+      }
+    }
+
+    @Override
+    protected StuckStateMachineState getState(int stateId) {
+      return StuckStateMachineState.values()[stateId];
+    }
+
+    @Override
+    protected int getStateId(StuckStateMachineState tState) {
+      return tState.ordinal();
+    }
+  }
 }