From 09e8aff79c6696bb2afdf39d1f6e2fdd72f01f5b Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Wed, 5 Feb 2020 22:01:19 -0800
Subject: [PATCH] HBASE-23805 [Flakey Test] TestRaceBetweenSCPAndDTP

---
 .../assignment/TestRaceBetweenSCPAndDTP.java  | 32 +++++++++++++------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java
index 2df791a17d3..d3054b7185e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndDTP.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.master.assignment;
 
 import java.io.IOException;
+import java.util.Optional;
 import java.util.concurrent.CountDownLatch;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -30,10 +31,12 @@ import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
 import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -46,7 +49,6 @@ import org.junit.experimental.categories.Category;
  */
 @Category({ MasterTests.class, MediumTests.class })
 public class TestRaceBetweenSCPAndDTP {
-
   @ClassRule
   public static final HBaseClassTestRule CLASS_RULE =
     HBaseClassTestRule.forClass(TestRaceBetweenSCPAndDTP.class);
@@ -120,15 +122,16 @@ public class TestRaceBetweenSCPAndDTP {
     UTIL.getAdmin().disableTableAsync(NAME);
     ARRIVE_GET_REGIONS_ON_TABLE.await();
 
-    UTIL.getMiniHBaseCluster().stopRegionServer(sn);
-    // Wait ServerCrashProcedure to init.
-    Thread.sleep(1000);
     ProcedureExecutor<?> procExec =
-        UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
-    long scpProcId =
-        procExec.getProcedures().stream().filter(p -> p instanceof ServerCrashProcedure)
-            .map(p -> (ServerCrashProcedure) p).findAny().get().getProcId();
-    UTIL.waitFor(60000, () -> procExec.isFinished(scpProcId));
+      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
+    UTIL.getMiniHBaseCluster().stopRegionServer(sn);
+    long pid = Procedure.NO_PROC_ID;
+    do {
+      Threads.sleep(1);
+      pid = getSCPPID(procExec);
+    } while (pid != Procedure.NO_PROC_ID);
+    final long scppid = pid;
+    UTIL.waitFor(60000, () -> procExec.isFinished(scppid));
     RESUME_GET_REGIONS_ON_SERVER.countDown();
 
     long dtpProcId =
@@ -136,4 +139,13 @@ public class TestRaceBetweenSCPAndDTP {
             .map(p -> (DisableTableProcedure) p).findAny().get().getProcId();
     UTIL.waitFor(60000, () -> procExec.isFinished(dtpProcId));
   }
+
+  /**
+   * @return Returns {@link Procedure#NO_PROC_ID} if no SCP found else actual pid.
+   */
+  private long getSCPPID(ProcedureExecutor<?> e) {
+    Optional<ServerCrashProcedure> optional = e.getProcedures().stream().
+      filter(p -> p instanceof ServerCrashProcedure).map(p -> (ServerCrashProcedure) p).findAny();
+    return optional.isPresent()? optional.get().getProcId(): Procedure.NO_PROC_ID;
+  }
 }