previouslyFailedServers =
master.getMasterWalManager().getFailedServersFromLogFolders();
if (serverName != null && previouslyFailedServers.contains(serverName)) {
+ am.getRegionStates().metaLogSplitting(serverName);
master.getMasterWalManager().splitMetaLog(serverName);
+ am.getRegionStates().metaLogSplit(serverName);
}
}
}
setNextState(RecoverMetaState.RECOVER_META_ASSIGN_REGIONS);
break;
-
case RECOVER_META_ASSIGN_REGIONS:
RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
RegionInfoBuilder.FIRST_META_REGIONINFO, this.replicaId);
@@ -258,7 +260,7 @@ public class RecoverMetaProcedure
@Override
protected LockState acquireLock(MasterProcedureEnv env) {
- if (env.getProcedureScheduler().waitTableExclusiveLock(this, TableName.META_TABLE_NAME)) {
+ if (env.getProcedureScheduler().waitMetaExclusiveLock(this)) {
return LockState.LOCK_EVENT_WAIT;
}
return LockState.LOCK_ACQUIRED;
@@ -266,7 +268,7 @@ public class RecoverMetaProcedure
@Override
protected void releaseLock(MasterProcedureEnv env) {
- env.getProcedureScheduler().wakeTableExclusiveLock(this, TableName.META_TABLE_NAME);
+ env.getProcedureScheduler().wakeMetaExclusiveLock(this);
}
@Override
@@ -274,16 +276,6 @@ public class RecoverMetaProcedure
ProcedurePrepareLatch.releaseLatch(syncLatch, this);
}
- @Override
- public TableName getTableName() {
- return TableName.META_TABLE_NAME;
- }
-
- @Override
- public TableOperationType getTableOperationType() {
- return TableOperationType.ENABLE;
- }
-
/**
* @return true if failedMetaServer is not null (meta carrying server crashed) or meta is
* already initialized
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SchemaLocking.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SchemaLocking.java
index 5dcc1218c2b..4bd35f3f250 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SchemaLocking.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SchemaLocking.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hbase.procedure2.LockedResourceType;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
/**
*
* Locks on namespaces, tables, and regions.
@@ -49,6 +51,7 @@ class SchemaLocking {
// Single map for all regions irrespective of tables. Key is encoded region name.
private final Map regionLocks = new HashMap<>();
private final Map peerLocks = new HashMap<>();
+ private final LockAndQueue metaLock = new LockAndQueue();
private LockAndQueue getLock(Map map, T key) {
LockAndQueue lock = map.get(key);
@@ -75,6 +78,10 @@ class SchemaLocking {
return getLock(regionLocks, encodedRegionName);
}
+ LockAndQueue getMetaLock() {
+ return metaLock;
+ }
+
LockAndQueue removeRegionLock(String encodedRegionName) {
return regionLocks.remove(encodedRegionName);
}
@@ -144,6 +151,8 @@ class SchemaLocking {
addToLockedResources(lockedResources, regionLocks, Function.identity(),
LockedResourceType.REGION);
addToLockedResources(lockedResources, peerLocks, Function.identity(), LockedResourceType.PEER);
+ addToLockedResources(lockedResources, ImmutableMap.of(TableName.META_TABLE_NAME, metaLock),
+ tn -> tn.getNameAsString(), LockedResourceType.META);
return lockedResources;
}
@@ -169,6 +178,8 @@ class SchemaLocking {
case PEER:
queue = peerLocks.get(resourceName);
break;
+ case META:
+ queue = metaLock;
default:
queue = null;
break;
@@ -193,7 +204,8 @@ class SchemaLocking {
return "serverLocks=" + filterUnlocked(this.serverLocks) + ", namespaceLocks=" +
filterUnlocked(this.namespaceLocks) + ", tableLocks=" + filterUnlocked(this.tableLocks) +
", regionLocks=" + filterUnlocked(this.regionLocks) + ", peerLocks=" +
- filterUnlocked(this.peerLocks);
+ filterUnlocked(this.peerLocks) + ", metaLocks=" +
+ filterUnlocked(ImmutableMap.of(TableName.META_TABLE_NAME, metaLock));
}
private String filterUnlocked(Map locks) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index ae709cdf92b..5a4c10fabcc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -55,8 +55,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.S
*/
@InterfaceAudience.Private
public class ServerCrashProcedure
-extends StateMachineProcedure
-implements ServerProcedureInterface {
+ extends StateMachineProcedure
+ implements ServerProcedureInterface {
private static final Logger LOG = LoggerFactory.getLogger(ServerCrashProcedure.class);
/**
@@ -163,11 +163,11 @@ implements ServerProcedureInterface {
"; cycles=" + getCycles());
}
// Handle RIT against crashed server. Will cancel any ongoing assigns/unassigns.
- // Returns list of regions we need to reassign. NOTE: there is nothing to stop a
- // dispatch happening AFTER this point. Check for the condition if a dispatch RPC fails
- // inside in AssignProcedure/UnassignProcedure. AssignProcedure just keeps retrying.
- // UnassignProcedure is more complicated. See where it does the check by calling
- // am#isDeadServerProcessed.
+ // Returns list of regions we need to reassign.
+ // NOTE: there is nothing to stop a dispatch happening AFTER this point. Check for the
+ // condition if a dispatch RPC fails inside in AssignProcedure/UnassignProcedure.
+ // AssignProcedure just keeps retrying. UnassignProcedure is more complicated. See where
+ // it does the check by calling am#isLogSplittingDone.
List toAssign = handleRIT(env, regionsOnCrashedServer);
AssignmentManager am = env.getAssignmentManager();
// CreateAssignProcedure will try to use the old location for the region deploy.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.java
index 798357ef272..f3c10efa425 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.java
@@ -19,14 +19,12 @@ package org.apache.hadoop.hbase.master.procedure;
import org.apache.hadoop.hbase.ServerName;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
/**
* Procedures that handle servers -- e.g. server crash -- must implement this Interface.
* It is used by the procedure runner to figure locking and what queuing.
*/
@InterfaceAudience.Private
-@InterfaceStability.Evolving
public interface ServerProcedureInterface {
public enum ServerOperationType {
CRASH_HANDLER
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
index b9ead26b5f3..4fd582dc5b3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
@@ -15,19 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.master.procedure;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
/**
* Procedures that operates on a specific Table (e.g. create, delete, snapshot, ...)
* must implement this interface to allow the system handle the lock/concurrency problems.
*/
@InterfaceAudience.Private
-@InterfaceStability.Evolving
public interface TableProcedureInterface {
public enum TableOperationType {
CREATE, DELETE, DISABLE, EDIT, ENABLE, READ,
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/DummyRegionProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/DummyRegionProcedure.java
new file mode 100644
index 00000000000..8d0df5f7916
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/DummyRegionProcedure.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineRegionProcedure;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+
+public class DummyRegionProcedure
+ extends AbstractStateMachineRegionProcedure {
+
+ private final CountDownLatch arrive = new CountDownLatch(1);
+
+ private final CountDownLatch resume = new CountDownLatch(1);
+
+ public DummyRegionProcedure() {
+ }
+
+ public DummyRegionProcedure(MasterProcedureEnv env, RegionInfo hri) {
+ super(env, hri);
+ }
+
+ @Override
+ public TableOperationType getTableOperationType() {
+ return TableOperationType.REGION_EDIT;
+ }
+
+ @Override
+ protected Flow executeFromState(MasterProcedureEnv env, DummyRegionProcedureState state)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ arrive.countDown();
+ resume.await();
+ return Flow.NO_MORE_STATE;
+ }
+
+ @Override
+ protected void rollbackState(MasterProcedureEnv env, DummyRegionProcedureState state)
+ throws IOException, InterruptedException {
+ }
+
+ @Override
+ protected DummyRegionProcedureState getState(int stateId) {
+ return DummyRegionProcedureState.STATE;
+ }
+
+ @Override
+ protected int getStateId(DummyRegionProcedureState state) {
+ return 0;
+ }
+
+ @Override
+ protected DummyRegionProcedureState getInitialState() {
+ return DummyRegionProcedureState.STATE;
+ }
+
+ public void waitUntilArrive() throws InterruptedException {
+ arrive.await();
+ }
+
+ public void resume() {
+ resume.countDown();
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/DummyRegionProcedureState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/DummyRegionProcedureState.java
new file mode 100644
index 00000000000..bcce7e67636
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/DummyRegionProcedureState.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+public enum DummyRegionProcedureState {
+ STATE
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index 95ab2295249..c50ae0b6287 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -261,8 +260,8 @@ public class TestMasterNoCluster {
HMaster master = new HMaster(conf) {
@Override
- MasterMetaBootstrap createMetaBootstrap(final HMaster master, final MonitoredTask status) {
- return new MasterMetaBootstrap(this, status) {
+ protected MasterMetaBootstrap createMetaBootstrap() {
+ return new MasterMetaBootstrap(this) {
@Override
protected void assignMetaReplicas()
throws IOException, InterruptedException, KeeperException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestServerCrashProcedureCarryingMetaStuck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestServerCrashProcedureCarryingMetaStuck.java
new file mode 100644
index 00000000000..748cd0e6ada
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestServerCrashProcedureCarryingMetaStuck.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncAdmin;
+import org.apache.hadoop.hbase.client.AsyncConnection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.assignment.AssignProcedure;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, LargeTests.class })
+public class TestServerCrashProcedureCarryingMetaStuck {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestServerCrashProcedureCarryingMetaStuck.class);
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.startMiniCluster(3);
+ UTIL.getAdmin().balancerSwitch(false, true);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void test() throws Exception {
+ RegionServerThread rsThread = null;
+ for (RegionServerThread t : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
+ if (!t.getRegionServer().getRegions(TableName.META_TABLE_NAME).isEmpty()) {
+ rsThread = t;
+ break;
+ }
+ }
+ HRegionServer rs = rsThread.getRegionServer();
+ RegionInfo hri = rs.getRegions(TableName.META_TABLE_NAME).get(0).getRegionInfo();
+ HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+ ProcedureExecutor executor = master.getMasterProcedureExecutor();
+ DummyRegionProcedure proc = new DummyRegionProcedure(executor.getEnvironment(), hri);
+ long procId = master.getMasterProcedureExecutor().submitProcedure(proc);
+ proc.waitUntilArrive();
+ try (AsyncConnection conn =
+ ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get()) {
+ AsyncAdmin admin = conn.getAdmin();
+ CompletableFuture future = admin.move(hri.getRegionName());
+ rs.abort("For testing!");
+
+ UTIL.waitFor(30000,
+ () -> executor.getProcedures().stream().filter(p -> p instanceof AssignProcedure)
+ .map(p -> (AssignProcedure) p)
+ .anyMatch(p -> Bytes.equals(hri.getRegionName(), p.getRegionInfo().getRegionName())));
+ proc.resume();
+ UTIL.waitFor(30000, () -> executor.isFinished(procId));
+ // see whether the move region procedure can finish properly
+ future.get(30, TimeUnit.SECONDS);
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestServerCrashProcedureStuck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestServerCrashProcedureStuck.java
index a83e0d2ca6d..26816573e23 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestServerCrashProcedureStuck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestServerCrashProcedureStuck.java
@@ -17,12 +17,8 @@
*/
package org.apache.hadoop.hbase.master;
-import java.io.IOException;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
@@ -31,11 +27,8 @@ import org.apache.hadoop.hbase.client.AsyncConnection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.master.assignment.AssignProcedure;
-import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineRegionProcedure;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
-import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
-import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -63,58 +56,6 @@ public class TestServerCrashProcedureStuck {
private static byte[] CF = Bytes.toBytes("cf");
- private static CountDownLatch ARRIVE = new CountDownLatch(1);
-
- private static CountDownLatch RESUME = new CountDownLatch(1);
-
- public enum DummyState {
- STATE
- }
-
- public static final class DummyRegionProcedure
- extends AbstractStateMachineRegionProcedure {
-
- public DummyRegionProcedure() {
- }
-
- public DummyRegionProcedure(MasterProcedureEnv env, RegionInfo hri) {
- super(env, hri);
- }
-
- @Override
- public TableOperationType getTableOperationType() {
- return TableOperationType.REGION_EDIT;
- }
-
- @Override
- protected Flow executeFromState(MasterProcedureEnv env, DummyState state)
- throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
- ARRIVE.countDown();
- RESUME.await();
- return Flow.NO_MORE_STATE;
- }
-
- @Override
- protected void rollbackState(MasterProcedureEnv env, DummyState state)
- throws IOException, InterruptedException {
- }
-
- @Override
- protected DummyState getState(int stateId) {
- return DummyState.STATE;
- }
-
- @Override
- protected int getStateId(DummyState state) {
- return 0;
- }
-
- @Override
- protected DummyState getInitialState() {
- return DummyState.STATE;
- }
- }
-
@BeforeClass
public static void setUp() throws Exception {
UTIL.startMiniCluster(3);
@@ -129,8 +70,7 @@ public class TestServerCrashProcedureStuck {
}
@Test
- public void test()
- throws IOException, InterruptedException, ExecutionException, TimeoutException {
+ public void test() throws Exception {
RegionServerThread rsThread = null;
for (RegionServerThread t : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
if (!t.getRegionServer().getRegions(TABLE_NAME).isEmpty()) {
@@ -144,7 +84,7 @@ public class TestServerCrashProcedureStuck {
ProcedureExecutor executor = master.getMasterProcedureExecutor();
DummyRegionProcedure proc = new DummyRegionProcedure(executor.getEnvironment(), hri);
long procId = master.getMasterProcedureExecutor().submitProcedure(proc);
- ARRIVE.await();
+ proc.waitUntilArrive();
try (AsyncConnection conn =
ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get()) {
AsyncAdmin admin = conn.getAdmin();
@@ -155,7 +95,7 @@ public class TestServerCrashProcedureStuck {
() -> executor.getProcedures().stream().filter(p -> p instanceof AssignProcedure)
.map(p -> (AssignProcedure) p)
.anyMatch(p -> Bytes.equals(hri.getRegionName(), p.getRegionInfo().getRegionName())));
- RESUME.countDown();
+ proc.resume();
UTIL.waitFor(30000, () -> executor.isFinished(procId));
// see whether the move region procedure can finish properly
future.get(30, TimeUnit.SECONDS);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index 785e85f3ce8..3f61de462f0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -53,7 +52,6 @@ import org.apache.hadoop.hbase.master.MasterMetaBootstrap;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.TableStateManager;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
@@ -98,8 +96,7 @@ public class MasterProcedureTestingUtility {
public Void call() throws Exception {
final AssignmentManager am = env.getAssignmentManager();
am.start();
- MasterMetaBootstrap metaBootstrap = new MasterMetaBootstrap(master,
- TaskMonitor.get().createStatus("meta"));
+ MasterMetaBootstrap metaBootstrap = new MasterMetaBootstrap(master);
metaBootstrap.recoverMeta();
metaBootstrap.processDeadServers();
am.joinCluster();