diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java index 2a678c04d2b..646bc1fccba 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java @@ -161,12 +161,6 @@ public abstract class AbstractProcedureScheduler implements ProcedureScheduler { // ========================================================================== // Utils // ========================================================================== - /** - * Removes all of the elements from the queue - * NOTE: this method is called with the sched lock held. - */ - protected abstract void clearQueue(); - /** * Returns the number of elements in this queue. * NOTE: this method is called with the sched lock held. @@ -181,17 +175,6 @@ public abstract class AbstractProcedureScheduler implements ProcedureScheduler { */ protected abstract boolean queueHasRunnables(); - @Override - public void clear() { - // NOTE: USED ONLY FOR TESTING - schedLock(); - try { - clearQueue(); - } finally { - schedUnlock(); - } - } - @Override public int size() { schedLock(); diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index 0dc16246558..b38b96c6520 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -550,6 +550,7 @@ public class ProcedureExecutor { timeoutExecutor.sendStopSignal(); } + @VisibleForTesting public void join() { assert !isRunning() : "expected not running"; diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java index 1793158c308..16ff7819e68 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java @@ -130,7 +130,9 @@ public interface ProcedureScheduler { int size(); /** - * Removes all of the elements from the queue + * Clear current state of scheduler such that it is equivalent to newly created scheduler. + * Used for testing failure and recovery. To emulate server crash/restart, + * {@link ProcedureExecutor} resets its own state and calls clear() on scheduler. */ @VisibleForTesting void clear(); diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SimpleProcedureScheduler.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SimpleProcedureScheduler.java index 1a6775ae15d..788f4ff4593 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SimpleProcedureScheduler.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SimpleProcedureScheduler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.procedure2; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -43,9 +44,15 @@ public class SimpleProcedureScheduler extends AbstractProcedureScheduler { return runnables.poll(); } + @VisibleForTesting @Override - protected void clearQueue() { - runnables.clear(); + public void clear() { + schedLock(); + try { + runnables.clear(); + } finally { + schedUnlock(); + } } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java index a3adf025414..508f7c4e135 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java @@ -227,8 +227,19 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { return pollResult; } + @VisibleForTesting @Override - protected void clearQueue() { + public void clear() { + schedLock(); + try { + clearQueue(); + locking.clear(); + } finally { + schedUnlock(); + } + } + + private void clearQueue() { // Remove Servers for (int i = 0; i < serverBuckets.length; ++i) { clear(serverBuckets[i], serverRunQueue, SERVER_QUEUE_KEY_COMPARATOR); @@ -922,30 +933,42 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { return lock; } - public LockAndQueue getTableLock(TableName tableName) { + LockAndQueue getTableLock(TableName tableName) { return getLock(tableLocks, tableName); } - public LockAndQueue removeTableLock(TableName tableName) { + LockAndQueue removeTableLock(TableName tableName) { return tableLocks.remove(tableName); } - public LockAndQueue getNamespaceLock(String namespace) { + LockAndQueue getNamespaceLock(String namespace) { return getLock(namespaceLocks, namespace); } - public LockAndQueue getRegionLock(String encodedRegionName) { + LockAndQueue getRegionLock(String encodedRegionName) { return getLock(regionLocks, encodedRegionName); } - public LockAndQueue removeRegionLock(String encodedRegionName) { + LockAndQueue removeRegionLock(String encodedRegionName) { return regionLocks.remove(encodedRegionName); } - public LockAndQueue getServerLock(ServerName serverName) { + LockAndQueue getServerLock(ServerName serverName) { return getLock(serverLocks, serverName); } + /** + * Removes all locks by clearing the maps. + * Used when procedure executor is stopped for failure and recovery testing. + */ + @VisibleForTesting + void clear() { + serverLocks.clear(); + namespaceLocks.clear(); + tableLocks.clear(); + regionLocks.clear(); + } + final Map serverLocks = new HashMap<>(); final Map namespaceLocks = new HashMap<>(); final Map tableLocks = new HashMap<>();