HBASE-20024 TestMergeTableRegionsProcedure is STILL flakey

This commit is contained in:
Michael Stack 2018-02-20 07:21:52 -08:00
parent a5443c18d2
commit 9be0360c5d
8 changed files with 29 additions and 19 deletions

View File

@ -931,18 +931,18 @@ public class ProcedureExecutor<TEnvironment> {
* Send an abort notification the specified procedure. * Send an abort notification the specified procedure.
* Depending on the procedure implementation the abort can be considered or ignored. * Depending on the procedure implementation the abort can be considered or ignored.
* @param procId the procedure to abort * @param procId the procedure to abort
* @return true if the procedure exist and has received the abort, otherwise false. * @return true if the procedure exists and has received the abort, otherwise false.
*/ */
public boolean abort(final long procId) { public boolean abort(final long procId) {
return abort(procId, true); return abort(procId, true);
} }
/** /**
* Send an abort notification the specified procedure. * Send an abort notification to the specified procedure.
* Depending on the procedure implementation the abort can be considered or ignored. * Depending on the procedure implementation, the abort can be considered or ignored.
* @param procId the procedure to abort * @param procId the procedure to abort
* @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted? * @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted?
* @return true if the procedure exist and has received the abort, otherwise false. * @return true if the procedure exists and has received the abort, otherwise false.
*/ */
public boolean abort(final long procId, final boolean mayInterruptIfRunning) { public boolean abort(final long procId, final boolean mayInterruptIfRunning) {
final Procedure proc = procedures.get(procId); final Procedure proc = procedures.get(procId);

View File

@ -209,18 +209,13 @@ public abstract class StateMachineProcedure<TEnvironment, TState>
@Override @Override
protected boolean abort(final TEnvironment env) { protected boolean abort(final TEnvironment env) {
final boolean isDebugEnabled = LOG.isDebugEnabled();
final TState state = getCurrentState(); final TState state = getCurrentState();
if (isDebugEnabled) { LOG.debug("Abort requested for {}", this);
LOG.debug("abort requested for " + this + " state=" + state);
}
if (hasMoreState()) { if (hasMoreState()) {
aborted.set(true); aborted.set(true);
return true; return true;
} else if (isDebugEnabled) {
LOG.debug("ignoring abort request on state=" + state + " for " + this);
} }
LOG.debug("Ignoring abort request on {}", this);
return false; return false;
} }

View File

@ -795,4 +795,12 @@ public class MergeTableRegionsProcedure
public RegionInfo getMergedRegion() { public RegionInfo getMergedRegion() {
return this.mergedRegion; return this.mergedRegion;
} }
@Override
protected boolean abort(MasterProcedureEnv env) {
// Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all
// Procedures. Here is a Procedure that has a PONR and cannot be aborted wants it enters this
// range of steps; what do we do for these should an operator want to cancel them? HBASE-20022.
return isRollbackSupported(getCurrentState())? super.abort(env): false;
}
} }

View File

@ -262,14 +262,13 @@ public class SplitTableRegionProcedure
throw new UnsupportedOperationException(this + " unhandled state=" + state); throw new UnsupportedOperationException(this + " unhandled state=" + state);
} }
} catch (IOException e) { } catch (IOException e) {
String msg = "Error trying to split region " + getParentRegion().getEncodedName() + " in the table " String msg = "Error trying to split region " + getParentRegion().getEncodedName() +
+ getTableName() + " (in state=" + state + ")"; " in the table " + getTableName() + " (in state=" + state + ")";
if (!isRollbackSupported(state)) { if (!isRollbackSupported(state)) {
// We reach a state that cannot be rolled back. We just need to keep retry. // We reach a state that cannot be rolled back. We just need to keep retry.
LOG.warn(msg, e); LOG.warn(msg, e);
} else { } else {
LOG.error(msg, e); LOG.error(msg, e);
setFailure(e);
setFailure("master-split-regions", e); setFailure("master-split-regions", e);
} }
} }
@ -831,4 +830,12 @@ public class SplitTableRegionProcedure
} }
return traceEnabled; return traceEnabled;
} }
@Override
protected boolean abort(MasterProcedureEnv env) {
// Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all
// Procedures. Here is a Procedure that has a PONR and cannot be aborted wants it enters this
// range of steps; what do we do for these should an operator want to cancel them? HBASE-20022.
return isRollbackSupported(getCurrentState())? super.abort(env): false;
}
} }

View File

@ -144,8 +144,8 @@ public class DeleteTableProcedure
@Override @Override
protected boolean abort(MasterProcedureEnv env) { protected boolean abort(MasterProcedureEnv env) {
// TODO: Current behavior is: with no rollback and no abort support, procedure may stuck // TODO: Current behavior is: with no rollback and no abort support, procedure may get stuck
// looping in retrying failing step forever. Default behavior of abort is changed to support // looping in retrying failing a step forever. Default behavior of abort is changed to support
// aborting all procedures. Override the default wisely. Following code retains the current // aborting all procedures. Override the default wisely. Following code retains the current
// behavior. Revisit it later. // behavior. Revisit it later.
return isRollbackSupported(getCurrentState()) ? super.abort(env) : false; return isRollbackSupported(getCurrentState()) ? super.abort(env) : false;

View File

@ -568,7 +568,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
boolean hasLock = true; boolean hasLock = true;
final LockAndQueue[] regionLocks = new LockAndQueue[regionInfo.length]; final LockAndQueue[] regionLocks = new LockAndQueue[regionInfo.length];
for (int i = 0; i < regionInfo.length; ++i) { for (int i = 0; i < regionInfo.length; ++i) {
LOG.info(procedure + ", table=" + table + ", " + regionInfo[i].getRegionNameAsString()); LOG.info(procedure + ", " + regionInfo[i].getRegionNameAsString());
assert table != null; assert table != null;
assert regionInfo[i] != null; assert regionInfo[i] != null;
assert regionInfo[i].getTable() != null; assert regionInfo[i].getTable() != null;

View File

@ -276,7 +276,7 @@ public class TestMergeTableRegionsProcedure {
@Test @Test
public void testMergeWithoutPONR() throws Exception { public void testMergeWithoutPONR() throws Exception {
final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution"); final TableName tableName = TableName.valueOf("testMergeWithoutPONR");
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
List<RegionInfo> tableRegions = createTable(tableName); List<RegionInfo> tableRegions = createTable(tableName);

View File

@ -401,7 +401,7 @@ public class MasterProcedureTestingUtility {
* idempotent. Use this version of the test when the order in which flow steps are executed is * idempotent. Use this version of the test when the order in which flow steps are executed is
* not start to finish; where the procedure may vary the flow steps dependent on circumstance * not start to finish; where the procedure may vary the flow steps dependent on circumstance
* found. * found.
* @see #testRecoveryAndDoubleExecution(ProcedureExecutor, long, int) * @see #testRecoveryAndDoubleExecution(ProcedureExecutor, long, int, boolean)
*/ */
public static void testRecoveryAndDoubleExecution( public static void testRecoveryAndDoubleExecution(
final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId) throws Exception { final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId) throws Exception {