mirror of https://github.com/apache/jclouds.git
Issue 966: retry when exitstatus shows process is still running
This commit is contained in:
Adrian Cole
parent
8b570b499c
commit
6eec5d5c24
|
|
@ -33,16 +33,15 @@ import org.jclouds.Constants;
|
|||
import org.jclouds.compute.domain.ExecResponse;
|
||||
import org.jclouds.compute.events.StatementOnNodeCompletion;
|
||||
import org.jclouds.compute.events.StatementOnNodeFailure;
|
||||
import org.jclouds.compute.predicates.ScriptStatusReturnsZero;
|
||||
import org.jclouds.compute.reference.ComputeServiceConstants;
|
||||
import org.jclouds.logging.Logger;
|
||||
import org.jclouds.predicates.RetryablePredicate;
|
||||
import org.jclouds.scriptbuilder.InitScript;
|
||||
import org.jclouds.ssh.SshClient;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Objects;
|
||||
import com.google.common.base.Predicate;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.base.Predicates;
|
||||
import com.google.common.eventbus.EventBus;
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.google.common.util.concurrent.AbstractFuture;
|
||||
|
|
@ -51,12 +50,11 @@ import com.google.inject.assistedinject.Assisted;
|
|||
import com.google.inject.name.Named;
|
||||
|
||||
/**
|
||||
* A future that works in tandem with a task that was invoked by
|
||||
* {@link InitScript}
|
||||
* A future that works in tandem with a task that was invoked by {@link InitScript}
|
||||
*
|
||||
* @author Adrian Cole
|
||||
*/
|
||||
public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFuture<ExecResponse> {
|
||||
public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFuture<ExecResponse> implements Runnable {
|
||||
|
||||
public static interface Factory {
|
||||
BlockUntilInitScriptStatusIsZeroThenReturnOutput create(SudoAwareInitManager commandRunner);
|
||||
|
|
@ -74,86 +72,98 @@ public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFu
|
|||
return commandRunner;
|
||||
}
|
||||
|
||||
private final RetryablePredicate<String> notRunningAnymore;
|
||||
private Predicate<String> notRunningAnymore;
|
||||
|
||||
@Inject
|
||||
public BlockUntilInitScriptStatusIsZeroThenReturnOutput(
|
||||
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userThreads, EventBus eventBus,
|
||||
ComputeServiceConstants.InitStatusProperties properties, final ScriptStatusReturnsZero stateRunning,
|
||||
@Assisted final SudoAwareInitManager commandRunner) {
|
||||
|
||||
long retryMaxWait = TimeUnit.DAYS.toMillis(365); // arbitrarily high
|
||||
// value, but
|
||||
// Long.MAX_VALUE doesn't
|
||||
// work!
|
||||
long retryInitialPeriod = properties.initStatusInitialPeriod;
|
||||
long retryMaxPeriod = properties.initStatusMaxPeriod;
|
||||
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userThreads, EventBus eventBus,
|
||||
ComputeServiceConstants.InitStatusProperties properties, @Assisted SudoAwareInitManager commandRunner) {
|
||||
this(userThreads, eventBus, Predicates.<String> alwaysTrue(), commandRunner);
|
||||
// this is mutable only until we can determine how to decouple "this" from here
|
||||
notRunningAnymore = new LoopUntilTrueOrThrowCancellationException(new ExitStatusOfCommandGreaterThanZero(
|
||||
commandRunner), properties.initStatusMaxPeriod, properties.initStatusInitialPeriod, this);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public BlockUntilInitScriptStatusIsZeroThenReturnOutput(ExecutorService userThreads, EventBus eventBus,
|
||||
Predicate<String> notRunningAnymore, SudoAwareInitManager commandRunner) {
|
||||
this.commandRunner = checkNotNull(commandRunner, "commandRunner");
|
||||
this.userThreads = checkNotNull(userThreads, "userThreads");
|
||||
this.eventBus = checkNotNull(eventBus, "eventBus");
|
||||
this.notRunningAnymore = new RetryablePredicate<String>(new Predicate<String>() {
|
||||
|
||||
@Override
|
||||
public boolean apply(String arg0) {
|
||||
return commandRunner.runAction(arg0).getOutput().trim().equals("");
|
||||
}
|
||||
}, retryMaxWait, retryInitialPeriod, retryMaxPeriod, TimeUnit.MILLISECONDS) {
|
||||
/**
|
||||
* make sure we stop the retry loop if someone cancelled the future,
|
||||
* this keeps threads from being consumed on dead tasks
|
||||
*/
|
||||
@Override
|
||||
protected boolean atOrAfter(Date end) {
|
||||
if (isCancelled())
|
||||
Throwables.propagate(new CancellationException("cancelled"));
|
||||
return super.atOrAfter(end);
|
||||
}
|
||||
};
|
||||
this.notRunningAnymore = checkNotNull(notRunningAnymore, "notRunningAnymore");
|
||||
}
|
||||
|
||||
/**
|
||||
* in case login credentials or user changes at runtime.
|
||||
*/
|
||||
public void setSshClient(SshClient client) {
|
||||
@VisibleForTesting
|
||||
static class ExitStatusOfCommandGreaterThanZero implements Predicate<String> {
|
||||
private final SudoAwareInitManager commandRunner;
|
||||
|
||||
ExitStatusOfCommandGreaterThanZero(SudoAwareInitManager commandRunner) {
|
||||
this.commandRunner = commandRunner;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean apply(String input) {
|
||||
return commandRunner.runAction(input).getExitStatus() > 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static class LoopUntilTrueOrThrowCancellationException extends RetryablePredicate<String> {
|
||||
|
||||
private final AbstractFuture<ExecResponse> futureWhichMightBeCancelled;
|
||||
|
||||
public LoopUntilTrueOrThrowCancellationException(Predicate<String> predicate, long period, long maxPeriod,
|
||||
AbstractFuture<ExecResponse> futureWhichMightBeCancelled) {
|
||||
// arbitrarily high value, but Long.MAX_VALUE doesn't work!
|
||||
super(predicate, TimeUnit.DAYS.toMillis(365), period, maxPeriod, TimeUnit.MILLISECONDS);
|
||||
this.futureWhichMightBeCancelled = futureWhichMightBeCancelled;
|
||||
}
|
||||
|
||||
/**
|
||||
* make sure we stop the retry loop if someone cancelled the future, this keeps threads from
|
||||
* being consumed on dead tasks
|
||||
*/
|
||||
@Override
|
||||
protected boolean atOrAfter(Date end) {
|
||||
if (futureWhichMightBeCancelled.isCancelled())
|
||||
throw new CancellationException(futureWhichMightBeCancelled + " is cancelled");
|
||||
return super.atOrAfter(end);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Submits a thread that will either set the result of the future or the
|
||||
* exception that took place
|
||||
* Submits a thread that will either set the result of the future or the exception that took
|
||||
* place
|
||||
*/
|
||||
public BlockUntilInitScriptStatusIsZeroThenReturnOutput init() {
|
||||
userThreads.submit(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
notRunningAnymore.apply("status");
|
||||
String stdout = commandRunner.runAction("stdout").getOutput();
|
||||
String stderr = commandRunner.runAction("stderr").getOutput();
|
||||
Integer exitStatus = Ints.tryParse(commandRunner.runAction("exitstatus").getOutput().trim());
|
||||
ExecResponse exec = new ExecResponse(stdout, stderr, exitStatus == null ? -1 : exitStatus);
|
||||
if (exitStatus == null) {
|
||||
Integer pid = Ints.tryParse(commandRunner.runAction("status").getOutput().trim());
|
||||
throw new ScriptStillRunningException(String.format("%s still running: pid(%s), last status: %s",
|
||||
BlockUntilInitScriptStatusIsZeroThenReturnOutput.this, pid, exec),
|
||||
BlockUntilInitScriptStatusIsZeroThenReturnOutput.this);
|
||||
}
|
||||
logger.debug("<< complete(%s) status(%s)", commandRunner.getStatement().getInstanceName(), exitStatus);
|
||||
set(exec);
|
||||
} catch (CancellationException e) {
|
||||
} catch (Exception e) {
|
||||
setException(e);
|
||||
}
|
||||
}
|
||||
});
|
||||
userThreads.submit(this);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
ExecResponse exec = null;
|
||||
do {
|
||||
notRunningAnymore.apply("status");
|
||||
String stdout = commandRunner.runAction("stdout").getOutput();
|
||||
String stderr = commandRunner.runAction("stderr").getOutput();
|
||||
Integer exitStatus = Ints.tryParse(commandRunner.runAction("exitstatus").getOutput().trim());
|
||||
exec = new ExecResponse(stdout, stderr, exitStatus == null ? -1 : exitStatus);
|
||||
} while (!isCancelled() && exec.getExitStatus() == -1);
|
||||
logger.debug("<< complete(%s) status(%s)", commandRunner.getStatement().getInstanceName(), exec
|
||||
.getExitStatus());
|
||||
set(exec);
|
||||
} catch (Exception e) {
|
||||
setException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean set(ExecResponse value) {
|
||||
eventBus.post(new StatementOnNodeCompletion(getCommandRunner().getStatement(), getCommandRunner().getNode(),
|
||||
value));
|
||||
value));
|
||||
return super.set(value);
|
||||
}
|
||||
|
||||
|
|
@ -162,8 +172,8 @@ public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFu
|
|||
logger.debug("<< cancelled(%s)", commandRunner.getStatement().getInstanceName());
|
||||
ExecResponse returnVal = commandRunner.refreshAndRunAction("stop");
|
||||
CancellationException e = new CancellationException(String.format(
|
||||
"cancelled %s on node: %s; stop command had exit status: %s", getCommandRunner().getStatement()
|
||||
.getInstanceName(), getCommandRunner().getNode().getId(), returnVal));
|
||||
"cancelled %s on node: %s; stop command had exit status: %s", getCommandRunner().getStatement()
|
||||
.getInstanceName(), getCommandRunner().getNode().getId(), returnVal));
|
||||
eventBus.post(new StatementOnNodeFailure(getCommandRunner().getStatement(), getCommandRunner().getNode(), e));
|
||||
super.interruptTask();
|
||||
}
|
||||
|
|
@ -185,13 +195,13 @@ public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFu
|
|||
if (!o.getClass().equals(getClass()))
|
||||
return false;
|
||||
BlockUntilInitScriptStatusIsZeroThenReturnOutput that = BlockUntilInitScriptStatusIsZeroThenReturnOutput.class
|
||||
.cast(o);
|
||||
.cast(o);
|
||||
return Objects.equal(this.commandRunner, that.commandRunner);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExecResponse get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException,
|
||||
ExecutionException {
|
||||
ExecutionException {
|
||||
try {
|
||||
return super.get(timeout, unit);
|
||||
} catch (TimeoutException e) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,294 @@
|
|||
/**
|
||||
* Licensed to jclouds, Inc. (jclouds) under one or more
|
||||
* contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. jclouds licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.jclouds.compute.callables;
|
||||
|
||||
import static org.easymock.EasyMock.createMockBuilder;
|
||||
import static org.easymock.EasyMock.expect;
|
||||
import static org.easymock.EasyMock.replay;
|
||||
import static org.easymock.EasyMock.verify;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.fail;
|
||||
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.jclouds.compute.callables.BlockUntilInitScriptStatusIsZeroThenReturnOutput.ExitStatusOfCommandGreaterThanZero;
|
||||
import org.jclouds.compute.callables.BlockUntilInitScriptStatusIsZeroThenReturnOutput.LoopUntilTrueOrThrowCancellationException;
|
||||
import org.jclouds.compute.domain.ExecResponse;
|
||||
import org.jclouds.compute.domain.NodeMetadata;
|
||||
import org.jclouds.compute.domain.NodeMetadataBuilder;
|
||||
import org.jclouds.compute.domain.NodeState;
|
||||
import org.jclouds.scriptbuilder.InitScript;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.common.base.Predicate;
|
||||
import com.google.common.base.Predicates;
|
||||
import com.google.common.eventbus.EventBus;
|
||||
import com.google.common.util.concurrent.AbstractFuture;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
|
||||
/**
|
||||
* @author Adrian Cole
|
||||
*/
|
||||
@Test(groups = "unit", singleThreaded = true, testName = "BlockUntilInitScriptStatusIsZeroThenReturnOutputTest")
|
||||
public class BlockUntilInitScriptStatusIsZeroThenReturnOutputTest {
|
||||
|
||||
public void testLoopUntilTrueOrThrowCancellationExceptionReturnsWhenPredicateIsTrue() {
|
||||
AbstractFuture<ExecResponse> future = new AbstractFuture<ExecResponse>() {
|
||||
|
||||
@Override
|
||||
public boolean isCancelled() {
|
||||
return false;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
LoopUntilTrueOrThrowCancellationException pred = new LoopUntilTrueOrThrowCancellationException(Predicates
|
||||
.<String> alwaysTrue(), 1, 1, future);
|
||||
assertEquals(pred.apply("foo"), true);
|
||||
|
||||
}
|
||||
|
||||
public void testLoopUntilTrueOrThrowCancellationExceptionReturnsWhenPredicateIsTrueSecondTimeWhileNotCancelled() {
|
||||
AbstractFuture<ExecResponse> future = new AbstractFuture<ExecResponse>() {
|
||||
|
||||
@Override
|
||||
public boolean isCancelled() {
|
||||
return false;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
Predicate<String> predicate = new Predicate<String>() {
|
||||
AtomicBoolean bool = new AtomicBoolean();
|
||||
|
||||
@Override
|
||||
public boolean apply(String input) {
|
||||
return bool.getAndSet(true);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
LoopUntilTrueOrThrowCancellationException pred = new LoopUntilTrueOrThrowCancellationException(predicate, 1, 1,
|
||||
future);
|
||||
assertEquals(pred.apply("foo"), true);
|
||||
|
||||
}
|
||||
|
||||
// need to break the loop when cancelled.
|
||||
public void testLoopUntilTrueOrThrowCancellationExceptionSkipsAndReturnsFalseOnCancelled() {
|
||||
AbstractFuture<ExecResponse> future = new AbstractFuture<ExecResponse>() {
|
||||
|
||||
@Override
|
||||
public boolean isCancelled() {
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
LoopUntilTrueOrThrowCancellationException pred = new LoopUntilTrueOrThrowCancellationException(Predicates
|
||||
.<String> alwaysFalse(), 1, 1, future);
|
||||
assertEquals(pred.apply("foo"), false);
|
||||
|
||||
}
|
||||
|
||||
public void testExitStatusOfCommandGreaterThanZeroTrueWhen1() {
|
||||
|
||||
SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction")
|
||||
.createStrictMock();
|
||||
expect(commandRunner.runAction("status")).andReturn(new ExecResponse("", "", 1));
|
||||
replay(commandRunner);
|
||||
|
||||
Predicate<String> pred = new ExitStatusOfCommandGreaterThanZero(commandRunner);
|
||||
assertEquals(pred.apply("status"), true);
|
||||
|
||||
verify(commandRunner);
|
||||
|
||||
}
|
||||
|
||||
public void testExitStatusOfCommandGreaterThanZeroFalseWhen0() {
|
||||
|
||||
SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction")
|
||||
.createStrictMock();
|
||||
expect(commandRunner.runAction("status")).andReturn(new ExecResponse("", "", 0));
|
||||
replay(commandRunner);
|
||||
|
||||
Predicate<String> pred = new ExitStatusOfCommandGreaterThanZero(commandRunner);
|
||||
assertEquals(pred.apply("status"), false);
|
||||
|
||||
verify(commandRunner);
|
||||
|
||||
}
|
||||
|
||||
EventBus eventBus = new EventBus();
|
||||
|
||||
public void testExitStatusZeroReturnsExecResponse() throws InterruptedException, ExecutionException {
|
||||
ExecutorService userThreads = MoreExecutors.sameThreadExecutor();
|
||||
Predicate<String> notRunningAnymore = Predicates.alwaysTrue();
|
||||
SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction")
|
||||
.addMockedMethod("getStatement").addMockedMethod("getNode").addMockedMethod("toString")
|
||||
.createStrictMock();
|
||||
InitScript initScript = createMockBuilder(InitScript.class).addMockedMethod("getInstanceName").createStrictMock();
|
||||
|
||||
expect(commandRunner.runAction("stdout")).andReturn(new ExecResponse("stdout", "", 0));
|
||||
expect(commandRunner.runAction("stderr")).andReturn(new ExecResponse("stderr", "", 0));
|
||||
expect(commandRunner.runAction("exitstatus")).andReturn(new ExecResponse("444\n", "", 0));
|
||||
|
||||
toStringAndEventBusExpectations(commandRunner, initScript);
|
||||
|
||||
replay(commandRunner, initScript);
|
||||
|
||||
BlockUntilInitScriptStatusIsZeroThenReturnOutput future = new BlockUntilInitScriptStatusIsZeroThenReturnOutput(
|
||||
userThreads, eventBus, notRunningAnymore, commandRunner);
|
||||
|
||||
future.run();
|
||||
|
||||
assertEquals(future.get(), new ExecResponse("stdout", "stderr", 444));
|
||||
|
||||
verify(commandRunner, initScript);
|
||||
|
||||
}
|
||||
|
||||
public void testFirstExitStatusOneButSecondExitStatusZeroReturnsExecResponse() throws InterruptedException,
|
||||
ExecutionException {
|
||||
ExecutorService userThreads = MoreExecutors.sameThreadExecutor();
|
||||
Predicate<String> notRunningAnymore = Predicates.alwaysTrue();
|
||||
|
||||
SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction")
|
||||
.addMockedMethod("getStatement").addMockedMethod("getNode").addMockedMethod("toString")
|
||||
.createStrictMock();
|
||||
InitScript initScript = createMockBuilder(InitScript.class).addMockedMethod("getInstanceName").createStrictMock();
|
||||
|
||||
// exit status is 1 means we are still running!
|
||||
expect(commandRunner.runAction("stdout")).andReturn(new ExecResponse("", "", 0));
|
||||
expect(commandRunner.runAction("stderr")).andReturn(new ExecResponse("", "", 0));
|
||||
expect(commandRunner.runAction("exitstatus")).andReturn(new ExecResponse("", "", 1));
|
||||
|
||||
// second time around, it did stop
|
||||
expect(commandRunner.runAction("stdout")).andReturn(new ExecResponse("stdout", "", 0));
|
||||
expect(commandRunner.runAction("stderr")).andReturn(new ExecResponse("stderr", "", 0));
|
||||
expect(commandRunner.runAction("exitstatus")).andReturn(new ExecResponse("444\n", "", 0));
|
||||
|
||||
toStringAndEventBusExpectations(commandRunner, initScript);
|
||||
|
||||
replay(commandRunner, initScript);
|
||||
|
||||
BlockUntilInitScriptStatusIsZeroThenReturnOutput future = new BlockUntilInitScriptStatusIsZeroThenReturnOutput(
|
||||
userThreads, eventBus, notRunningAnymore, commandRunner);
|
||||
|
||||
future.run();
|
||||
|
||||
assertEquals(future.get(), new ExecResponse("stdout", "stderr", 444));
|
||||
|
||||
verify(commandRunner, initScript);
|
||||
|
||||
}
|
||||
|
||||
public void testCancelInterruptStopsCommand() throws InterruptedException, ExecutionException {
|
||||
ExecutorService userThreads = MoreExecutors.sameThreadExecutor();
|
||||
Predicate<String> notRunningAnymore = Predicates.alwaysTrue();
|
||||
SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod(
|
||||
"refreshAndRunAction").addMockedMethod("runAction").addMockedMethod("getStatement").addMockedMethod(
|
||||
"getNode").addMockedMethod("toString").createStrictMock();
|
||||
InitScript initScript = createMockBuilder(InitScript.class).addMockedMethod("getInstanceName").createStrictMock();
|
||||
|
||||
// log what we are stopping
|
||||
expect(commandRunner.getStatement()).andReturn(initScript);
|
||||
expect(initScript.getInstanceName()).andReturn("init-script");
|
||||
|
||||
// stop
|
||||
expect(commandRunner.refreshAndRunAction("stop")).andReturn(new ExecResponse("stdout", "", 0));
|
||||
|
||||
// create cancellation exception
|
||||
expect(commandRunner.getStatement()).andReturn(initScript);
|
||||
expect(initScript.getInstanceName()).andReturn("init-script");
|
||||
expect(commandRunner.getNode()).andReturn(
|
||||
new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).build()).atLeastOnce();
|
||||
|
||||
// StatementOnNodeFailure event
|
||||
expect(commandRunner.getStatement()).andReturn(initScript);
|
||||
expect(commandRunner.getNode()).andReturn(
|
||||
new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).build()).atLeastOnce();
|
||||
|
||||
replay(commandRunner, initScript);
|
||||
|
||||
BlockUntilInitScriptStatusIsZeroThenReturnOutput future = new BlockUntilInitScriptStatusIsZeroThenReturnOutput(
|
||||
userThreads, eventBus, notRunningAnymore, commandRunner);
|
||||
|
||||
future.cancel(true);
|
||||
|
||||
try {
|
||||
future.get();
|
||||
fail();
|
||||
} catch (CancellationException e) {
|
||||
|
||||
}
|
||||
|
||||
verify(commandRunner, initScript);
|
||||
|
||||
}
|
||||
|
||||
public void testCancelDontInterruptLeavesCommandRunningAndReturnsLastStatus() throws InterruptedException,
|
||||
ExecutionException {
|
||||
ExecutorService userThreads = MoreExecutors.sameThreadExecutor();
|
||||
Predicate<String> notRunningAnymore = Predicates.alwaysTrue();
|
||||
SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction")
|
||||
.addMockedMethod("getStatement").addMockedMethod("getNode").addMockedMethod("toString")
|
||||
.createStrictMock();
|
||||
InitScript initScript = createMockBuilder(InitScript.class).addMockedMethod("getInstanceName").createStrictMock();
|
||||
|
||||
expect(commandRunner.runAction("stdout")).andReturn(new ExecResponse("stillrunning", "", 0));
|
||||
expect(commandRunner.runAction("stderr")).andReturn(new ExecResponse("", "", 0));
|
||||
expect(commandRunner.runAction("exitstatus")).andReturn(new ExecResponse("", "", 1));
|
||||
|
||||
toStringAndEventBusExpectations(commandRunner, initScript);
|
||||
|
||||
replay(commandRunner, initScript);
|
||||
|
||||
BlockUntilInitScriptStatusIsZeroThenReturnOutput future = new BlockUntilInitScriptStatusIsZeroThenReturnOutput(
|
||||
userThreads, eventBus, notRunningAnymore, commandRunner);
|
||||
|
||||
future.cancel(false);
|
||||
|
||||
// note if this didn't cancel properly, the loop would never end!
|
||||
future.run();
|
||||
|
||||
try {
|
||||
future.get();
|
||||
fail();
|
||||
} catch (CancellationException e) {
|
||||
|
||||
}
|
||||
verify(commandRunner, initScript);
|
||||
|
||||
}
|
||||
|
||||
private void toStringAndEventBusExpectations(SudoAwareInitManager commandRunner, InitScript initScript) {
|
||||
toStringExpectations(commandRunner, initScript);
|
||||
expect(commandRunner.getStatement()).andReturn(initScript);
|
||||
expect(commandRunner.getNode()).andReturn(
|
||||
new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).build());
|
||||
}
|
||||
|
||||
private void toStringExpectations(SudoAwareInitManager commandRunner, InitScript initScript) {
|
||||
expect(commandRunner.getStatement()).andReturn(initScript);
|
||||
expect(initScript.getInstanceName()).andReturn("init-script");
|
||||
}
|
||||
}
|
||||
|
|
@ -25,10 +25,6 @@ import static org.easymock.EasyMock.verify;
|
|||
import static org.jclouds.scriptbuilder.domain.Statements.exec;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.easymock.IAnswer;
|
||||
import org.jclouds.Constants;
|
||||
import org.jclouds.compute.domain.ExecResponse;
|
||||
import org.jclouds.compute.domain.NodeMetadata;
|
||||
|
|
@ -40,7 +36,6 @@ import org.jclouds.compute.reference.ComputeServiceConstants.Timeouts;
|
|||
import org.jclouds.concurrent.MoreExecutors;
|
||||
import org.jclouds.concurrent.config.ExecutorServiceModule;
|
||||
import org.jclouds.domain.LoginCredentials;
|
||||
import org.jclouds.predicates.RetryablePredicateTest;
|
||||
import org.jclouds.scriptbuilder.InitScript;
|
||||
import org.jclouds.scriptbuilder.domain.OsFamily;
|
||||
import org.jclouds.scriptbuilder.domain.Statement;
|
||||
|
|
@ -48,7 +43,6 @@ import org.jclouds.ssh.SshClient;
|
|||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.common.base.Functions;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.eventbus.EventBus;
|
||||
import com.google.inject.AbstractModule;
|
||||
|
|
@ -107,37 +101,6 @@ public class RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest {
|
|||
}
|
||||
|
||||
public void testDefault() {
|
||||
runDefaults(null, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRepeatedlyChecksIfInitScriptCompleted() {
|
||||
final List<Long> callTimes = new ArrayList<Long>();
|
||||
final int succeedOnAttempt = 3;
|
||||
final Stopwatch stopwatch = new Stopwatch();
|
||||
stopwatch.start();
|
||||
|
||||
IAnswer<ExecResponse> answerForScriptStatus = new IAnswer<ExecResponse>() {
|
||||
private int count = 0;
|
||||
@Override
|
||||
public ExecResponse answer() throws Throwable {
|
||||
callTimes.add(stopwatch.elapsedMillis());
|
||||
String stdout = (++count < succeedOnAttempt) ? "someresult" : "";
|
||||
return new ExecResponse(stdout, "", 1);
|
||||
}
|
||||
};
|
||||
|
||||
runDefaults(answerForScriptStatus, succeedOnAttempt);
|
||||
|
||||
// Expect checking-status to be called repeatedly, until process had finished
|
||||
RetryablePredicateTest.assertCallTimes(callTimes, 0, 500, (int)(500+(500*1.5)));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param answerForScriptStatus Answer to use for `jclouds-script-0 status`, or null for default of succeed immediately
|
||||
* @param timesForScriptStatus Num times to expect call for `jclouds-script-0 status`; ignored if answer is null
|
||||
*/
|
||||
private void runDefaults(IAnswer<ExecResponse> answerForScriptStatus, int timesForScriptStatus) {
|
||||
Statement command = exec("doFoo");
|
||||
NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING)
|
||||
.credentials(LoginCredentials.builder().user("tester").password("testpassword!").build()).build();
|
||||
|
|
@ -163,11 +126,7 @@ public class RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest {
|
|||
expect(sshClient.exec("sudo /tmp/init-jclouds-script-0 start")).andReturn(new ExecResponse("", "", 0));
|
||||
|
||||
// signal the command completed
|
||||
if (answerForScriptStatus == null) {
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andReturn(new ExecResponse("", "", 1)).times(1);
|
||||
} else {
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andAnswer(answerForScriptStatus).times(timesForScriptStatus);
|
||||
}
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andReturn(new ExecResponse("", "", 1)).times(1);
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 stdout")).andReturn(new ExecResponse("out", "", 0));
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 stderr")).andReturn(new ExecResponse("err", "", 0));
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 exitstatus")).andReturn(new ExecResponse("0", "", 0));
|
||||
|
|
@ -240,6 +199,60 @@ public class RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest {
|
|||
verify(sshClient);
|
||||
}
|
||||
|
||||
/**
|
||||
* in a couple versions of ubuntu on aws-ec2, status returneds no pid (ex. empty stdout w/exit code 1) transiently. sadly, we need to doublecheck status before assuming it has failed.
|
||||
*
|
||||
*/
|
||||
public void testDoublecheckStatusInCaseTransientlyWrong() {
|
||||
Statement command = exec("doFoo");
|
||||
NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).credentials(
|
||||
new LoginCredentials("tester", "testpassword!", null, true)).build();
|
||||
|
||||
SshClient sshClient = createMock(SshClient.class);
|
||||
|
||||
InitScript init = InitScript.builder().name("jclouds-script-0").home("/tmp/jclouds-script-0").run(command)
|
||||
.build();
|
||||
|
||||
sshClient.connect();
|
||||
sshClient.put("/tmp/init-jclouds-script-0", init.render(OsFamily.UNIX));
|
||||
expect(sshClient.getUsername()).andReturn("tester").atLeastOnce();
|
||||
expect(sshClient.getHostAddress()).andReturn("somewhere.example.com").atLeastOnce();
|
||||
|
||||
// setup script as default user
|
||||
expect(sshClient.exec("chmod 755 /tmp/init-jclouds-script-0")).andReturn(new ExecResponse("", "", 0));
|
||||
expect(sshClient.exec("ln -fs /tmp/init-jclouds-script-0 jclouds-script-0")).andReturn(
|
||||
new ExecResponse("", "", 0));
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 init")).andReturn(new ExecResponse("", "", 0));
|
||||
|
||||
// since there's an adminPassword we must pass this in
|
||||
expect(sshClient.exec("echo 'testpassword!'|sudo -S /tmp/init-jclouds-script-0 start")).andReturn(new ExecResponse("", "", 0));
|
||||
|
||||
// signal the command completed
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andReturn(new ExecResponse("8001", "", 0));
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andReturn(new ExecResponse("", "", 1));
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 stdout")).andReturn(new ExecResponse("out", "", 0));
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 stderr")).andReturn(new ExecResponse("err", "", 0));
|
||||
expect(sshClient.exec("/tmp/init-jclouds-script-0 exitstatus")).andReturn(new ExecResponse("0", "", 0));
|
||||
|
||||
sshClient.disconnect();
|
||||
replay(sshClient);
|
||||
|
||||
RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilComplete testMe = new RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilComplete(
|
||||
statusFactory, timeouts, Functions.forMap(ImmutableMap.of(node, sshClient)),
|
||||
eventBus, InitScriptConfigurationForTasks.create().appendIncrementingNumberToAnonymousTaskNames(), node, command,
|
||||
new RunScriptOptions());
|
||||
|
||||
assertEquals(testMe.getInitFile(), "/tmp/init-jclouds-script-0");
|
||||
assertEquals(testMe.getNode(), node);
|
||||
assertEquals(testMe.getStatement(), init);
|
||||
|
||||
testMe.init();
|
||||
|
||||
assertEquals(testMe.call(), new ExecResponse("out", "err", 0));
|
||||
|
||||
verify(sshClient);
|
||||
}
|
||||
|
||||
public void testNotRoot() {
|
||||
Statement command = exec("doFoo");
|
||||
NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).credentials(
|
||||
|
|
|
|||
Loading…
Reference in New Issue