From 6eec5d5c2409ace7592cb7a9300bb4d62eed0578 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Fri, 8 Jun 2012 21:44:43 -0700 Subject: [PATCH] Issue 966: retry when exitstatus shows process is still running --- ...nitScriptStatusIsZeroThenReturnOutput.java | 148 +++++---- ...criptStatusIsZeroThenReturnOutputTest.java | 294 ++++++++++++++++++ ...riptUsingSshAndBlockUntilCompleteTest.java | 97 +++--- 3 files changed, 428 insertions(+), 111 deletions(-) create mode 100644 compute/src/test/java/org/jclouds/compute/callables/BlockUntilInitScriptStatusIsZeroThenReturnOutputTest.java diff --git a/compute/src/main/java/org/jclouds/compute/callables/BlockUntilInitScriptStatusIsZeroThenReturnOutput.java b/compute/src/main/java/org/jclouds/compute/callables/BlockUntilInitScriptStatusIsZeroThenReturnOutput.java index a6f07fbc96..29379db117 100644 --- a/compute/src/main/java/org/jclouds/compute/callables/BlockUntilInitScriptStatusIsZeroThenReturnOutput.java +++ b/compute/src/main/java/org/jclouds/compute/callables/BlockUntilInitScriptStatusIsZeroThenReturnOutput.java @@ -33,16 +33,15 @@ import org.jclouds.Constants; import org.jclouds.compute.domain.ExecResponse; import org.jclouds.compute.events.StatementOnNodeCompletion; import org.jclouds.compute.events.StatementOnNodeFailure; -import org.jclouds.compute.predicates.ScriptStatusReturnsZero; import org.jclouds.compute.reference.ComputeServiceConstants; import org.jclouds.logging.Logger; import org.jclouds.predicates.RetryablePredicate; import org.jclouds.scriptbuilder.InitScript; -import org.jclouds.ssh.SshClient; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; import com.google.common.base.Predicate; -import com.google.common.base.Throwables; +import com.google.common.base.Predicates; import com.google.common.eventbus.EventBus; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.AbstractFuture; @@ -51,12 +50,11 @@ import com.google.inject.assistedinject.Assisted; import com.google.inject.name.Named; /** - * A future that works in tandem with a task that was invoked by - * {@link InitScript} + * A future that works in tandem with a task that was invoked by {@link InitScript} * * @author Adrian Cole */ -public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFuture { +public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFuture implements Runnable { public static interface Factory { BlockUntilInitScriptStatusIsZeroThenReturnOutput create(SudoAwareInitManager commandRunner); @@ -74,86 +72,98 @@ public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFu return commandRunner; } - private final RetryablePredicate notRunningAnymore; + private Predicate notRunningAnymore; @Inject public BlockUntilInitScriptStatusIsZeroThenReturnOutput( - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userThreads, EventBus eventBus, - ComputeServiceConstants.InitStatusProperties properties, final ScriptStatusReturnsZero stateRunning, - @Assisted final SudoAwareInitManager commandRunner) { - - long retryMaxWait = TimeUnit.DAYS.toMillis(365); // arbitrarily high - // value, but - // Long.MAX_VALUE doesn't - // work! - long retryInitialPeriod = properties.initStatusInitialPeriod; - long retryMaxPeriod = properties.initStatusMaxPeriod; + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userThreads, EventBus eventBus, + ComputeServiceConstants.InitStatusProperties properties, @Assisted SudoAwareInitManager commandRunner) { + this(userThreads, eventBus, Predicates. alwaysTrue(), commandRunner); + // this is mutable only until we can determine how to decouple "this" from here + notRunningAnymore = new LoopUntilTrueOrThrowCancellationException(new ExitStatusOfCommandGreaterThanZero( + commandRunner), properties.initStatusMaxPeriod, properties.initStatusInitialPeriod, this); + } + @VisibleForTesting + public BlockUntilInitScriptStatusIsZeroThenReturnOutput(ExecutorService userThreads, EventBus eventBus, + Predicate notRunningAnymore, SudoAwareInitManager commandRunner) { this.commandRunner = checkNotNull(commandRunner, "commandRunner"); this.userThreads = checkNotNull(userThreads, "userThreads"); this.eventBus = checkNotNull(eventBus, "eventBus"); - this.notRunningAnymore = new RetryablePredicate(new Predicate() { - - @Override - public boolean apply(String arg0) { - return commandRunner.runAction(arg0).getOutput().trim().equals(""); - } - }, retryMaxWait, retryInitialPeriod, retryMaxPeriod, TimeUnit.MILLISECONDS) { - /** - * make sure we stop the retry loop if someone cancelled the future, - * this keeps threads from being consumed on dead tasks - */ - @Override - protected boolean atOrAfter(Date end) { - if (isCancelled()) - Throwables.propagate(new CancellationException("cancelled")); - return super.atOrAfter(end); - } - }; + this.notRunningAnymore = checkNotNull(notRunningAnymore, "notRunningAnymore"); } - /** - * in case login credentials or user changes at runtime. - */ - public void setSshClient(SshClient client) { + @VisibleForTesting + static class ExitStatusOfCommandGreaterThanZero implements Predicate { + private final SudoAwareInitManager commandRunner; + + ExitStatusOfCommandGreaterThanZero(SudoAwareInitManager commandRunner) { + this.commandRunner = commandRunner; + } + + @Override + public boolean apply(String input) { + return commandRunner.runAction(input).getExitStatus() > 0; + } } + @VisibleForTesting + static class LoopUntilTrueOrThrowCancellationException extends RetryablePredicate { + + private final AbstractFuture futureWhichMightBeCancelled; + + public LoopUntilTrueOrThrowCancellationException(Predicate predicate, long period, long maxPeriod, + AbstractFuture futureWhichMightBeCancelled) { + // arbitrarily high value, but Long.MAX_VALUE doesn't work! + super(predicate, TimeUnit.DAYS.toMillis(365), period, maxPeriod, TimeUnit.MILLISECONDS); + this.futureWhichMightBeCancelled = futureWhichMightBeCancelled; + } + + /** + * make sure we stop the retry loop if someone cancelled the future, this keeps threads from + * being consumed on dead tasks + */ + @Override + protected boolean atOrAfter(Date end) { + if (futureWhichMightBeCancelled.isCancelled()) + throw new CancellationException(futureWhichMightBeCancelled + " is cancelled"); + return super.atOrAfter(end); + } + } + /** - * Submits a thread that will either set the result of the future or the - * exception that took place + * Submits a thread that will either set the result of the future or the exception that took + * place */ public BlockUntilInitScriptStatusIsZeroThenReturnOutput init() { - userThreads.submit(new Runnable() { - @Override - public void run() { - try { - notRunningAnymore.apply("status"); - String stdout = commandRunner.runAction("stdout").getOutput(); - String stderr = commandRunner.runAction("stderr").getOutput(); - Integer exitStatus = Ints.tryParse(commandRunner.runAction("exitstatus").getOutput().trim()); - ExecResponse exec = new ExecResponse(stdout, stderr, exitStatus == null ? -1 : exitStatus); - if (exitStatus == null) { - Integer pid = Ints.tryParse(commandRunner.runAction("status").getOutput().trim()); - throw new ScriptStillRunningException(String.format("%s still running: pid(%s), last status: %s", - BlockUntilInitScriptStatusIsZeroThenReturnOutput.this, pid, exec), - BlockUntilInitScriptStatusIsZeroThenReturnOutput.this); - } - logger.debug("<< complete(%s) status(%s)", commandRunner.getStatement().getInstanceName(), exitStatus); - set(exec); - } catch (CancellationException e) { - } catch (Exception e) { - setException(e); - } - } - }); + userThreads.submit(this); return this; } + @Override + public void run() { + try { + ExecResponse exec = null; + do { + notRunningAnymore.apply("status"); + String stdout = commandRunner.runAction("stdout").getOutput(); + String stderr = commandRunner.runAction("stderr").getOutput(); + Integer exitStatus = Ints.tryParse(commandRunner.runAction("exitstatus").getOutput().trim()); + exec = new ExecResponse(stdout, stderr, exitStatus == null ? -1 : exitStatus); + } while (!isCancelled() && exec.getExitStatus() == -1); + logger.debug("<< complete(%s) status(%s)", commandRunner.getStatement().getInstanceName(), exec + .getExitStatus()); + set(exec); + } catch (Exception e) { + setException(e); + } + } + @Override protected boolean set(ExecResponse value) { eventBus.post(new StatementOnNodeCompletion(getCommandRunner().getStatement(), getCommandRunner().getNode(), - value)); + value)); return super.set(value); } @@ -162,8 +172,8 @@ public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFu logger.debug("<< cancelled(%s)", commandRunner.getStatement().getInstanceName()); ExecResponse returnVal = commandRunner.refreshAndRunAction("stop"); CancellationException e = new CancellationException(String.format( - "cancelled %s on node: %s; stop command had exit status: %s", getCommandRunner().getStatement() - .getInstanceName(), getCommandRunner().getNode().getId(), returnVal)); + "cancelled %s on node: %s; stop command had exit status: %s", getCommandRunner().getStatement() + .getInstanceName(), getCommandRunner().getNode().getId(), returnVal)); eventBus.post(new StatementOnNodeFailure(getCommandRunner().getStatement(), getCommandRunner().getNode(), e)); super.interruptTask(); } @@ -185,13 +195,13 @@ public class BlockUntilInitScriptStatusIsZeroThenReturnOutput extends AbstractFu if (!o.getClass().equals(getClass())) return false; BlockUntilInitScriptStatusIsZeroThenReturnOutput that = BlockUntilInitScriptStatusIsZeroThenReturnOutput.class - .cast(o); + .cast(o); return Objects.equal(this.commandRunner, that.commandRunner); } @Override public ExecResponse get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException, - ExecutionException { + ExecutionException { try { return super.get(timeout, unit); } catch (TimeoutException e) { diff --git a/compute/src/test/java/org/jclouds/compute/callables/BlockUntilInitScriptStatusIsZeroThenReturnOutputTest.java b/compute/src/test/java/org/jclouds/compute/callables/BlockUntilInitScriptStatusIsZeroThenReturnOutputTest.java new file mode 100644 index 0000000000..735a3cf246 --- /dev/null +++ b/compute/src/test/java/org/jclouds/compute/callables/BlockUntilInitScriptStatusIsZeroThenReturnOutputTest.java @@ -0,0 +1,294 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.compute.callables; + +import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.jclouds.compute.callables.BlockUntilInitScriptStatusIsZeroThenReturnOutput.ExitStatusOfCommandGreaterThanZero; +import org.jclouds.compute.callables.BlockUntilInitScriptStatusIsZeroThenReturnOutput.LoopUntilTrueOrThrowCancellationException; +import org.jclouds.compute.domain.ExecResponse; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.domain.NodeMetadataBuilder; +import org.jclouds.compute.domain.NodeState; +import org.jclouds.scriptbuilder.InitScript; +import org.testng.annotations.Test; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.eventbus.EventBus; +import com.google.common.util.concurrent.AbstractFuture; +import com.google.common.util.concurrent.MoreExecutors; + +/** + * @author Adrian Cole + */ +@Test(groups = "unit", singleThreaded = true, testName = "BlockUntilInitScriptStatusIsZeroThenReturnOutputTest") +public class BlockUntilInitScriptStatusIsZeroThenReturnOutputTest { + + public void testLoopUntilTrueOrThrowCancellationExceptionReturnsWhenPredicateIsTrue() { + AbstractFuture future = new AbstractFuture() { + + @Override + public boolean isCancelled() { + return false; + } + + }; + + LoopUntilTrueOrThrowCancellationException pred = new LoopUntilTrueOrThrowCancellationException(Predicates + . alwaysTrue(), 1, 1, future); + assertEquals(pred.apply("foo"), true); + + } + + public void testLoopUntilTrueOrThrowCancellationExceptionReturnsWhenPredicateIsTrueSecondTimeWhileNotCancelled() { + AbstractFuture future = new AbstractFuture() { + + @Override + public boolean isCancelled() { + return false; + } + + }; + + Predicate predicate = new Predicate() { + AtomicBoolean bool = new AtomicBoolean(); + + @Override + public boolean apply(String input) { + return bool.getAndSet(true); + } + + }; + + LoopUntilTrueOrThrowCancellationException pred = new LoopUntilTrueOrThrowCancellationException(predicate, 1, 1, + future); + assertEquals(pred.apply("foo"), true); + + } + + // need to break the loop when cancelled. + public void testLoopUntilTrueOrThrowCancellationExceptionSkipsAndReturnsFalseOnCancelled() { + AbstractFuture future = new AbstractFuture() { + + @Override + public boolean isCancelled() { + return true; + } + + }; + LoopUntilTrueOrThrowCancellationException pred = new LoopUntilTrueOrThrowCancellationException(Predicates + . alwaysFalse(), 1, 1, future); + assertEquals(pred.apply("foo"), false); + + } + + public void testExitStatusOfCommandGreaterThanZeroTrueWhen1() { + + SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction") + .createStrictMock(); + expect(commandRunner.runAction("status")).andReturn(new ExecResponse("", "", 1)); + replay(commandRunner); + + Predicate pred = new ExitStatusOfCommandGreaterThanZero(commandRunner); + assertEquals(pred.apply("status"), true); + + verify(commandRunner); + + } + + public void testExitStatusOfCommandGreaterThanZeroFalseWhen0() { + + SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction") + .createStrictMock(); + expect(commandRunner.runAction("status")).andReturn(new ExecResponse("", "", 0)); + replay(commandRunner); + + Predicate pred = new ExitStatusOfCommandGreaterThanZero(commandRunner); + assertEquals(pred.apply("status"), false); + + verify(commandRunner); + + } + + EventBus eventBus = new EventBus(); + + public void testExitStatusZeroReturnsExecResponse() throws InterruptedException, ExecutionException { + ExecutorService userThreads = MoreExecutors.sameThreadExecutor(); + Predicate notRunningAnymore = Predicates.alwaysTrue(); + SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction") + .addMockedMethod("getStatement").addMockedMethod("getNode").addMockedMethod("toString") + .createStrictMock(); + InitScript initScript = createMockBuilder(InitScript.class).addMockedMethod("getInstanceName").createStrictMock(); + + expect(commandRunner.runAction("stdout")).andReturn(new ExecResponse("stdout", "", 0)); + expect(commandRunner.runAction("stderr")).andReturn(new ExecResponse("stderr", "", 0)); + expect(commandRunner.runAction("exitstatus")).andReturn(new ExecResponse("444\n", "", 0)); + + toStringAndEventBusExpectations(commandRunner, initScript); + + replay(commandRunner, initScript); + + BlockUntilInitScriptStatusIsZeroThenReturnOutput future = new BlockUntilInitScriptStatusIsZeroThenReturnOutput( + userThreads, eventBus, notRunningAnymore, commandRunner); + + future.run(); + + assertEquals(future.get(), new ExecResponse("stdout", "stderr", 444)); + + verify(commandRunner, initScript); + + } + + public void testFirstExitStatusOneButSecondExitStatusZeroReturnsExecResponse() throws InterruptedException, + ExecutionException { + ExecutorService userThreads = MoreExecutors.sameThreadExecutor(); + Predicate notRunningAnymore = Predicates.alwaysTrue(); + + SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction") + .addMockedMethod("getStatement").addMockedMethod("getNode").addMockedMethod("toString") + .createStrictMock(); + InitScript initScript = createMockBuilder(InitScript.class).addMockedMethod("getInstanceName").createStrictMock(); + + // exit status is 1 means we are still running! + expect(commandRunner.runAction("stdout")).andReturn(new ExecResponse("", "", 0)); + expect(commandRunner.runAction("stderr")).andReturn(new ExecResponse("", "", 0)); + expect(commandRunner.runAction("exitstatus")).andReturn(new ExecResponse("", "", 1)); + + // second time around, it did stop + expect(commandRunner.runAction("stdout")).andReturn(new ExecResponse("stdout", "", 0)); + expect(commandRunner.runAction("stderr")).andReturn(new ExecResponse("stderr", "", 0)); + expect(commandRunner.runAction("exitstatus")).andReturn(new ExecResponse("444\n", "", 0)); + + toStringAndEventBusExpectations(commandRunner, initScript); + + replay(commandRunner, initScript); + + BlockUntilInitScriptStatusIsZeroThenReturnOutput future = new BlockUntilInitScriptStatusIsZeroThenReturnOutput( + userThreads, eventBus, notRunningAnymore, commandRunner); + + future.run(); + + assertEquals(future.get(), new ExecResponse("stdout", "stderr", 444)); + + verify(commandRunner, initScript); + + } + + public void testCancelInterruptStopsCommand() throws InterruptedException, ExecutionException { + ExecutorService userThreads = MoreExecutors.sameThreadExecutor(); + Predicate notRunningAnymore = Predicates.alwaysTrue(); + SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod( + "refreshAndRunAction").addMockedMethod("runAction").addMockedMethod("getStatement").addMockedMethod( + "getNode").addMockedMethod("toString").createStrictMock(); + InitScript initScript = createMockBuilder(InitScript.class).addMockedMethod("getInstanceName").createStrictMock(); + + // log what we are stopping + expect(commandRunner.getStatement()).andReturn(initScript); + expect(initScript.getInstanceName()).andReturn("init-script"); + + // stop + expect(commandRunner.refreshAndRunAction("stop")).andReturn(new ExecResponse("stdout", "", 0)); + + // create cancellation exception + expect(commandRunner.getStatement()).andReturn(initScript); + expect(initScript.getInstanceName()).andReturn("init-script"); + expect(commandRunner.getNode()).andReturn( + new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).build()).atLeastOnce(); + + // StatementOnNodeFailure event + expect(commandRunner.getStatement()).andReturn(initScript); + expect(commandRunner.getNode()).andReturn( + new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).build()).atLeastOnce(); + + replay(commandRunner, initScript); + + BlockUntilInitScriptStatusIsZeroThenReturnOutput future = new BlockUntilInitScriptStatusIsZeroThenReturnOutput( + userThreads, eventBus, notRunningAnymore, commandRunner); + + future.cancel(true); + + try { + future.get(); + fail(); + } catch (CancellationException e) { + + } + + verify(commandRunner, initScript); + + } + + public void testCancelDontInterruptLeavesCommandRunningAndReturnsLastStatus() throws InterruptedException, + ExecutionException { + ExecutorService userThreads = MoreExecutors.sameThreadExecutor(); + Predicate notRunningAnymore = Predicates.alwaysTrue(); + SudoAwareInitManager commandRunner = createMockBuilder(SudoAwareInitManager.class).addMockedMethod("runAction") + .addMockedMethod("getStatement").addMockedMethod("getNode").addMockedMethod("toString") + .createStrictMock(); + InitScript initScript = createMockBuilder(InitScript.class).addMockedMethod("getInstanceName").createStrictMock(); + + expect(commandRunner.runAction("stdout")).andReturn(new ExecResponse("stillrunning", "", 0)); + expect(commandRunner.runAction("stderr")).andReturn(new ExecResponse("", "", 0)); + expect(commandRunner.runAction("exitstatus")).andReturn(new ExecResponse("", "", 1)); + + toStringAndEventBusExpectations(commandRunner, initScript); + + replay(commandRunner, initScript); + + BlockUntilInitScriptStatusIsZeroThenReturnOutput future = new BlockUntilInitScriptStatusIsZeroThenReturnOutput( + userThreads, eventBus, notRunningAnymore, commandRunner); + + future.cancel(false); + + // note if this didn't cancel properly, the loop would never end! + future.run(); + + try { + future.get(); + fail(); + } catch (CancellationException e) { + + } + verify(commandRunner, initScript); + + } + + private void toStringAndEventBusExpectations(SudoAwareInitManager commandRunner, InitScript initScript) { + toStringExpectations(commandRunner, initScript); + expect(commandRunner.getStatement()).andReturn(initScript); + expect(commandRunner.getNode()).andReturn( + new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).build()); + } + + private void toStringExpectations(SudoAwareInitManager commandRunner, InitScript initScript) { + expect(commandRunner.getStatement()).andReturn(initScript); + expect(initScript.getInstanceName()).andReturn("init-script"); + } +} diff --git a/compute/src/test/java/org/jclouds/compute/callables/RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest.java b/compute/src/test/java/org/jclouds/compute/callables/RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest.java index 4e44027680..6d7102bef3 100644 --- a/compute/src/test/java/org/jclouds/compute/callables/RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest.java +++ b/compute/src/test/java/org/jclouds/compute/callables/RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest.java @@ -25,10 +25,6 @@ import static org.easymock.EasyMock.verify; import static org.jclouds.scriptbuilder.domain.Statements.exec; import static org.testng.Assert.assertEquals; -import java.util.ArrayList; -import java.util.List; - -import org.easymock.IAnswer; import org.jclouds.Constants; import org.jclouds.compute.domain.ExecResponse; import org.jclouds.compute.domain.NodeMetadata; @@ -40,7 +36,6 @@ import org.jclouds.compute.reference.ComputeServiceConstants.Timeouts; import org.jclouds.concurrent.MoreExecutors; import org.jclouds.concurrent.config.ExecutorServiceModule; import org.jclouds.domain.LoginCredentials; -import org.jclouds.predicates.RetryablePredicateTest; import org.jclouds.scriptbuilder.InitScript; import org.jclouds.scriptbuilder.domain.OsFamily; import org.jclouds.scriptbuilder.domain.Statement; @@ -48,7 +43,6 @@ import org.jclouds.ssh.SshClient; import org.testng.annotations.Test; import com.google.common.base.Functions; -import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; import com.google.common.eventbus.EventBus; import com.google.inject.AbstractModule; @@ -107,37 +101,6 @@ public class RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest { } public void testDefault() { - runDefaults(null, 1); - } - - @Test - public void testRepeatedlyChecksIfInitScriptCompleted() { - final List callTimes = new ArrayList(); - final int succeedOnAttempt = 3; - final Stopwatch stopwatch = new Stopwatch(); - stopwatch.start(); - - IAnswer answerForScriptStatus = new IAnswer() { - private int count = 0; - @Override - public ExecResponse answer() throws Throwable { - callTimes.add(stopwatch.elapsedMillis()); - String stdout = (++count < succeedOnAttempt) ? "someresult" : ""; - return new ExecResponse(stdout, "", 1); - } - }; - - runDefaults(answerForScriptStatus, succeedOnAttempt); - - // Expect checking-status to be called repeatedly, until process had finished - RetryablePredicateTest.assertCallTimes(callTimes, 0, 500, (int)(500+(500*1.5))); - } - - /** - * @param answerForScriptStatus Answer to use for `jclouds-script-0 status`, or null for default of succeed immediately - * @param timesForScriptStatus Num times to expect call for `jclouds-script-0 status`; ignored if answer is null - */ - private void runDefaults(IAnswer answerForScriptStatus, int timesForScriptStatus) { Statement command = exec("doFoo"); NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING) .credentials(LoginCredentials.builder().user("tester").password("testpassword!").build()).build(); @@ -163,11 +126,7 @@ public class RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest { expect(sshClient.exec("sudo /tmp/init-jclouds-script-0 start")).andReturn(new ExecResponse("", "", 0)); // signal the command completed - if (answerForScriptStatus == null) { - expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andReturn(new ExecResponse("", "", 1)).times(1); - } else { - expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andAnswer(answerForScriptStatus).times(timesForScriptStatus); - } + expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andReturn(new ExecResponse("", "", 1)).times(1); expect(sshClient.exec("/tmp/init-jclouds-script-0 stdout")).andReturn(new ExecResponse("out", "", 0)); expect(sshClient.exec("/tmp/init-jclouds-script-0 stderr")).andReturn(new ExecResponse("err", "", 0)); expect(sshClient.exec("/tmp/init-jclouds-script-0 exitstatus")).andReturn(new ExecResponse("0", "", 0)); @@ -240,6 +199,60 @@ public class RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilCompleteTest { verify(sshClient); } + /** + * in a couple versions of ubuntu on aws-ec2, status returneds no pid (ex. empty stdout w/exit code 1) transiently. sadly, we need to doublecheck status before assuming it has failed. + * + */ + public void testDoublecheckStatusInCaseTransientlyWrong() { + Statement command = exec("doFoo"); + NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).credentials( + new LoginCredentials("tester", "testpassword!", null, true)).build(); + + SshClient sshClient = createMock(SshClient.class); + + InitScript init = InitScript.builder().name("jclouds-script-0").home("/tmp/jclouds-script-0").run(command) + .build(); + + sshClient.connect(); + sshClient.put("/tmp/init-jclouds-script-0", init.render(OsFamily.UNIX)); + expect(sshClient.getUsername()).andReturn("tester").atLeastOnce(); + expect(sshClient.getHostAddress()).andReturn("somewhere.example.com").atLeastOnce(); + + // setup script as default user + expect(sshClient.exec("chmod 755 /tmp/init-jclouds-script-0")).andReturn(new ExecResponse("", "", 0)); + expect(sshClient.exec("ln -fs /tmp/init-jclouds-script-0 jclouds-script-0")).andReturn( + new ExecResponse("", "", 0)); + expect(sshClient.exec("/tmp/init-jclouds-script-0 init")).andReturn(new ExecResponse("", "", 0)); + + // since there's an adminPassword we must pass this in + expect(sshClient.exec("echo 'testpassword!'|sudo -S /tmp/init-jclouds-script-0 start")).andReturn(new ExecResponse("", "", 0)); + + // signal the command completed + expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andReturn(new ExecResponse("8001", "", 0)); + expect(sshClient.exec("/tmp/init-jclouds-script-0 status")).andReturn(new ExecResponse("", "", 1)); + expect(sshClient.exec("/tmp/init-jclouds-script-0 stdout")).andReturn(new ExecResponse("out", "", 0)); + expect(sshClient.exec("/tmp/init-jclouds-script-0 stderr")).andReturn(new ExecResponse("err", "", 0)); + expect(sshClient.exec("/tmp/init-jclouds-script-0 exitstatus")).andReturn(new ExecResponse("0", "", 0)); + + sshClient.disconnect(); + replay(sshClient); + + RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilComplete testMe = new RunScriptOnNodeAsInitScriptUsingSshAndBlockUntilComplete( + statusFactory, timeouts, Functions.forMap(ImmutableMap.of(node, sshClient)), + eventBus, InitScriptConfigurationForTasks.create().appendIncrementingNumberToAnonymousTaskNames(), node, command, + new RunScriptOptions()); + + assertEquals(testMe.getInitFile(), "/tmp/init-jclouds-script-0"); + assertEquals(testMe.getNode(), node); + assertEquals(testMe.getStatement(), init); + + testMe.init(); + + assertEquals(testMe.call(), new ExecResponse("out", "err", 0)); + + verify(sshClient); + } + public void testNotRoot() { Statement command = exec("doFoo"); NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.RUNNING).credentials(