Issue 244: made error handling more resilient to ssh connection problems by increasing scope of error retries and larger backoff delay

compute/src/main/java/org/jclouds/compute/ComputeService.java

@@ -18,11 +18,9 @@
  */
 package org.jclouds.compute;
 
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.google.common.base.Predicate;
 import org.jclouds.compute.domain.ComputeMetadata;
 import org.jclouds.compute.domain.Image;
 import org.jclouds.compute.domain.NodeMetadata;
@@ -35,6 +33,7 @@ import org.jclouds.compute.options.RunScriptOptions;
 import org.jclouds.domain.Location;
 import org.jclouds.ssh.ExecResponse;
 
+import com.google.common.base.Predicate;
 import com.google.inject.ImplementedBy;
 
 /**

compute/src/main/java/org/jclouds/compute/internal/BaseComputeService.java

@@ -25,6 +25,7 @@ package org.jclouds.compute.internal;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static org.jclouds.compute.predicates.NodePredicates.withTag;
 import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
 import static org.jclouds.concurrent.ConcurrentUtils.makeListenable;
 
@@ -41,8 +42,6 @@ import javax.inject.Named;
 import javax.inject.Provider;
 import javax.inject.Singleton;
 
-import com.google.common.base.Predicates;
-import com.google.common.collect.Lists;
 import org.jclouds.Constants;
 import org.jclouds.compute.ComputeService;
 import org.jclouds.compute.ComputeServiceContext;
@@ -52,7 +51,6 @@ import org.jclouds.compute.domain.ComputeMetadata;
 import org.jclouds.compute.domain.ComputeType;
 import org.jclouds.compute.domain.Image;
 import org.jclouds.compute.domain.NodeMetadata;
-import org.jclouds.compute.domain.NodeState;
 import org.jclouds.compute.domain.Size;
 import org.jclouds.compute.domain.Template;
 import org.jclouds.compute.domain.TemplateBuilder;
@@ -74,11 +72,12 @@ import org.jclouds.ssh.SshClient;
 
 import com.google.common.base.Function;
 import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.ListenableFuture;
-import static org.jclouds.compute.predicates.NodePredicates.withTag;
 
 /**
  * 
@@ -214,8 +213,7 @@ public class BaseComputeService implements ComputeService {
                .execute(GetNodesOptions.NONE), computeMetadataToNodeMetadata), withTag(tag)));
    }
 
-   class ComputeMetadataToNodeMetadata
+   class ComputeMetadataToNodeMetadata implements Function<ComputeMetadata, NodeMetadata> {
-           implements Function<ComputeMetadata, NodeMetadata> {
 
       @Override
       public NodeMetadata apply(ComputeMetadata from) {
@@ -224,7 +222,6 @@ public class BaseComputeService implements ComputeService {
       }
    }
 
-
    @Override
    public Set<? extends NodeMetadata> listNodesWithTag(String tag) {
       logger.debug(">> listing nodes by tag(%s)", tag);
@@ -291,11 +288,12 @@ public class BaseComputeService implements ComputeService {
 
    /**
     * @throws RunScriptOnNodesException
-    * @see #runScriptOnNodesMatching(Predicate, byte[], org.jclouds.compute.options.RunScriptOptions)
+    * @see #runScriptOnNodesMatching(Predicate, byte[],
+    *      org.jclouds.compute.options.RunScriptOptions)
     * @see org.jclouds.compute.predicates.NodePredicates#activeWithTag(String)
     */
-   public Map<NodeMetadata, ExecResponse> runScriptOnNodesMatching(Predicate<NodeMetadata> filter, byte[] runScript)
+   public Map<NodeMetadata, ExecResponse> runScriptOnNodesMatching(Predicate<NodeMetadata> filter,
-           throws RunScriptOnNodesException {
+            byte[] runScript) throws RunScriptOnNodesException {
       return runScriptOnNodesMatching(filter, runScript, RunScriptOptions.NONE);
    }
 
@@ -303,15 +301,15 @@ public class BaseComputeService implements ComputeService {
     * Run the script on all nodes with the specific tag.
     * 
     * @param filter
-    *           Predicate-based filter to define on which nodes the script is to be
+    *           Predicate-based filter to define on which nodes the script is to be executed
-    *           executed
     * @param runScript
     *           script to run in byte format. If the script is a string, use
     *           {@link String#getBytes()} to retrieve the bytes
     * @param options
     *           nullable options to how to run the script, whether to override credentials
     * @return map with node identifiers and corresponding responses
-    * @throws RunScriptOnNodesException if anything goes wrong during script execution
+    * @throws RunScriptOnNodesException
+    *            if anything goes wrong during script execution
     * 
     * @see org.jclouds.compute.predicates.NodePredicates#activeWithTag(String)
     */
@@ -365,15 +363,14 @@ public class BaseComputeService implements ComputeService {
 
    }
 
-   private Iterable<? extends NodeMetadata> verifyParametersAndGetNodes(Predicate<NodeMetadata> filter,
+   private Iterable<? extends NodeMetadata> verifyParametersAndGetNodes(
-                                                                        byte[] runScript, final RunScriptOptions options) {
+            Predicate<NodeMetadata> filter, byte[] runScript, final RunScriptOptions options) {
       checkNotNull(filter, "Filter must be provided");
       checkNotNull(runScript,
                "The script (represented by bytes array - use \"script\".getBytes() must be provided");
       checkNotNull(options, "options");
-      Iterable<? extends NodeMetadata> nodes = Iterables.filter(
+      Iterable<? extends NodeMetadata> nodes = Iterables.filter(Iterables.transform(listNodes(),
-                           Iterables.transform(listNodes(), computeMetadataToNodeMetadata),
+               computeMetadataToNodeMetadata), filter);
-                           filter);
 
       return Iterables.transform(nodes, new Function<NodeMetadata, NodeMetadata>() {
 

compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java

@@ -43,7 +43,6 @@ import javax.annotation.Nullable;
 import javax.annotation.Resource;
 import javax.inject.Named;
 
-import com.google.common.base.Function;
 import org.jclouds.Constants;
 import org.jclouds.compute.domain.ComputeMetadata;
 import org.jclouds.compute.domain.NodeMetadata;

compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java

@@ -78,8 +78,6 @@ import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Module;
 
-import javax.annotation.Nullable;
-
 /**
  * 
  * @author Adrian Cole
@@ -248,12 +246,17 @@ public abstract class BaseComputeServiceLiveTest {
       }
    }
 
-   private Map<NodeMetadata, ExecResponse> runScriptWithCreds(String tag, OsFamily osFamily,
+   private Map<NodeMetadata, ExecResponse> runScriptWithCreds(final String tag, OsFamily osFamily,
             Credentials creds) throws RunScriptOnNodesException {
       try {
-         return client.runScriptOnNodesMatching(Predicates.<NodeMetadata>alwaysTrue(),
+         return client.runScriptOnNodesMatching(new Predicate<NodeMetadata>() {
-                  buildScript(osFamily).getBytes(),
+
-                  RunScriptOptions.Builder.overrideCredentialsWith(creds));
+            @Override
+            public boolean apply(NodeMetadata arg0) {
+               return arg0.getState() == NodeState.RUNNING && tag.equals(arg0.getTag());
+            }
+         }, buildScript(osFamily).getBytes(), RunScriptOptions.Builder
+                  .overrideCredentialsWith(creds));
       } catch (SshException e) {
          if (Throwables.getRootCause(e).getMessage().contains("Auth fail")) {
             System.err.printf("bad credentials: %s:%s for %s%n", creds.account, creds.key, client

extensions/ssh/jsch/src/main/java/org/jclouds/ssh/jsch/JschSshClient.java

@@ -55,6 +55,7 @@ import com.jcraft.jsch.Session;
 import com.jcraft.jsch.SftpException;
 
 /**
+ * This class needs refactoring. It is not thread safe.
  * 
  * @author Adrian Cole
  */
@@ -168,16 +169,22 @@ public class JschSshClient implements SshClient {
          } catch (Exception from) {
             disconnect();
             String rootMessage = Throwables.getRootCause(from).getMessage();
-            if (i + 1 == sshRetries)
+            if (i == sshRetries)
                throw propagate(from);
             if (Iterables.size(Iterables.filter(Throwables.getCausalChain(from),
                      ConnectException.class)) >= 1
-                     || rootMessage.indexOf("Auth fail") != -1// auth fail sometimes happens in EC2
+                     || Iterables.size(Iterables.filter(Throwables.getCausalChain(from),
+                              IOException.class)) >= 1
+                     || rootMessage.indexOf("invalid privatekey") != -1
+                     || rootMessage.indexOf("Auth fail") != -1// auth fail sometimes happens in EC2,
+                     // as the script that injects the
+                     // authorized key executes after ssh
+                     // has started
                      || rootMessage.indexOf("invalid data") != -1
-                     || rootMessage.indexOf("invalid privatekey") != -1) {
+                     || rootMessage.indexOf("End of IO Stream Read") != -1) {
-               backoffLimitedRetryHandler.imposeBackoffExponentialDelay(i + 1, String.format(
+               backoffLimitedRetryHandler.imposeBackoffExponentialDelay(200L, 2, i + 1, String
-                        "%s@%s:%d: connection error: %s", username, host.getHostAddress(), port,
+                        .format("%s@%s:%d: connection error: %s", username, host.getHostAddress(),
-                        from.getMessage()));
+                                 port, from.getMessage()));
                continue;
             }
             throw propagate(from);