From 4b00bd96497979a5675c9c24d44ec23203779915 Mon Sep 17 00:00:00 2001
From: "adrian.f.cole" <adrian.f.cole@3d8758e0-26b5-11de-8745-db77d3ebf521>
Date: Fri, 22 Jan 2010 21:35:45 +0000
Subject: [PATCH] Issue 130: added retry logic for ssh runscript

git-svn-id: http://jclouds.googlecode.com/svn/trunk@2724 3d8758e0-26b5-11de-8745-db77d3ebf521
---
 .../jclouds/compute/util/ComputeUtils.java    | 57 ++++++++++++-------
 .../antcontrib/samples/cargooverssh/build.xml |  2 +-
 2 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java b/compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java
index 6ccfd5a006..10ae7e51d2 100644
--- a/compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java
+++ b/compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java
@@ -21,6 +21,7 @@ package org.jclouds.compute.util;
 import static com.google.common.base.Preconditions.checkState;
 
 import java.io.ByteArrayInputStream;
+import java.net.ConnectException;
 import java.net.InetSocketAddress;
 
 import javax.annotation.Resource;
@@ -33,6 +34,7 @@ import org.jclouds.logging.Logger;
 import org.jclouds.ssh.SshClient;
 
 import com.google.common.base.Predicate;
+import com.google.common.base.Throwables;
 import com.google.common.collect.Iterables;
 import com.google.inject.Inject;
 
@@ -71,26 +73,43 @@ public class ComputeUtils {
       SshClient ssh = isKeyAuth(node) ? sshFactory.create(socket, node.getCredentials().account,
                node.getCredentials().key.getBytes()) : sshFactory.create(socket, node
                .getCredentials().account, node.getCredentials().key);
-      try {
-         ssh.connect();
-         String scriptName = node.getId() + ".sh";
-         ssh.put(scriptName, new ByteArrayInputStream(script));
-         ssh.exec("chmod 755 " + scriptName);
-         if (node.getCredentials().account.equals("root")) {
-            logger.debug(">> running %s as %s", scriptName, node.getCredentials().account);
-            logger.debug("<< complete(%d)", ssh.exec("./" + scriptName).getExitCode());
-         } else if (isKeyAuth(node)) {
-            logger.debug(">> running sudo %s as %s", scriptName, node.getCredentials().account);
-            logger.debug("<< complete(%d)", ssh.exec("sudo ./" + scriptName).getExitCode());
-         } else {
-            logger.debug(">> running sudo -S %s as %s", scriptName, node.getCredentials().account);
-            logger.debug("<< complete(%d)", ssh.exec(
-                     String.format("echo %s|sudo -S ./%s", node.getCredentials().key, scriptName))
-                     .getExitCode());
+      for (int i = 0; i < 3; i++) {
+         try {
+            ssh.connect();
+            runScriptOnNodeWithClient(ssh, node, script);
+         } catch (RuntimeException from) {
+            if (Iterables.size(Iterables.filter(Throwables.getCausalChain(from),
+                     ConnectException.class)) >= 1) {
+               try {
+                  Thread.sleep(100);
+               } catch (InterruptedException e) {
+                  Throwables.propagate(e);
+               }
+               continue;
+            }
+            Throwables.propagate(from);
+         } finally {
+            if (ssh != null)
+               ssh.disconnect();
          }
-      } finally {
-         if (ssh != null)
-            ssh.disconnect();
+      }
+   }
+
+   private void runScriptOnNodeWithClient(SshClient ssh, CreateNodeResponse node, byte[] script) {
+      String scriptName = node.getId() + ".sh";
+      ssh.put(scriptName, new ByteArrayInputStream(script));
+      ssh.exec("chmod 755 " + scriptName);
+      if (node.getCredentials().account.equals("root")) {
+         logger.debug(">> running %s as %s", scriptName, node.getCredentials().account);
+         logger.debug("<< complete(%d)", ssh.exec("./" + scriptName).getExitCode());
+      } else if (isKeyAuth(node)) {
+         logger.debug(">> running sudo %s as %s", scriptName, node.getCredentials().account);
+         logger.debug("<< complete(%d)", ssh.exec("sudo ./" + scriptName).getExitCode());
+      } else {
+         logger.debug(">> running sudo -S %s as %s", scriptName, node.getCredentials().account);
+         logger.debug("<< complete(%d)", ssh.exec(
+                  String.format("echo %s|sudo -S ./%s", node.getCredentials().key, scriptName))
+                  .getExitCode());
       }
    }
 
diff --git a/tools/antcontrib/samples/cargooverssh/build.xml b/tools/antcontrib/samples/cargooverssh/build.xml
index fea9d803f8..9cda0f0c52 100644
--- a/tools/antcontrib/samples/cargooverssh/build.xml
+++ b/tools/antcontrib/samples/cargooverssh/build.xml
@@ -67,7 +67,7 @@
     <compute action="destroy" provider="${url}">
        <node name="${nodename}" />
     </compute>
-    <sleep seconds="2" />
+    <sleep seconds="5" />
   </target>
 
   <target name="create" description="create the node ${nodename}" depends="destroy" >