Issue 721:Connect errors not automatically retried in sshj

drivers/sshj/src/main/java/org/jclouds/sshj/SshjSshClient.java

@@ -30,6 +30,7 @@ import static org.jclouds.crypto.SshKeys.sha1PrivateKey;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.ConnectException;
 import java.util.concurrent.TimeUnit;
 
 import javax.annotation.PostConstruct;
@@ -58,6 +59,7 @@ import org.jclouds.net.IPSocket;
 import org.jclouds.rest.AuthorizationException;
 import org.jclouds.ssh.SshClient;
 import org.jclouds.ssh.SshException;
+import org.jclouds.util.Throwables2;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
@@ -71,6 +73,7 @@ import com.google.inject.Inject;
  * 
  * @author Adrian Cole
  */
+@SuppressWarnings("unchecked")
 public class SshjSshClient implements SshClient {
 
    private final class CloseFtpChannelOnCloseInputStream extends ProxyInputStream {
@@ -115,7 +118,7 @@ public class SshjSshClient implements SshClient {
    @Named("jclouds.ssh.retry-predicate")
    // NOTE cannot retry io exceptions, as SSHException is a part of the chain
    private Predicate<Throwable> retryPredicate = or(instanceOf(ConnectionException.class),
-            instanceOf(TransportException.class));
+            instanceOf(ConnectException.class), instanceOf(TransportException.class));
 
    @Resource
    @Named("jclouds.ssh")
@@ -222,11 +225,21 @@ public class SshjSshClient implements SshClient {
                logger.warn(from, "<< (%s) error closing connection", toString());
             }
             if (i + 1 == sshRetries) {
+               logger.error(from, "<< " + errorMessage + ": out of retries %d", sshRetries);
                throw propagate(from, errorMessage);
+            } else if (Throwables2.getFirstThrowableOfType(from, IllegalStateException.class) != null) {
+               logger.warn(from, "<< " + errorMessage + ": " + from.getMessage());
+               disconnect();
+               backoffForAttempt(i + 1, errorMessage + ": " + from.getMessage());
+               connect();
+               continue;
             } else if (shouldRetry(from)) {
                logger.warn(from, "<< " + errorMessage + ": " + from.getMessage());
                backoffForAttempt(i + 1, errorMessage + ": " + from.getMessage());
                continue;
+            } else {
+               logger.error(from, "<< " + errorMessage + ": exception not retryable");
+               throw propagate(from, errorMessage);
             }
          }
       }

drivers/sshj/src/test/java/org/jclouds/sshj/SshjSshClientLiveTest.java

@@ -127,7 +127,7 @@ public class SshjSshClientLiveTest {
       temp.deleteOnExit();
       SshClient client = setupClient();
       client.put(temp.getAbsolutePath(), Payloads.newStringPayload("rabbit"));
-      Payload input = setupClient().get(temp.getAbsolutePath());
+      Payload input = client.get(temp.getAbsolutePath());
       String contents = Strings2.toStringAndClose(input.getInput());
       assertEquals(contents, "rabbit");
    }
@@ -138,8 +138,9 @@ public class SshjSshClientLiveTest {
       assert contents.indexOf("root") >= 0 : "no root in " + contents;
    }
 
-   public void testExecHostname() throws IOException {
-      ExecResponse response = setupClient().exec("hostname");
+   public void testExecHostname() throws IOException, InterruptedException {
+      SshClient client = setupClient();
+      ExecResponse response = client.exec("hostname");
       assertEquals(response.getError(), "");
       assertEquals(response.getOutput().trim(), "localhost".equals(sshHost) ? InetAddress.getLocalHost().getHostName()
                : sshHost);

drivers/sshj/src/test/java/org/jclouds/sshj/SshjSshClientTest.java

@@ -21,6 +21,7 @@ package org.jclouds.sshj;
 import static com.google.inject.name.Names.bindProperties;
 
 import java.io.IOException;
+import java.net.ConnectException;
 import java.util.Properties;
 
 import net.schmizz.sshj.common.SSHException;
@@ -87,6 +88,7 @@ public class SshjSshClientTest {
    public void testExceptionClassesRetry() {
       assert ssh.shouldRetry(new TransportException("socket closed"));
       assert ssh.shouldRetry(new ConnectionException("problem"));
+      assert ssh.shouldRetry(new ConnectException("Connection refused"));
       assert !ssh.shouldRetry(new IOException("channel %s is not open", new NullPointerException()));
    }