JCLOUDS-1293 Add custom IOException retry handler for AWS-EC2

As all methods use POST we can not use the method to determine if funciton is idempotent.
We therefore set all as idempotent to nullify that check and add a custom
IOException retry handler to determine if commands should be retried on IOException.

The custom hander extends the BackoffLimitedRetryHandler, the current handler, so all other behaviour is not affected.

This does not retry any POST methods unless it's ACTION starts with 'Describe'. These functions are idempotent, and therefore safe to retry.

See JCLOUDS-1293

providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ApiMetadata.java

@@ -21,6 +21,7 @@ import static org.jclouds.ec2.reference.EC2Constants.PROPERTY_EC2_AMI_OWNERS;
 import java.net.URI;
 import java.util.Properties;
 
+import org.jclouds.Constants;
 import org.jclouds.aws.ec2.compute.AWSEC2ComputeServiceContext;
 import org.jclouds.aws.ec2.compute.config.AWSEC2ComputeServiceContextModule;
 import org.jclouds.aws.ec2.config.AWSEC2HttpApiModule;
@@ -53,6 +54,8 @@ public final class AWSEC2ApiMetadata extends BaseHttpApiMetadata<AWSEC2Api> {
       // authorized key executes after ssh has started.  
       properties.setProperty("jclouds.ssh.max-retries", "7");
       properties.setProperty("jclouds.ssh.retry-auth", "true");
+      // required for custom retry handler 
+      properties.setProperty(Constants.PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,PUT,POST");
       return properties;
    }
 

providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceContextModule.java

@@ -29,6 +29,7 @@ import org.jclouds.aws.ec2.compute.functions.PresentSpotRequestsAndInstances;
 import org.jclouds.aws.ec2.compute.strategy.AWSEC2CreateNodesInGroupThenAddToSet;
 import org.jclouds.aws.ec2.compute.strategy.AWSEC2DestroyNodeStrategy;
 import org.jclouds.aws.ec2.compute.strategy.AWSEC2GetNodeMetadataStrategy;
+import org.jclouds.aws.ec2.compute.strategy.AWSEC2IOExceptionRetryHandler;
 import org.jclouds.aws.ec2.compute.strategy.AWSEC2ListNodesStrategy;
 import org.jclouds.aws.ec2.compute.strategy.AWSEC2ReviseParsedImage;
 import org.jclouds.aws.ec2.compute.strategy.CreateKeyPairPlacementAndSecurityGroupsAsNeededAndReturnRunOptions;
@@ -51,6 +52,7 @@ import org.jclouds.ec2.compute.strategy.EC2ListNodesStrategy;
 import org.jclouds.ec2.compute.strategy.ReviseParsedImage;
 import org.jclouds.ec2.compute.suppliers.EC2HardwareSupplier;
 import org.jclouds.ec2.compute.suppliers.RegionAndNameToImageSupplier;
+import org.jclouds.http.IOExceptionRetryHandler;
 import org.jclouds.rest.AuthorizationException;
 import org.jclouds.rest.suppliers.SetAndThrowAuthorizationExceptionSupplier;
 
@@ -84,6 +86,7 @@ public class AWSEC2ComputeServiceContextModule extends BaseComputeServiceContext
       bind(PresentInstances.class).to(PresentSpotRequestsAndInstances.class);
       bind(EC2CreateNodesInGroupThenAddToSet.class).to(AWSEC2CreateNodesInGroupThenAddToSet.class);
       bind(RunningInstanceToNodeMetadata.class).to(AWSRunningInstanceToNodeMetadata.class);
+      bind(IOExceptionRetryHandler.class).to(AWSEC2IOExceptionRetryHandler.class);
    }
 
    protected void installDependencies() {

providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/strategy/AWSEC2IOExceptionRetryHandler.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.aws.ec2.compute.strategy;
+
+import java.io.IOException;
+
+import org.jclouds.aws.reference.FormParameters;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
+import org.jclouds.io.Payload;
+
+public class AWSEC2IOExceptionRetryHandler extends BackoffLimitedRetryHandler {
+
+   private static final String DESCRIBE_ACTION = FormParameters.ACTION + "=Describe";
+
+   @Override
+   public boolean shouldRetryRequest(HttpCommand command, IOException error) {
+      HttpRequest request = command.getCurrentRequest();
+      if ("POST".equals(request.getMethod())) {
+         Payload payload = request.getPayload();
+         if (!payload.getRawContent().toString().contains(DESCRIBE_ACTION)){
+            logger.error("Command not considered safe to retry because request method is POST and action may not be idempotent: %1$s",
+               command);
+            return false;
+         }
+      }
+      return super.shouldRetryRequest(command, error);
+   }
+
+}

providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/compute/strategy/AWSEC2IOExceptionRetryHandlerTest.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.aws.ec2.compute.strategy;
+
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.assertFalse;
+
+import java.io.IOException;
+
+import org.jclouds.aws.reference.FormParameters;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpRequest;
+import org.testng.annotations.Test;
+
+@Test(groups = "unit", testName = "AWSEC2IOExceptionRetryHandlerTest")
+public class AWSEC2IOExceptionRetryHandlerTest {
+
+   @Test
+   public void testDescribeMethodIsRetried() throws Exception {
+
+      AWSEC2IOExceptionRetryHandler handler = new AWSEC2IOExceptionRetryHandler();
+      IOException e = new IOException("test exception");
+      HttpRequest request = HttpRequest.builder().method("POST").endpoint("http://test.endpoint.com/").addFormParam(FormParameters.ACTION, "DescribeInstance").build();
+      HttpCommand command = new HttpCommand(request);
+
+      assertTrue(handler.shouldRetryRequest(command, e));
+    	
+   }
+    
+   @Test
+   public void testNonDescribeMethodIsNotRetried() throws Exception {
+    	
+      AWSEC2IOExceptionRetryHandler handler = new AWSEC2IOExceptionRetryHandler();
+      IOException e = new IOException("test exception");
+      HttpRequest request = HttpRequest.builder().method("POST").endpoint("http://test.endpoint.com/").addFormParam(FormParameters.ACTION, "RunInstances").build();
+      HttpCommand command = new HttpCommand(request);
+    	
+      assertFalse(handler.shouldRetryRequest(command, e));
+    	
+    }
+    
+}