JCLOUDS-1028: Configure idempotent methods

Enable POST for Atmos, S3, and Swift.

apis/atmos/src/main/java/org/jclouds/atmos/AtmosApiMetadata.java

@@ -16,6 +16,7 @@
  */
 package org.jclouds.atmos;
 
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX;
 import static org.jclouds.location.reference.LocationConstants.PROPERTY_REGIONS;
 import static org.jclouds.reflect.Reflection2.typeToken;
@@ -57,6 +58,7 @@ public class AtmosApiMetadata extends BaseHttpApiMetadata {
       Properties properties = BaseHttpApiMetadata.defaultProperties();
       properties.setProperty(PROPERTY_REGIONS, "DEFAULT");
       properties.setProperty(PROPERTY_USER_METADATA_PREFIX, "X-Object-Meta-");
+      properties.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,POST,PUT");
       return properties;
    }
 

apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java

@@ -16,6 +16,7 @@
  */
 package org.jclouds.openstack.swift.v1;
 
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties.CREDENTIAL_TYPE;
 import static org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties.SERVICE_TYPE;
 import static org.jclouds.reflect.Reflection2.typeToken;
@@ -58,6 +59,7 @@ public class SwiftApiMetadata extends BaseHttpApiMetadata<SwiftApi> {
    public static Properties defaultProperties() {
       Properties properties = BaseHttpApiMetadata.defaultProperties();
       properties.setProperty(SERVICE_TYPE, ServiceType.OBJECT_STORE);
+      properties.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,POST,PUT");
       // Can alternatively be set to "tempAuthCredentials"
       properties.setProperty(CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS);
       return properties;

apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java

@@ -16,6 +16,7 @@
  */
 package org.jclouds.s3;
 
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.Constants.PROPERTY_RELAX_HOSTNAME;
 import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
 import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
@@ -79,6 +80,7 @@ public class S3ApiMetadata extends BaseHttpApiMetadata {
       properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true");
       properties.setProperty(PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX, "/");
       properties.setProperty(PROPERTY_USER_METADATA_PREFIX, String.format("x-${%s}-meta-", PROPERTY_HEADER_TAG));
+      properties.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,POST,PUT");
 
       // Chunk size must be at least 8 KB. We recommend a chunk size of a least 64 KB for better performance.
       properties.setProperty(PROPERTY_JCLOUDS_S3_CHUNKED_SIZE, String.valueOf(64 * 1024));

core/src/main/java/org/jclouds/Constants.java

@@ -350,6 +350,9 @@ public final class Constants {
     */
    public static final String PROPERTY_MAX_PARALLEL_DELETES = "jclouds.max-parallel-deletes";
 
+   /** Comma-separated list of methods considered idempotent for purposes of retries.  By default jclouds uses DELETE,GET,HEAD,OPTIONS,PUT. */
+   public static final String PROPERTY_IDEMPOTENT_METHODS = "jclouds.idempotent-methods";
+
    private Constants() {
       throw new AssertionError("intentionally unimplemented");
    }

core/src/main/java/org/jclouds/apis/internal/BaseApiMetadata.java

@@ -20,6 +20,7 @@ import static com.google.common.base.Objects.equal;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.jclouds.Constants.PROPERTY_CONNECTION_CLOSE_HEADER;
 import static org.jclouds.Constants.PROPERTY_CONNECTION_TIMEOUT;
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.Constants.PROPERTY_ISO3166_CODES;
 import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT;
 import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST;
@@ -79,6 +80,8 @@ public abstract class BaseApiMetadata implements ApiMetadata {
       // By default, we allow maximum parallel deletes to be equal to the number
       // of user threads since one thread is used to delete on blob.
       props.setProperty(PROPERTY_MAX_PARALLEL_DELETES, numUserThreads + "");
+
+      props.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,PUT");
       return props;
    }
 

core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java

@@ -18,6 +18,7 @@ package org.jclouds.http.internal;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Throwables.propagate;
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.http.HttpUtils.checkRequestHasContentLengthOrChunkedEncoding;
 import static org.jclouds.http.HttpUtils.releasePayload;
 import static org.jclouds.http.HttpUtils.wirePayloadIfEnabled;
@@ -48,8 +49,6 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 
 public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandExecutorService {
-   private static final Set<String> IDEMPOTENT_METHODS = ImmutableSet.of("GET", "HEAD", "OPTIONS", "PUT", "DELETE");
-
    protected final HttpUtils utils;
    protected final ContentMetadataCodec contentMetadataCodec;
 
@@ -65,16 +64,20 @@ public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandEx
 
    protected final HttpWire wire;
 
+   private final Set<String> idempotentMethods;
+
    @Inject
    protected BaseHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
          DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
-         DelegatingErrorHandler errorHandler, HttpWire wire) {
+         DelegatingErrorHandler errorHandler, HttpWire wire,
+         @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
       this.utils = checkNotNull(utils, "utils");
       this.contentMetadataCodec = checkNotNull(contentMetadataCodec, "contentMetadataCodec");
       this.retryHandler = checkNotNull(retryHandler, "retryHandler");
       this.ioRetryHandler = checkNotNull(ioRetryHandler, "ioRetryHandler");
       this.errorHandler = checkNotNull(errorHandler, "errorHandler");
       this.wire = checkNotNull(wire, "wire");
+      this.idempotentMethods = ImmutableSet.copyOf(idempotentMethods.split(","));
    }
 
    @Override
@@ -147,7 +150,7 @@ public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandEx
 
    private boolean isIdempotent(HttpCommand command) {
       String method = command.getCurrentRequest().getMethod();
-      if (!IDEMPOTENT_METHODS.contains(method)) {
+      if (!idempotentMethods.contains(method)) {
          logger.error("Command not considered safe to retry because request method is %1$s: %2$s", method, command);
          return false;
       } else {

core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java

@@ -21,6 +21,7 @@ import static com.google.common.base.Throwables.propagate;
 import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.HOST;
 import static com.google.common.net.HttpHeaders.USER_AGENT;
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.http.HttpUtils.filterOutContentHeaders;
 import static org.jclouds.io.Payloads.newInputStreamPayload;
 import static org.jclouds.util.Closeables2.closeQuietly;
@@ -77,8 +78,9 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
    public JavaUrlHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
          DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
          DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
-         @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI) {
+         @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
-      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
+         @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
+      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
       if (utils.getMaxConnections() > 0) {
          System.setProperty("http.maxConnections", String.valueOf(checkNotNull(utils, "utils").getMaxConnections()));
       }

core/src/test/java/org/jclouds/http/internal/BaseHttpCommandExecutorServiceTest.java

@@ -21,6 +21,7 @@ import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.getCurrentArguments;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
 import static org.jclouds.http.HttpUtils.releasePayload;
 import static org.jclouds.io.Payloads.newInputStreamPayload;
@@ -33,6 +34,7 @@ import java.io.IOException;
 import java.io.InputStream;
 
 import javax.inject.Inject;
+import javax.inject.Named;
 
 import org.easymock.EasyMock;
 import org.easymock.IAnswer;
@@ -307,8 +309,9 @@ public class BaseHttpCommandExecutorServiceTest {
       @Inject
       MockHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
             DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
-            DelegatingErrorHandler errorHandler, HttpWire wire) {
+            DelegatingErrorHandler errorHandler, HttpWire wire,
-         super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
+            @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
+         super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
       }
 
       @Override

core/src/test/java/org/jclouds/http/internal/TrackingJavaUrlHttpCommandExecutorService.java

@@ -16,6 +16,8 @@
  */
 package org.jclouds.http.internal;
 
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
+
 import java.net.Proxy;
 import java.net.URI;
 import java.util.Collection;
@@ -87,9 +89,11 @@ public class TrackingJavaUrlHttpCommandExecutorService extends JavaUrlHttpComman
             DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
             DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
             @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
-            List<HttpCommand> commandsInvoked) throws SecurityException, NoSuchFieldException {
+            List<HttpCommand> commandsInvoked,
+            @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods)
+            throws SecurityException, NoSuchFieldException {
       super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier,
-            untrustedSSLContextProvider, proxyForURI);
+            untrustedSSLContextProvider, proxyForURI, idempotentMethods);
       this.commandsInvoked = commandsInvoked;
    }
 

core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java

@@ -19,6 +19,7 @@ package org.jclouds.rest.internal;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
 import static com.google.inject.name.Names.named;
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.Constants.PROPERTY_MAX_RETRIES;
 import static org.jclouds.Constants.PROPERTY_USER_THREADS;
 import static org.testng.Assert.assertEquals;
@@ -191,8 +192,9 @@ public abstract class BaseRestApiExpectTest<S> {
       @Inject
       public ExpectHttpCommandExecutorService(Function<HttpRequest, HttpResponse> fn, HttpUtils utils,
             ContentMetadataCodec contentMetadataCodec, IOExceptionRetryHandler ioRetryHandler,
-            DelegatingRetryHandler retryHandler, DelegatingErrorHandler errorHandler, HttpWire wire) {
+            DelegatingRetryHandler retryHandler, DelegatingErrorHandler errorHandler, HttpWire wire,
-         super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
+            @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
+         super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
          this.fn = checkNotNull(fn, "fn");
       }
 

drivers/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorService.java

@@ -18,12 +18,14 @@ package org.jclouds.http.apachehc;
 
 import static com.google.common.hash.Hashing.md5;
 import static com.google.common.io.BaseEncoding.base64;
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.http.HttpUtils.filterOutContentHeaders;
 
 import java.io.IOException;
 import java.net.URI;
 
 import javax.inject.Inject;
+import javax.inject.Named;
 
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
@@ -56,8 +58,9 @@ public class ApacheHCHttpCommandExecutorService extends BaseHttpCommandExecutorS
    @Inject
    ApacheHCHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
          DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
-         DelegatingErrorHandler errorHandler, HttpWire wire, HttpClient client) {
+         DelegatingErrorHandler errorHandler, HttpWire wire, HttpClient client,
-      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
+         @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
+      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
       this.client = client;
       this.apacheHCUtils = new ApacheHCUtils(contentMetadataCodec);
    }

drivers/gae/src/main/java/org/jclouds/gae/GaeHttpCommandExecutorService.java

@@ -16,9 +16,12 @@
  */
 package org.jclouds.gae;
 
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
+
 import java.io.IOException;
 
 import javax.inject.Inject;
+import javax.inject.Named;
 import javax.inject.Singleton;
 
 import org.jclouds.JcloudsVersion;
@@ -56,8 +59,9 @@ public class GaeHttpCommandExecutorService extends BaseHttpCommandExecutorServic
             ContentMetadataCodec contentMetadataCodec,
             IOExceptionRetryHandler ioRetryHandler, DelegatingRetryHandler retryHandler,
             DelegatingErrorHandler errorHandler, HttpWire wire, ConvertToGaeRequest convertToGaeRequest,
-            ConvertToJcloudsResponse convertToJcloudsResponse) {
+            ConvertToJcloudsResponse convertToJcloudsResponse,
-      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
+            @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
+      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
       this.urlFetchService = urlFetchService;
       this.convertToGaeRequest = convertToGaeRequest;
       this.convertToJcloudsResponse = convertToJcloudsResponse;

drivers/okhttp/src/main/java/org/jclouds/http/okhttp/OkHttpCommandExecutorService.java

@@ -19,6 +19,7 @@ package org.jclouds.http.okhttp;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.net.HttpHeaders.ACCEPT;
 import static com.google.common.net.HttpHeaders.USER_AGENT;
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.http.HttpUtils.filterOutContentHeaders;
 import static org.jclouds.io.Payloads.newInputStreamPayload;
 
@@ -27,6 +28,8 @@ import java.net.Proxy;
 import java.net.URI;
 import java.util.Map;
 
+import javax.inject.Named;
+
 import okio.BufferedSink;
 import okio.Okio;
 import okio.Source;
@@ -66,8 +69,9 @@ public final class OkHttpCommandExecutorService extends BaseHttpCommandExecutorS
    @Inject
    OkHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
          DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
-         DelegatingErrorHandler errorHandler, HttpWire wire, Function<URI, Proxy> proxyForURI, OkHttpClient okHttpClient) {
+         DelegatingErrorHandler errorHandler, HttpWire wire, Function<URI, Proxy> proxyForURI, OkHttpClient okHttpClient,
-      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
+         @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
+      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
       this.proxyForURI = proxyForURI;
       this.globalClient = okHttpClient;
    }

providers/dynect/src/main/java/org/jclouds/dynect/v3/config/DynECTHttpApiModule.java

@@ -16,6 +16,7 @@
  */
 package org.jclouds.dynect.v3.config;
 
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
 import static org.jclouds.rest.config.BinderUtils.bindHttpApi;
 
@@ -102,10 +103,11 @@ public class DynECTHttpApiModule extends HttpApiModule<DynECTApi> {
       private SillyRabbit200sAreForSuccess(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
             DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
             DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
-            @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI)
+            @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
+            @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods)
             throws SecurityException, NoSuchFieldException {
          super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier,
-               untrustedSSLContextProvider, proxyForURI);
+               untrustedSSLContextProvider, proxyForURI, idempotentMethods);
       }
 
       /**

providers/profitbricks/src/main/java/org/jclouds/profitbricks/http/ResponseStatusFromPayloadHttpCommandExecutorService.java

@@ -16,6 +16,7 @@
  */
 package org.jclouds.profitbricks.http;
 
+import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
 import static org.jclouds.util.Closeables2.closeQuietly;
 
 import java.io.ByteArrayInputStream;
@@ -67,8 +68,9 @@ public class ResponseStatusFromPayloadHttpCommandExecutorService extends JavaUrl
            DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
            DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
            @Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
-           ParseSax<ServiceFault> faultHandler) {
+           ParseSax<ServiceFault> faultHandler,
-      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier, untrustedSSLContextProvider, proxyForURI);
+           @Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
+      super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier, untrustedSSLContextProvider, proxyForURI, idempotentMethods);
       this.faultHandler = faultHandler;
    }