Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-1028: Configure idempotent methods 

Enable POST for Atmos, S3, and Swift.
This commit is contained in:
Andrew Gaul 2016-06-15 15:14:02 -07:00
parent f98116ec83
commit 5fec2346a6
15 changed files with 63 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apis/atmos/src/main/java/org/jclouds/atmos/AtmosApiMetadata.java
View File

@ -16,6 +16,7 @@
*/
package org.jclouds.atmos;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX;
import static org.jclouds.location.reference.LocationConstants.PROPERTY_REGIONS;
import static org.jclouds.reflect.Reflection2.typeToken;
@ -57,6 +58,7 @@ public class AtmosApiMetadata extends BaseHttpApiMetadata {
Properties properties = BaseHttpApiMetadata.defaultProperties();
properties.setProperty(PROPERTY_REGIONS, "DEFAULT");
properties.setProperty(PROPERTY_USER_METADATA_PREFIX, "X-Object-Meta-");
properties.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,POST,PUT");
return properties;
}

2
apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java
View File

@ -16,6 +16,7 @@
*/
package org.jclouds.openstack.swift.v1;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties.CREDENTIAL_TYPE;
import static org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties.SERVICE_TYPE;
import static org.jclouds.reflect.Reflection2.typeToken;
@ -58,6 +59,7 @@ public class SwiftApiMetadata extends BaseHttpApiMetadata<SwiftApi> {
public static Properties defaultProperties() {
Properties properties = BaseHttpApiMetadata.defaultProperties();
properties.setProperty(SERVICE_TYPE, ServiceType.OBJECT_STORE);
properties.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,POST,PUT");
// Can alternatively be set to "tempAuthCredentials"
properties.setProperty(CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS);
return properties;

2
apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java
View File

@ -16,6 +16,7 @@
*/
package org.jclouds.s3;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.Constants.PROPERTY_RELAX_HOSTNAME;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
@ -79,6 +80,7 @@ public class S3ApiMetadata extends BaseHttpApiMetadata {
properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true");
properties.setProperty(PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX, "/");
properties.setProperty(PROPERTY_USER_METADATA_PREFIX, String.format("x-${%s}-meta-", PROPERTY_HEADER_TAG));
properties.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,POST,PUT");
// Chunk size must be at least 8 KB. We recommend a chunk size of a least 64 KB for better performance.
properties.setProperty(PROPERTY_JCLOUDS_S3_CHUNKED_SIZE, String.valueOf(64 * 1024));

3
core/src/main/java/org/jclouds/Constants.java
View File

@ -350,6 +350,9 @@ public final class Constants {
*/
public static final String PROPERTY_MAX_PARALLEL_DELETES = "jclouds.max-parallel-deletes";
/** Comma-separated list of methods considered idempotent for purposes of retries. By default jclouds uses DELETE,GET,HEAD,OPTIONS,PUT. */
public static final String PROPERTY_IDEMPOTENT_METHODS = "jclouds.idempotent-methods";
private Constants() {
throw new AssertionError("intentionally unimplemented");
}

3
core/src/main/java/org/jclouds/apis/internal/BaseApiMetadata.java
View File

@ -20,6 +20,7 @@ import static com.google.common.base.Objects.equal;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.Constants.PROPERTY_CONNECTION_CLOSE_HEADER;
import static org.jclouds.Constants.PROPERTY_CONNECTION_TIMEOUT;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.Constants.PROPERTY_ISO3166_CODES;
import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT;
import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST;
@ -79,6 +80,8 @@ public abstract class BaseApiMetadata implements ApiMetadata {
// By default, we allow maximum parallel deletes to be equal to the number
// of user threads since one thread is used to delete on blob.
props.setProperty(PROPERTY_MAX_PARALLEL_DELETES, numUserThreads + "");
props.setProperty(PROPERTY_IDEMPOTENT_METHODS, "DELETE,GET,HEAD,OPTIONS,PUT");
return props;
}

11
core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java
View File

@ -18,6 +18,7 @@ package org.jclouds.http.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Throwables.propagate;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.http.HttpUtils.checkRequestHasContentLengthOrChunkedEncoding;
import static org.jclouds.http.HttpUtils.releasePayload;
import static org.jclouds.http.HttpUtils.wirePayloadIfEnabled;
@ -48,8 +49,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandExecutorService {
private static final Set<String> IDEMPOTENT_METHODS = ImmutableSet.of("GET", "HEAD", "OPTIONS", "PUT", "DELETE");
protected final HttpUtils utils;
protected final ContentMetadataCodec contentMetadataCodec;
@ -65,16 +64,20 @@ public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandEx
protected final HttpWire wire;
private final Set<String> idempotentMethods;
@Inject
protected BaseHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire) {
DelegatingErrorHandler errorHandler, HttpWire wire,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
this.utils = checkNotNull(utils, "utils");
this.contentMetadataCodec = checkNotNull(contentMetadataCodec, "contentMetadataCodec");
this.retryHandler = checkNotNull(retryHandler, "retryHandler");
this.ioRetryHandler = checkNotNull(ioRetryHandler, "ioRetryHandler");
this.errorHandler = checkNotNull(errorHandler, "errorHandler");
this.wire = checkNotNull(wire, "wire");
this.idempotentMethods = ImmutableSet.copyOf(idempotentMethods.split(","));
}
@Override
@ -147,7 +150,7 @@ public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandEx
private boolean isIdempotent(HttpCommand command) {
String method = command.getCurrentRequest().getMethod();
if (!IDEMPOTENT_METHODS.contains(method)) {
if (!idempotentMethods.contains(method)) {
logger.error("Command not considered safe to retry because request method is %1$s: %2$s", method, command);
return false;
} else {

6
core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java
View File

@ -21,6 +21,7 @@ import static com.google.common.base.Throwables.propagate;
import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
import static com.google.common.net.HttpHeaders.HOST;
import static com.google.common.net.HttpHeaders.USER_AGENT;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.http.HttpUtils.filterOutContentHeaders;
import static org.jclouds.io.Payloads.newInputStreamPayload;
import static org.jclouds.util.Closeables2.closeQuietly;
@ -77,8 +78,9 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
public JavaUrlHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
if (utils.getMaxConnections() > 0) {
System.setProperty("http.maxConnections", String.valueOf(checkNotNull(utils, "utils").getMaxConnections()));
}

7
core/src/test/java/org/jclouds/http/internal/BaseHttpCommandExecutorServiceTest.java
View File

@ -21,6 +21,7 @@ import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.getCurrentArguments;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
import static org.jclouds.http.HttpUtils.releasePayload;
import static org.jclouds.io.Payloads.newInputStreamPayload;
@ -33,6 +34,7 @@ import java.io.IOException;
import java.io.InputStream;
import javax.inject.Inject;
import javax.inject.Named;
import org.easymock.EasyMock;
import org.easymock.IAnswer;
@ -307,8 +309,9 @@ public class BaseHttpCommandExecutorServiceTest {
@Inject
MockHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
DelegatingErrorHandler errorHandler, HttpWire wire,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
}
@Override

8
core/src/test/java/org/jclouds/http/internal/TrackingJavaUrlHttpCommandExecutorService.java
View File

@ -16,6 +16,8 @@
*/
package org.jclouds.http.internal;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import java.net.Proxy;
import java.net.URI;
import java.util.Collection;
@ -87,9 +89,11 @@ public class TrackingJavaUrlHttpCommandExecutorService extends JavaUrlHttpComman
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
List<HttpCommand> commandsInvoked) throws SecurityException, NoSuchFieldException {
List<HttpCommand> commandsInvoked,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods)
throws SecurityException, NoSuchFieldException {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier,
untrustedSSLContextProvider, proxyForURI);
untrustedSSLContextProvider, proxyForURI, idempotentMethods);
this.commandsInvoked = commandsInvoked;
}

6
core/src/test/java/org/jclouds/rest/internal/BaseRestApiExpectTest.java
View File

@ -19,6 +19,7 @@ package org.jclouds.rest.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
import static com.google.inject.name.Names.named;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.Constants.PROPERTY_MAX_RETRIES;
import static org.jclouds.Constants.PROPERTY_USER_THREADS;
import static org.testng.Assert.assertEquals;
@ -191,8 +192,9 @@ public abstract class BaseRestApiExpectTest<S> {
@Inject
public ExpectHttpCommandExecutorService(Function<HttpRequest, HttpResponse> fn, HttpUtils utils,
ContentMetadataCodec contentMetadataCodec, IOExceptionRetryHandler ioRetryHandler,
DelegatingRetryHandler retryHandler, DelegatingErrorHandler errorHandler, HttpWire wire) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
DelegatingRetryHandler retryHandler, DelegatingErrorHandler errorHandler, HttpWire wire,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
this.fn = checkNotNull(fn, "fn");
}

7
drivers/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorService.java
View File

@ -18,12 +18,14 @@ package org.jclouds.http.apachehc;
import static com.google.common.hash.Hashing.md5;
import static com.google.common.io.BaseEncoding.base64;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.http.HttpUtils.filterOutContentHeaders;
import java.io.IOException;
import java.net.URI;
import javax.inject.Inject;
import javax.inject.Named;
import org.apache.http.Header;
import org.apache.http.HttpHost;
@ -56,8 +58,9 @@ public class ApacheHCHttpCommandExecutorService extends BaseHttpCommandExecutorS
@Inject
ApacheHCHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, HttpClient client) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
DelegatingErrorHandler errorHandler, HttpWire wire, HttpClient client,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
this.client = client;
this.apacheHCUtils = new ApacheHCUtils(contentMetadataCodec);
}

8
drivers/gae/src/main/java/org/jclouds/gae/GaeHttpCommandExecutorService.java
View File

@ -16,9 +16,12 @@
*/
package org.jclouds.gae;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import java.io.IOException;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.JcloudsVersion;
@ -56,8 +59,9 @@ public class GaeHttpCommandExecutorService extends BaseHttpCommandExecutorServic
ContentMetadataCodec contentMetadataCodec,
IOExceptionRetryHandler ioRetryHandler, DelegatingRetryHandler retryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, ConvertToGaeRequest convertToGaeRequest,
ConvertToJcloudsResponse convertToJcloudsResponse) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
ConvertToJcloudsResponse convertToJcloudsResponse,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
this.urlFetchService = urlFetchService;
this.convertToGaeRequest = convertToGaeRequest;
this.convertToJcloudsResponse = convertToJcloudsResponse;

8
drivers/okhttp/src/main/java/org/jclouds/http/okhttp/OkHttpCommandExecutorService.java
View File

@ -19,6 +19,7 @@ package org.jclouds.http.okhttp;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.net.HttpHeaders.ACCEPT;
import static com.google.common.net.HttpHeaders.USER_AGENT;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.http.HttpUtils.filterOutContentHeaders;
import static org.jclouds.io.Payloads.newInputStreamPayload;
@ -27,6 +28,8 @@ import java.net.Proxy;
import java.net.URI;
import java.util.Map;
import javax.inject.Named;
import okio.BufferedSink;
import okio.Okio;
import okio.Source;
@ -66,8 +69,9 @@ public final class OkHttpCommandExecutorService extends BaseHttpCommandExecutorS
@Inject
OkHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, Function<URI, Proxy> proxyForURI, OkHttpClient okHttpClient) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
DelegatingErrorHandler errorHandler, HttpWire wire, Function<URI, Proxy> proxyForURI, OkHttpClient okHttpClient,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, idempotentMethods);
this.proxyForURI = proxyForURI;
this.globalClient = okHttpClient;
}

6
providers/dynect/src/main/java/org/jclouds/dynect/v3/config/DynECTHttpApiModule.java
View File

@ -16,6 +16,7 @@
*/
package org.jclouds.dynect.v3.config;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
import static org.jclouds.rest.config.BinderUtils.bindHttpApi;
@ -102,10 +103,11 @@ public class DynECTHttpApiModule extends HttpApiModule<DynECTApi> {
private SillyRabbit200sAreForSuccess(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI)
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods)
throws SecurityException, NoSuchFieldException {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier,
untrustedSSLContextProvider, proxyForURI);
untrustedSSLContextProvider, proxyForURI, idempotentMethods);
}
/**

6
providers/profitbricks/src/main/java/org/jclouds/profitbricks/http/ResponseStatusFromPayloadHttpCommandExecutorService.java
View File

@ -16,6 +16,7 @@
*/
package org.jclouds.profitbricks.http;
import static org.jclouds.Constants.PROPERTY_IDEMPOTENT_METHODS;
import static org.jclouds.util.Closeables2.closeQuietly;
import java.io.ByteArrayInputStream;
@ -67,8 +68,9 @@ public class ResponseStatusFromPayloadHttpCommandExecutorService extends JavaUrl
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI,
ParseSax<ServiceFault> faultHandler) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier, untrustedSSLContextProvider, proxyForURI);
ParseSax<ServiceFault> faultHandler,
@Named(PROPERTY_IDEMPOTENT_METHODS) String idempotentMethods) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier, untrustedSSLContextProvider, proxyForURI, idempotentMethods);
this.faultHandler = faultHandler;
}