Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-744: Upgrade to OkHttp 2.1.0 and use its native API

This commit is contained in:
Ignasi Barrera 2014-11-27 18:06:01 +01:00
parent 7775f1a6b0
commit 902f1b4105
16 changed files with 226 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apis/ec2/src/test/java/org/jclouds/ec2/internal/BaseEC2ApiMockTest.java
View File

@ -63,9 +63,10 @@ public class BaseEC2ApiMockTest {
protected ContextBuilder builder(Properties overrides) {
overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, "1");
MockWebServer defaultServer = regionToServers.get(DEFAULT_REGION);
return ContextBuilder.newBuilder(new EC2ApiMetadata())
.credentials(ACCESS_KEY, SECRET_KEY)
.endpoint("http://localhost:" + regionToServers.get(DEFAULT_REGION).getPort())
.endpoint(defaultServer.getUrl("").toString())
.overrides(overrides)
.modules(modules);
}
@ -101,7 +102,8 @@ public class BaseEC2ApiMockTest {
server.play();
regionToServers.put(region, server);
}
String regionEndpoint = "http://localhost:" + regionToServers.get(region).getPort();
MockWebServer server = regionToServers.get(region);
String regionEndpoint = server.getUrl("").toString();
describeRegionsResponse.append("<regionEndpoint>").append(regionEndpoint).append("</regionEndpoint>");
describeRegionsResponse.append("</item>");
}

7
apis/openstack-keystone/src/main/java/org/jclouds/openstack/keystone/v2_0/handlers/RetryOnRenew.java
View File

@ -17,7 +17,6 @@
package org.jclouds.openstack.keystone.v2_0.handlers;
import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
import static org.jclouds.http.HttpUtils.releasePayload;
import java.util.concurrent.TimeUnit;
@ -122,7 +121,11 @@ public class RetryOnRenew implements HttpRetryHandler {
}
return retry;
} finally {
releasePayload(response);
// If the request is failed and is not going to be retried, the
// ErrorHandler will be invoked and it might need to read the payload.
// For some kind of payload sources, such as the OkHttp Source, if the
// payload is released, the upcoming operations will fail.
closeClientButKeepContentStream(response);
}
}

2
apis/openstack-keystone/src/test/java/org/jclouds/openstack/keystone/v2_0/handlers/RetryOnRenewTest.java
View File

@ -112,7 +112,7 @@ public class RetryOnRenewTest {
BackoffLimitedRetryHandler backoffHandler = createMock(BackoffLimitedRetryHandler.class);
expect(response.getPayload()).andReturn(Payloads.newStringPayload(
"The server has waited too long for the request to be sent by the client.")).times(2);
"The server has waited too long for the request to be sent by the client.")).times(3);
expect(backoffHandler.shouldRetryRequest(command, response)).andReturn(true).once();
expect(response.getStatusCode()).andReturn(408).once();

2
apis/openstack-keystone/src/test/java/org/jclouds/openstack/v2_0/internal/BaseOpenStackMockTest.java
View File

@ -108,7 +108,7 @@ public class BaseOpenStackMockTest<A extends Closeable> {
* access.json or accessRackspace.json) for the declared service
* endpoints.
*/
String newBody = urlTokenPattern.matcher(new String(response.getBody())).replaceAll(": \"" + url.toString());
String newBody = urlTokenPattern.matcher(new String(response.getBody().readByteArray())).replaceAll(": \"" + url.toString());
response = response.setBody(newBody);
}

10
apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/TempAuthMockTest.java
View File

@ -43,19 +43,19 @@ public class TempAuthMockTest {
private MockWebServer tempAuthServer;
public void testGenerateJWTRequest() throws Exception {
public void testGenerateJWTRequest() throws Exception {
tempAuthServer.enqueue(new MockResponse().setResponseCode(204)
.addHeader("X-Auth-Token", "token")
.addHeader("X-Storage-Url", "http://127.0.0.1:" + swiftServer.getPort()));
.addHeader("X-Storage-Url", swiftServer.getUrl("").toString()));
swiftServer.enqueue(new MockResponse().setBody("[{\"name\":\"test_container_1\",\"count\":2,\"bytes\":78}]"));
SwiftApi api = api("http://127.0.0.1:" + tempAuthServer.getPort());
SwiftApi api = api(tempAuthServer.getUrl("").toString());
// Region name is derived from the swift server host.
assertEquals(api.getConfiguredRegions(), ImmutableSet.of("127.0.0.1"));
assertEquals(api.getConfiguredRegions(), ImmutableSet.of(tempAuthServer.getHostName()));
assertTrue(api.getContainerApi("127.0.0.1").list().iterator().hasNext());
assertTrue(api.getContainerApi(tempAuthServer.getHostName()).list().iterator().hasNext());
RecordedRequest auth = tempAuthServer.takeRequest();
assertEquals(auth.getMethod(), "GET");

9
apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectApiMockTest.java
View File

@ -339,7 +339,14 @@ public class ObjectApiMockTest extends BaseOpenStackMockTest<SwiftApi> {
fail("testReplaceTimeout test should have failed with an HttpResponseException.");
} finally {
server.shutdown();
try {
server.shutdown();
} catch (IOException e) {
// MockWebServer 2.1.0 introduces an active wait for its executor termination.
// That active wait is a hardcoded value and throws an IOE if the executor has not
// terminated in that timeout. It is safe to ignore this exception as the functionality
// has been properly verified.
}
}
}

2
apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategyMockTest.java
View File

@ -127,7 +127,7 @@ public class SequentialMultipartUploadStrategyMockTest {
public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
MockResponse response = responseQueue.take();
if (response.getBody() != null) {
String newBody = new String(response.getBody()).replace("URL", url.get().toString());
String newBody = new String(response.getBody().readByteArray()).replace("URL", url.get().toString());
response = response.setBody(newBody);
}
return response;

2
core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java
View File

@ -134,6 +134,6 @@ public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandEx
protected abstract HttpResponse invoke(Q nativeRequest) throws IOException, InterruptedException;
protected abstract void cleanup(Q nativeResponse);
protected abstract void cleanup(Q nativeRequest);
}

11
core/src/test/java/org/jclouds/http/BaseHttpCommandExecutorServiceIntegrationTest.java
View File

@ -619,14 +619,21 @@ public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends Base
.method("GET")
.endpoint(server.getUrl("/").toURI())
.build());
InputStream is = response.getPayload().getInput();
InputStream is = response.getPayload().openStream();
long now = System.currentTimeMillis();
is.close();
long diff = System.currentTimeMillis() - now;
assertTrue(diff < timeoutMillis / 2, "expected " + diff + " to be less than " + (timeoutMillis / 2));
} finally {
closeQuietly(client);
server.shutdown();
try {
server.shutdown();
} catch (IOException ex) {
// MockWebServer 2.1.0 introduces an active wait for its executor termination.
// That active wait is a hardcoded value and throws an IOE if the executor has not
// terminated in that timeout. It is safe to ignore this exception (related to how
// throttling works internally in MWS), as the functionality has been properly verified.
}
}
}
}

19
core/src/test/java/org/jclouds/http/BaseMockWebServerTest.java
View File

@ -33,14 +33,11 @@ import org.jclouds.providers.AnonymousProviderMetadata;
import org.testng.annotations.BeforeClass;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.HttpHeaders;
import com.google.inject.Module;
import com.squareup.okhttp.internal.SslContextBuilder;
import com.squareup.okhttp.mockwebserver.Dispatcher;
import com.squareup.okhttp.mockwebserver.MockResponse;
import com.squareup.okhttp.mockwebserver.MockWebServer;
import com.squareup.okhttp.mockwebserver.QueueDispatcher;
import com.squareup.okhttp.mockwebserver.RecordedRequest;
/**
* Base class for integration tests that use {@link MockWebServer} to verify the
@ -61,19 +58,6 @@ public abstract class BaseMockWebServerTest {
}
}
protected static class GlobalChecksRequestDispatcher extends QueueDispatcher {
@Override
public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
MockResponse response = responseQueue.take();
if (!HttpRequest.NON_PAYLOAD_METHODS.contains(request.getMethod())
&& request.getHeader(HttpHeaders.CONTENT_LENGTH) == null) {
response.setResponseCode(500);
response.setBody("No content length!");
}
return response;
}
}
/**
* Creates a {@link MockWebServer} that uses the
* {@link GlobalChecksRequestDispatcher}.
@ -81,7 +65,6 @@ public abstract class BaseMockWebServerTest {
protected static MockWebServer mockWebServer(MockResponse... responses) throws IOException {
MockWebServer server = new MockWebServer();
server.play();
server.setDispatcher(new GlobalChecksRequestDispatcher());
for (MockResponse response : responses) {
server.enqueue(response);
}
@ -120,7 +103,7 @@ public abstract class BaseMockWebServerTest {
* tests.
* <p>
* Unless a concrete HTTP is required, subclasses may want to use the
* {@link JavaUrlHttpCommandExecutorServiceModule}.
* {@link org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule}.
*/
protected abstract Module createConnectionModule();

2
drivers/okhttp/README.md
View File

@ -3,8 +3,6 @@ jclouds OkHttp driver
A driver to use the OkHttp (http://square.github.io/okhttp/) client as an HTTP library in jclouds.
This driver adds support for use of modern HTTP verbs such as PATCH in providers and APIs, and also supports SPDY.
To use the driver, you just need to include the `OkHttpCommandExecutorServiceModule` when creating
the context:

168
drivers/okhttp/src/main/java/org/jclouds/http/okhttp/OkHttpCommandExecutorService.java
View File

@ -16,78 +16,162 @@
*/
package org.jclouds.http.okhttp;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.net.HttpHeaders.ACCEPT;
import static com.google.common.net.HttpHeaders.USER_AGENT;
import static org.jclouds.http.HttpUtils.filterOutContentHeaders;
import static org.jclouds.io.Payloads.newInputStreamPayload;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.Proxy;
import java.net.URI;
import java.net.URL;
import java.util.Map;
import javax.inject.Named;
import javax.inject.Singleton;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import okio.BufferedSink;
import okio.Okio;
import okio.Source;
import org.jclouds.JcloudsVersion;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.IOExceptionRetryHandler;
import org.jclouds.http.handlers.DelegatingErrorHandler;
import org.jclouds.http.handlers.DelegatingRetryHandler;
import org.jclouds.http.internal.BaseHttpCommandExecutorService;
import org.jclouds.http.internal.HttpWire;
import org.jclouds.http.internal.JavaUrlHttpCommandExecutorService;
import org.jclouds.io.ContentMetadataCodec;
import org.jclouds.io.MutableContentMetadata;
import org.jclouds.io.Payload;
import com.google.common.base.Function;
import com.google.common.base.Supplier;
import com.google.common.net.HttpHeaders;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableMultimap.Builder;
import com.google.inject.Inject;
import com.squareup.okhttp.Headers;
import com.squareup.okhttp.MediaType;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.RequestBody;
import com.squareup.okhttp.Response;
/**
* Implementation of the <code>HttpCommandExecutorService</code> that uses the
* OkHttp client to support modern HTTP methods such as PATCH.
*/
@Singleton
public class OkHttpCommandExecutorService extends JavaUrlHttpCommandExecutorService {
public final class OkHttpCommandExecutorService extends BaseHttpCommandExecutorService<Request> {
private static final String DEFAULT_USER_AGENT = String.format("jclouds/%s java/%s", JcloudsVersion.get(),
System.getProperty("java.version"));
private final Function<URI, Proxy> proxyForURI;
private final OkHttpClient globalClient;
@Inject
public OkHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
OkHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider, Function<URI, Proxy> proxyForURI)
throws SecurityException, NoSuchFieldException {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire, verifier,
untrustedSSLContextProvider, proxyForURI);
DelegatingErrorHandler errorHandler, HttpWire wire, Function<URI, Proxy> proxyForURI, OkHttpClient okHttpClient) {
super(utils, contentMetadataCodec, retryHandler, ioRetryHandler, errorHandler, wire);
this.proxyForURI = proxyForURI;
this.globalClient = okHttpClient;
}
@Override
protected HttpURLConnection initConnection(HttpRequest request) throws IOException {
OkHttpClient client = new OkHttpClient();
URL url = request.getEndpoint().toURL();
client.setProxy(proxyForURI.apply(request.getEndpoint()));
if (url.getProtocol().equalsIgnoreCase("https")) {
if (utils.relaxHostname()) {
client.setHostnameVerifier(verifier);
}
if (sslContextSupplier != null) {
// used for providers which e.g. use certs for authentication (like
// FGCP) Provider provides SSLContext impl (which inits context with
// key manager)
client.setSslSocketFactory(sslContextSupplier.get().getSocketFactory());
} else if (utils.trustAllCerts()) {
client.setSslSocketFactory(untrustedSSLContextProvider.get().getSocketFactory());
protected Request convert(HttpRequest request) throws IOException, InterruptedException {
Request.Builder builder = new Request.Builder();
builder.url(request.getEndpoint().toString());
populateHeaders(request, builder);
RequestBody body = null;
Payload payload = request.getPayload();
if (payload != null) {
Long length = checkNotNull(payload.getContentMetadata().getContentLength(), "payload.getContentLength");
if (length > 0) {
body = generateRequestBody(request, payload);
}
}
return client.open(url);
builder.method(request.getMethod(), body);
return builder.build();
}
@Override
protected void configureRequestHeaders(HttpURLConnection connection, HttpRequest request) {
super.configureRequestHeaders(connection, request);
protected void populateHeaders(HttpRequest request, Request.Builder builder) {
// OkHttp does not set the Accept header if not present in the request.
// Make sure we send a flexible one.
if (request.getFirstHeaderOrNull(HttpHeaders.ACCEPT) == null) {
connection.setRequestProperty(HttpHeaders.ACCEPT, "*/*");
if (request.getFirstHeaderOrNull(ACCEPT) == null) {
builder.addHeader(ACCEPT, "*/*");
}
if (request.getFirstHeaderOrNull(USER_AGENT) == null) {
builder.addHeader(USER_AGENT, DEFAULT_USER_AGENT);
}
for (Map.Entry<String, String> entry : request.getHeaders().entries()) {
builder.addHeader(entry.getKey(), entry.getValue());
}
if (request.getPayload() != null) {
MutableContentMetadata md = request.getPayload().getContentMetadata();
for (Map.Entry<String, String> entry : contentMetadataCodec.toHeaders(md).entries()) {
builder.addHeader(entry.getKey(), entry.getValue());
}
}
}
protected RequestBody generateRequestBody(final HttpRequest request, final Payload payload) {
checkNotNull(payload.getContentMetadata().getContentType(), "payload.getContentType");
return new RequestBody() {
@Override
public void writeTo(BufferedSink sink) throws IOException {
Source source = Okio.source(payload.openStream());
try {
sink.writeAll(source);
} catch (IOException ex) {
logger.error(ex, "error writing bytes to %s", request.getEndpoint());
throw ex;
} finally {
source.close();
}
}
@Override
public MediaType contentType() {
return MediaType.parse(payload.getContentMetadata().getContentType());
}
};
}
@Override
protected HttpResponse invoke(Request nativeRequest) throws IOException, InterruptedException {
OkHttpClient requestScopedClient = globalClient.clone();
requestScopedClient.setProxy(proxyForURI.apply(nativeRequest.uri()));
Response response = requestScopedClient.newCall(nativeRequest).execute();
HttpResponse.Builder<?> builder = HttpResponse.builder();
builder.statusCode(response.code());
builder.message(response.message());
Builder<String, String> headerBuilder = ImmutableMultimap.builder();
Headers responseHeaders = response.headers();
for (String header : responseHeaders.names()) {
headerBuilder.putAll(header, responseHeaders.values(header));
}
ImmutableMultimap<String, String> headers = headerBuilder.build();
if (response.code() == 204 && response.body() != null) {
response.body().close();
} else {
Payload payload = newInputStreamPayload(response.body().byteStream());
contentMetadataCodec.fromHeaders(payload.getContentMetadata(), headers);
builder.payload(payload);
}
builder.headers(filterOutContentHeaders(headers));
return builder.build();
}
@Override
protected void cleanup(Request nativeResponse) {
}
}

57
drivers/okhttp/src/main/java/org/jclouds/http/okhttp/config/OkHttpCommandExecutorServiceModule.java
View File

@ -16,17 +16,28 @@
*/
package org.jclouds.http.okhttp.config;
import java.util.concurrent.TimeUnit;
import javax.inject.Named;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import org.jclouds.http.HttpCommandExecutorService;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.config.ConfiguresHttpCommandExecutorService;
import org.jclouds.http.config.SSLModule;
import org.jclouds.http.okhttp.OkHttpCommandExecutorService;
import com.google.common.base.Supplier;
import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Scopes;
import com.squareup.okhttp.OkHttpClient;
/**
* Configures the {@link OkHttpCommandExecutorService}.
*
*
* Note that this uses threads.
*/
@ConfiguresHttpCommandExecutorService
@ -36,6 +47,50 @@ public class OkHttpCommandExecutorServiceModule extends AbstractModule {
protected void configure() {
install(new SSLModule());
bind(HttpCommandExecutorService.class).to(OkHttpCommandExecutorService.class).in(Scopes.SINGLETON);
bind(OkHttpClient.class).toProvider(OkHttpClientProvider.class).in(Scopes.SINGLETON);
}
private static final class OkHttpClientProvider implements Provider<OkHttpClient> {
@Inject(optional = true)
private Supplier<SSLContext> sslContextSupplier;
private final HostnameVerifier verifier;
private final Supplier<SSLContext> untrustedSSLContextProvider;
private final HttpUtils utils;
@Inject
OkHttpClientProvider(HttpUtils utils, @Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider) {
this.utils = utils;
this.verifier = verifier;
this.untrustedSSLContextProvider = untrustedSSLContextProvider;
}
@Override
public OkHttpClient get() {
OkHttpClient client = new OkHttpClient();
client.setConnectTimeout(utils.getConnectionTimeout(), TimeUnit.MILLISECONDS);
client.setReadTimeout(utils.getSocketOpenTimeout(), TimeUnit.MILLISECONDS);
// do not follow redirects since https redirects don't work properly
// ex. Caused by: java.io.IOException: HTTPS hostname wrong: should be
// <adriancole.s3int0.s3-external-3.amazonaws.com>
client.setFollowRedirects(false);
if (utils.relaxHostname()) {
client.setHostnameVerifier(verifier);
}
if (sslContextSupplier != null) {
// used for providers which e.g. use certs for authentication (like
// FGCP) Provider provides SSLContext impl (which inits context with
// key manager)
client.setSslSocketFactory(sslContextSupplier.get().getSocketFactory());
} else if (utils.trustAllCerts()) {
client.setSslSocketFactory(untrustedSSLContextProvider.get().getSocketFactory());
}
return client;
}
}
}

2
project/pom.xml
View File

@ -206,7 +206,7 @@
<maven.site.url.base>gitsite:git@github.com/jclouds/jclouds-maven-site.git</maven.site.url.base>
<clojure.version>1.3.0</clojure.version>
<guava.version>16.0.1</guava.version>
<okhttp.version>1.6.0</okhttp.version>
<okhttp.version>2.1.0</okhttp.version>
<surefire.version>2.17</surefire.version>
<assertj-core.version>1.6.1</assertj-core.version>
<assertj-guava.version>1.2.0</assertj-guava.version>

9
providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/internal/BaseAWSEC2ApiMockTest.java
View File

@ -75,10 +75,11 @@ public class BaseAWSEC2ApiMockTest {
}
protected ContextBuilder builder(Properties overrides) {
MockWebServer defaultServer = regionToServers.get(DEFAULT_REGION);
overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, "1");
return ContextBuilder.newBuilder(new AWSEC2ProviderMetadata())
.credentials(ACCESS_KEY, SECRET_KEY)
.endpoint("http://localhost:" + regionToServers.get(DEFAULT_REGION).getPort())
.endpoint(defaultServer.getUrl("").toString())
.overrides(overrides)
.modules(modules);
}
@ -102,7 +103,8 @@ public class BaseAWSEC2ApiMockTest {
@Override public String region(String host) {
for (Map.Entry<String, MockWebServer> regionToServer : regionToServers.entrySet()) {
if (host.equals("localhost:" + regionToServer.getValue().getPort())) {
MockWebServer server = regionToServer.getValue();
if (host.equals(server.getHostName() + ":" + regionToServer.getValue().getPort())) {
return regionToServer.getKey();
}
}
@ -141,7 +143,8 @@ public class BaseAWSEC2ApiMockTest {
server.play();
regionToServers.put(region, server);
}
String regionEndpoint = "http://localhost:" + regionToServers.get(region).getPort();
MockWebServer server = regionToServers.get(region);
String regionEndpoint = server.getUrl("").toString();
describeRegionsResponse.append("<regionEndpoint>").append(regionEndpoint).append("</regionEndpoint>");
describeRegionsResponse.append("</item>");
}

2
providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/internal/BaseHPCloudObjectStorageMockTest.java
View File

@ -70,7 +70,7 @@ public class BaseHPCloudObjectStorageMockTest {
public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
MockResponse response = responseQueue.take();
if (response.getBody() != null) {
String newBody = new String(response.getBody()).replace(":\"URL", ":\"" + url.toString());
String newBody = new String(response.getBody().readByteArray()).replace(":\"URL", ":\"" + url.toString());
response = response.setBody(newBody);
}
return response;