diff --git a/apis/openstack-keystone/pom.xml b/apis/openstack-keystone/pom.xml index d61a150c85..4f4abf4b40 100644 --- a/apis/openstack-keystone/pom.xml +++ b/apis/openstack-keystone/pom.xml @@ -76,7 +76,6 @@ com.squareup.okhttp mockwebserver - 1.2.1 test diff --git a/core/pom.xml b/core/pom.xml index cf07cea014..967a3ad7d6 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -82,6 +82,11 @@ jetty-security test + + com.squareup.okhttp + mockwebserver + test + com.google.code.gson gson diff --git a/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java b/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java index 55595d7291..9c7640328b 100644 --- a/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java +++ b/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java @@ -19,12 +19,12 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Throwables.propagate; import static com.google.common.io.ByteStreams.toByteArray; +import static com.google.common.io.Closeables.close; import static com.google.common.net.HttpHeaders.CONTENT_LENGTH; import static com.google.common.net.HttpHeaders.HOST; import static com.google.common.net.HttpHeaders.USER_AGENT; import static org.jclouds.http.HttpUtils.filterOutContentHeaders; import static org.jclouds.io.Payloads.newInputStreamPayload; -import static org.jclouds.util.Closeables2.closeQuietly; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -46,6 +46,7 @@ import javax.net.ssl.SSLContext; import org.jclouds.Constants; import org.jclouds.JcloudsVersion; +import org.jclouds.http.HttpCommandExecutorService; import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpUtils; @@ -75,11 +76,11 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe public static final String DEFAULT_USER_AGENT = String.format("jclouds/%s java/%s", JcloudsVersion.get(), System .getProperty("java.version")); - private final Supplier untrustedSSLContextProvider; - private final Function proxyForURI; - private final HostnameVerifier verifier; + protected final Supplier untrustedSSLContextProvider; + protected final Function proxyForURI; + protected final HostnameVerifier verifier; @Inject(optional = true) - Supplier sslContextSupplier; + protected Supplier sslContextSupplier; @Inject public JavaUrlHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec, @@ -105,13 +106,13 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe } catch (IOException e) { in = bufferAndCloseStream(connection.getErrorStream()); } catch (RuntimeException e) { - closeQuietly(in); + close(in, true); throw propagate(e); } int responseCode = connection.getResponseCode(); if (responseCode == 204) { - closeQuietly(in); + close(in, true); in = null; } builder.statusCode(responseCode); @@ -141,7 +142,7 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe in = new ByteArrayInputStream(toByteArray(inputStream)); } } finally { - closeQuietly(inputStream); + close(inputStream, true); } return in; } @@ -149,21 +150,8 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe @Override protected HttpURLConnection convert(HttpRequest request) throws IOException, InterruptedException { boolean chunked = "chunked".equals(request.getFirstHeaderOrNull("Transfer-Encoding")); - URL url = request.getEndpoint().toURL(); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxyForURI.apply(request.getEndpoint())); - if (connection instanceof HttpsURLConnection) { - HttpsURLConnection sslCon = (HttpsURLConnection) connection; - if (utils.relaxHostname()) - sslCon.setHostnameVerifier(verifier); - if (sslContextSupplier != null) { - // used for providers which e.g. use certs for authentication (like FGCP) - // Provider provides SSLContext impl (which inits context with key manager) - sslCon.setSSLSocketFactory(sslContextSupplier.get().getSocketFactory()); - } else if (utils.trustAllCerts()) { - sslCon.setSSLSocketFactory(untrustedSSLContextProvider.get().getSocketFactory()); - } - } + HttpURLConnection connection = initConnection(request); connection.setConnectTimeout(utils.getConnectionTimeout()); connection.setReadTimeout(utils.getSocketOpenTimeout()); connection.setAllowUserInteraction(false); @@ -173,10 +161,7 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe connection.setInstanceFollowRedirects(false); setRequestMethodBypassingJREMethodLimitation(connection, request.getMethod()); - - for (Map.Entry entry : request.getHeaders().entries()) { - connection.setRequestProperty(entry.getKey(), entry.getValue()); - } + configureRequestHeaders(connection, request); String host = request.getEndpoint().getHost(); if (request.getEndpoint().getPort() != -1) { @@ -216,6 +201,36 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe return connection; } + /** + * Creates and initializes the connection. + */ + protected HttpURLConnection initConnection(HttpRequest request) throws IOException { + URL url = request.getEndpoint().toURL(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxyForURI.apply(request.getEndpoint())); + if (connection instanceof HttpsURLConnection) { + HttpsURLConnection sslCon = (HttpsURLConnection) connection; + if (utils.relaxHostname()) + sslCon.setHostnameVerifier(verifier); + if (sslContextSupplier != null) { + // used for providers which e.g. use certs for authentication (like FGCP) + // Provider provides SSLContext impl (which inits context with key manager) + sslCon.setSSLSocketFactory(sslContextSupplier.get().getSocketFactory()); + } else if (utils.trustAllCerts()) { + sslCon.setSSLSocketFactory(untrustedSSLContextProvider.get().getSocketFactory()); + } + } + return connection; + } + + /** + * Configure the HTTP request headers in the connection. + */ + protected void configureRequestHeaders(HttpURLConnection connection, HttpRequest request) { + for (Map.Entry entry : request.getHeaders().entries()) { + connection.setRequestProperty(entry.getKey(), entry.getValue()); + } + } + /** * Workaround for a bug in HttpURLConnection.setRequestMethod(String) * The implementation of Sun Microsystems is throwing a ProtocolException diff --git a/core/src/test/java/org/jclouds/http/BackoffLimitedRetryJavaTest.java b/core/src/test/java/org/jclouds/http/BackoffLimitedRetryJavaTest.java index 3fccbba958..d46adecbbe 100644 --- a/core/src/test/java/org/jclouds/http/BackoffLimitedRetryJavaTest.java +++ b/core/src/test/java/org/jclouds/http/BackoffLimitedRetryJavaTest.java @@ -16,38 +16,37 @@ */ package org.jclouds.http; +import static com.google.common.io.Closeables.close; +import static org.jclouds.Constants.PROPERTY_MAX_RETRIES; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; -import java.io.IOException; import java.util.Properties; -import java.util.concurrent.ExecutionException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.eclipse.jetty.server.Request; import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.testng.annotations.Test; import com.google.inject.Module; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; /** * Tests the retry behavior of the default {@link RetryHandler} implementation - * {@link BackoffLimitedRetryHandler} to ensure that retries up to the default limit succeed. - * - * TODO: Should either explicitly set retry limit or get it from Guice, rather than assuming it's 5. + * {@link BackoffLimitedRetryHandler} to ensure that retries up to the default + * limit succeed. * * @author James Murty + * @author Ignasi Barrera */ -@Test(sequential = true) -public class BackoffLimitedRetryJavaTest extends BaseJettyTest { - private int beginToFailOnRequestNumber = 0; - private int endFailuresOnRequestNumber = 0; - private int requestCount = 0; +@Test(groups = "integration") +public class BackoffLimitedRetryJavaTest extends BaseMockWebServerTest { + + private final int maxRetries = 5; @Override - protected void addConnectionProperties(Properties props) { + protected void addOverrideProperties(Properties props) { + props.setProperty(PROPERTY_MAX_RETRIES, "" + maxRetries); } @Override @@ -55,84 +54,98 @@ public class BackoffLimitedRetryJavaTest extends BaseJettyTest { return new JavaUrlHttpCommandExecutorServiceModule(); } - @Override - protected boolean failEveryTenRequests(HttpServletRequest request, HttpServletResponse response) - throws IOException { - requestCount++; - boolean shouldFail = requestCount >= beginToFailOnRequestNumber - && requestCount <= endFailuresOnRequestNumber; - if (shouldFail) { - response.sendError(500); - ((Request) request).setHandled(true); - return true; - } else { - return false; - } - } - - protected String submitGetRequest() throws InterruptedException, ExecutionException { - return client.download(""); + protected IntegrationTestClient client(String url) { + return api(IntegrationTestClient.class, url); } @Test - public void testNoRetriesSuccessful() throws InterruptedException, ExecutionException { - beginToFailOnRequestNumber = 1; - endFailuresOnRequestNumber = 1; - requestCount = 0; - - assertEquals(submitGetRequest().trim(), XML); - } - - @Test - public void testSingleRetrySuccessful() throws InterruptedException, ExecutionException { - beginToFailOnRequestNumber = 0; - endFailuresOnRequestNumber = 1; - requestCount = 0; - - assertEquals(submitGetRequest().trim(), XML); - } - - @Test - public void testMaximumRetriesSuccessful() throws InterruptedException, ExecutionException { - beginToFailOnRequestNumber = 0; - endFailuresOnRequestNumber = 5; - requestCount = 0; - - assertEquals(submitGetRequest().trim(), XML); - } - - @Test - public void testMaximumRetriesExceeded() throws InterruptedException, ExecutionException { - beginToFailOnRequestNumber = 0; - endFailuresOnRequestNumber = 6; - requestCount = 0; - + public void testNoRetriesSuccessful() throws Exception { + MockWebServer server = mockWebServer(new MockResponse()); + IntegrationTestClient client = client(server.getUrl("/").toString()); try { - submitGetRequest(); - fail("Request should not succeed within " + endFailuresOnRequestNumber + " requests"); - } catch (HttpResponseException e) { - assertEquals(e.getResponse().getStatusCode(), 500); + client.download(""); + assertEquals(server.getRequestCount(), 1); + } finally { + close(client, true); + server.shutdown(); } } @Test - public void testInterleavedSuccessesAndFailures() throws InterruptedException, - ExecutionException { - beginToFailOnRequestNumber = 3; - endFailuresOnRequestNumber = 3 + 5; // Force third request to fail completely - requestCount = 0; - - assertEquals(submitGetRequest().trim(), XML); - assertEquals(submitGetRequest().trim(), XML); - + public void testSingleRetrySuccessful() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(500), new MockResponse()); + IntegrationTestClient client = client(server.getUrl("/").toString()); try { - submitGetRequest(); - fail("Third request should not succeed by attempt number " + requestCount); - } catch (HttpResponseException e) { - assertEquals(e.getResponse().getStatusCode(), 500); + client.download(""); + assertEquals(server.getRequestCount(), 2); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testMaximumRetriesSuccessful() throws Exception { + MockWebServer server = mockWebServer(); + for (int i = 0; i < maxRetries - 1; i++) { + server.enqueue(new MockResponse().setResponseCode(500)); + } + server.enqueue(new MockResponse()); + + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + client.download(""); + assertEquals(server.getRequestCount(), maxRetries); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testMaximumRetriesExceeded() throws Exception { + MockWebServer server = mockWebServer(); + for (int i = 0; i <= maxRetries; i++) { + server.enqueue(new MockResponse().setResponseCode(500)); } - assertEquals(submitGetRequest().trim(), XML); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + + client.download(""); + fail("Request should not succeed within " + maxRetries + " requests"); + } catch (HttpResponseException ex) { + assertEquals(ex.getResponse().getStatusCode(), 500); + assertEquals(server.getRequestCount(), maxRetries + 1); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testInterleavedSuccessesAndFailures() throws Exception { + MockWebServer server = mockWebServer(new MockResponse(), new MockResponse()); + for (int i = 0; i <= maxRetries; i++) { + server.enqueue(new MockResponse().setResponseCode(500)); + } + + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + client.download(""); + client.download(""); + + try { + client.download(""); + fail("Request should not succeed within " + maxRetries + " requests"); + } catch (HttpResponseException ex) { + assertEquals(ex.getResponse().getStatusCode(), 500); + assertEquals(server.getRequestCount(), maxRetries + 3); + } + } finally { + close(client, true); + server.shutdown(); + } } } diff --git a/core/src/test/java/org/jclouds/http/BaseHttpCommandExecutorServiceIntegrationTest.java b/core/src/test/java/org/jclouds/http/BaseHttpCommandExecutorServiceIntegrationTest.java index 40855c2a21..dbd3fcfcc8 100644 --- a/core/src/test/java/org/jclouds/http/BaseHttpCommandExecutorServiceIntegrationTest.java +++ b/core/src/test/java/org/jclouds/http/BaseHttpCommandExecutorServiceIntegrationTest.java @@ -18,72 +18,148 @@ package org.jclouds.http; import static com.google.common.hash.Hashing.md5; import static com.google.common.io.BaseEncoding.base64; -import static java.lang.String.format; +import static com.google.common.io.ByteStreams.join; +import static com.google.common.io.ByteStreams.newInputStreamSupplier; +import static com.google.common.io.ByteStreams.toByteArray; +import static com.google.common.io.Closeables.close; +import static com.google.common.io.Files.asByteSource; import static org.jclouds.http.options.GetOptions.Builder.tail; import static org.jclouds.io.ByteSources.asByteSource; -import static org.jclouds.io.Payloads.newFilePayload; -import static org.jclouds.io.Payloads.newStringPayload; -import static org.jclouds.util.Closeables2.closeQuietly; -import static org.jclouds.util.Throwables2.getFirstThrowableOfType; +import static org.jclouds.io.Payloads.newByteSourcePayload; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Writer; -import java.net.MalformedURLException; -import java.net.URI; +import java.net.URLDecoder; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.Random; +import java.util.zip.GZIPInputStream; import org.jclouds.io.Payload; import org.jclouds.util.Strings2; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import com.google.common.base.Charsets; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; +import com.google.common.io.ByteSource; import com.google.common.io.CharSink; import com.google.common.io.Files; +import com.google.common.io.InputSupplier; +import com.squareup.okhttp.mockwebserver.Dispatcher; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; +import com.squareup.okhttp.mockwebserver.RecordedRequest; /** - * Tests for functionality all {@link HttpCommandExecutorService http executor - * services} must express. These tests will operate against an in-memory http + * Tests for functionality all {@link HttpCommandExecutorService} http executor + * services must express. These tests will operate against an in-memory http * engine, so as to ensure end-to-end functionality works. * * @author Adrian Cole + * @author Ignasi Barrera */ -@Test(threadPoolSize = 10, groups = "integration") -public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends BaseJettyTest { +@Test(groups = "integration") +public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends BaseMockWebServerTest { - @Test(invocationCount = 25, timeOut = 5000) - public void testRequestFilter() { - assertEquals(client.downloadFilter("", "filterme").trim(), "test"); + private static final String XML = "whoppers"; + private static final String XML2 = "chubbs"; + + private String constitutionsMd5; + private long constitutionsLength; + private InputSupplier oneHundredOneConstitutions; + + @BeforeClass(groups = "integration") + public void setup() throws IOException { + oneHundredOneConstitutions = getTestDataSupplier(); + constitutionsMd5 = base64().encode(asByteSource(oneHundredOneConstitutions.getInput()).hash(md5()).asBytes()); } - @Test(invocationCount = 5, timeOut = 5000) - public void testGetStringWithHeader() { - assertEquals(client.download("", "test").trim(), "test"); + protected IntegrationTestClient client(String url) { + return api(IntegrationTestClient.class, url); } - @Test(invocationCount = 1, timeOut = 5000) - public void testAlternateMethod() { - assertEquals(client.rowdy("").trim(), XML); + @Test + public void testRequestFilter() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody("test")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.downloadFilter("", "filterme"); + + RecordedRequest request = server.takeRequest(); + assertEquals(request.getHeader("filterme"), "filterme"); + assertEquals(request.getHeader("test"), "test"); + assertEquals(result, "test"); + } finally { + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testGetString() { - assertEquals(client.download("").trim(), XML); + @Test + public void testGetStringWithHeader() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody("test")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.download("", "test"); + + RecordedRequest request = server.takeRequest(); + assertEquals(request.getHeader("test"), "test"); + assertEquals(result, "test"); + } finally { + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testGetStringViaRequest() throws IOException { - HttpResponse getStringResponse = client.invoke(HttpRequest.builder().method("GET") - .endpoint(format("http://localhost:%d/objects/", testPort)).build()); - assertEquals(Strings2.toString(getStringResponse.getPayload()).trim(), XML); + @Test + public void testGetString() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody(XML)); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + assertEquals(client.download(""), XML); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testGetStringIsRetriedOnFailure() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(500), new MockResponse().setBody(XML)); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.download(""); + assertEquals(server.getRequestCount(), 2); + assertEquals(result, XML); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testGetStringViaRequest() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody(XML)); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + HttpResponse getStringResponse = client.invoke(HttpRequest.builder().method("GET") + .endpoint(server.getUrl("/objects").toString()).build()); + assertEquals(Strings2.toString(getStringResponse.getPayload()).trim(), XML); + } finally { + close(client, true); + server.shutdown(); + } } @DataProvider(name = "gets") @@ -91,68 +167,123 @@ public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends Base return new Object[][] { { "object" }, { "/path" }, { "sp ace" }, { "unic₪de" }, { "qu?stion" } }; } - @Test(invocationCount = 5, timeOut = 5000, dataProvider = "gets") - public void testGetStringSynch(String uri) { - assertEquals(client.synch(uri).trim(), XML); - } - - @Test(invocationCount = 5, timeOut = 5000) - public void testGetException() { - assertEquals(client.downloadException("", tail(1)).trim(), "foo"); - } - - @Test(invocationCount = 5, timeOut = 5000) - public void testGetSynchException() { - assertEquals(client.synchException("", "").trim(), "foo"); - } - - @Test(invocationCount = 5, timeOut = 5000) - public void testGetStringRedirect() { - assertEquals(client.download("redirect").trim(), XML2); - } - - @Test(invocationCount = 100, timeOut = 5000) - public void testGetBigFile() throws IOException { - InputStream input = getConsitution(); + @Test(dataProvider = "gets") + public void testGetStringSynch(String uri) throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody(XML)); + IntegrationTestClient client = client(server.getUrl("/").toString()); try { - assertValidMd5(input); - } catch (RuntimeException e) { - closeQuietly(input); - // since we are parsing client side, and not through a response - // handler, the user must retry directly. In this case, we are assuming - // lightning doesn't strike twice in the same spot. - if (getFirstThrowableOfType(e, IOException.class) != null) { - input = getConsitution(); - assertValidMd5(input); - } + String result = client.synch(uri); + RecordedRequest request = server.takeRequest(); + assertTrue(URLDecoder.decode(request.getPath(), "UTF-8").endsWith(uri)); + assertEquals(result, XML); } finally { - closeQuietly(input); + close(client, true); + server.shutdown(); } } - private void assertValidMd5(final InputStream input) throws IOException { + @Test + public void testGetException() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(404)); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.downloadException("", tail(1)); + assertEquals(result, "foo"); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testGetSynchException() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(404)); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.synchException("", ""); + assertEquals(result, "foo"); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testGetStringRedirect() throws Exception { + MockWebServer redirectTarget = mockWebServer(new MockResponse().setBody(XML2)); + redirectTarget.useHttps(sslContext.getSocketFactory(), false); + MockWebServer server = mockWebServer((new MockResponse().setResponseCode(302).setHeader("Location", + redirectTarget.getUrl("/").toString()))); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.download("redirect"); + assertEquals(result, XML2); + assertEquals(server.getRequestCount(), 1); + assertEquals(redirectTarget.getRequestCount(), 1); + } finally { + close(client, true); + redirectTarget.shutdown(); + server.shutdown(); + } + } + + @Test + public void testGetBigFile() throws Exception { + MockResponse response = new MockResponse().addHeader("Content-MD5", constitutionsMd5) + .addHeader("Content-type", "text/plain") + .setBody(oneHundredOneConstitutions.getInput(), constitutionsLength); + + MockWebServer server = mockWebServer(response, response); + InputStream input = server.getUrl("/101constitutions").openStream(); + + try { + assertValidMd5(input, constitutionsMd5); + } catch (RuntimeException e) { + } finally { + close(input, true); + } + } + + private void assertValidMd5(final InputStream input, String md5) throws IOException { assertEquals(base64().encode(asByteSource(input).hash(md5()).asBytes()), md5); } - private InputStream getConsitution() throws MalformedURLException, IOException { - URI constitutionUri = URI.create(format("http://localhost:%d/101constitutions", testPort)); - return constitutionUri.toURL().openStream(); + private static class MD5CheckDispatcher extends Dispatcher { + + @Override + public MockResponse dispatch(RecordedRequest request) throws InterruptedException { + try { + MockResponse response = new MockResponse(); + String expectedMd5 = request.getHeader("Content-MD5"); + ByteSource body = ByteSource.wrap(request.getBody()); + String realMd5FromRequest = base64().encode(body.hash(md5()).asBytes()); + boolean matched = expectedMd5.equals(realMd5FromRequest); + if (matched) { + response.addHeader("x-Content-MD5", realMd5FromRequest); + } else { + response.setResponseCode(500); + } + return response; + } catch (IOException ex) { + throw Throwables.propagate(ex); + } + } + } - /** - * Tests sending a big file to the server. Note: this is a heavy test, takes - * several minutes to finish. - */ - @Test(invocationCount = 1) - public void testUploadBigFile() throws IOException { - String filename = "jclouds"; + @Test + public void testUploadBigFile() throws Exception { + MockWebServer server = mockWebServer(new MD5CheckDispatcher()); + IntegrationTestClient client = client(server.getUrl("/").toString()); + File f = null; + Payload payload = null; + try { - // create a file, twice big as free heap memory - f = File.createTempFile(filename, "tmp"); + f = File.createTempFile("jclouds", "tmp"); f.deleteOnExit(); - long length = (long) (Runtime.getRuntime().freeMemory() * 1.1); - + long length = (new Random().nextInt(32) + 1) * 1024 * 1024; + MessageDigest digester = md5Digest(); CharSink fileSink = Files.asCharSink(f, Charsets.UTF_8); @@ -165,18 +296,28 @@ public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends Base } out.flush(); } finally { - closeQuietly(out); + close(out, true); } - Payload payload = newFilePayload(f); + payload = newByteSourcePayload(asByteSource(f)); byte[] digest = digester.digest(); + String strDigest = base64().encode(digest); + payload.getContentMetadata().setContentMD5(digest); + payload.getContentMetadata().setContentLength(f.length()); Multimap headers = client.postPayloadAndReturnHeaders("", payload); - assertEquals(headers.get("x-Content-MD5"), ImmutableList.of(base64().encode(digest))); - payload.release(); + RecordedRequest request = server.takeRequest(); + assertEquals(request.getHeader("Content-MD5"), strDigest); + assertEquals(headers.get("x-Content-MD5"), ImmutableList.of(strDigest)); } finally { - if (f != null && f.exists()) + if (payload != null) { + payload.release(); + } + if (f != null && f.exists()) { f.delete(); + } + close(client, true); + server.shutdown(); } } @@ -188,77 +329,334 @@ public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends Base } } - @Test(invocationCount = 5, timeOut = 5000) - public void testPost() { - assertEquals(client.post("", "foo").trim(), "fooPOST"); + @Test + public void testPost() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody("fooPOST")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.post("", "foo"); + // Verify that the body is properly populated + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + assertEquals(result, "fooPOST"); + } finally { + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 1, timeOut = 5000) - public void testPostAsInputStream() { - AtomicInteger postFailures = new AtomicInteger(); - for (int i = 0; i < 5; i++) - try { - assertEquals(client.postAsInputStream("", "foo").trim(), "fooPOST"); - } catch (Exception e) { - postFailures.incrementAndGet(); + @Test + public void testZeroLengthPost() throws Exception { + MockWebServer server = mockWebServer(new MockResponse()); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + client.postNothing(""); + assertEquals(server.getRequestCount(), 1); + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), ""); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testPostIsRetriedOnFailure() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(500), + new MockResponse().setBody("fooPOST")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.post("", "foo"); + assertEquals(server.getRequestCount(), 2); + assertEquals(result, "fooPOST"); + // Verify that the body was properly sent in the two requests + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testPostRedirect() throws Exception { + MockWebServer redirectTarget = mockWebServer(new MockResponse().setBody("fooPOSTREDIRECT")); + redirectTarget.useHttps(sslContext.getSocketFactory(), false); + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(302).setHeader("Location", + redirectTarget.getUrl("/").toString())); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.post("redirect", "foo"); + assertEquals(result, "fooPOSTREDIRECT"); + assertEquals(server.getRequestCount(), 1); + assertEquals(redirectTarget.getRequestCount(), 1); + // Verify that the body was populated after the redirect + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + request = redirectTarget.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + } finally { + close(client, true); + redirectTarget.shutdown(); + server.shutdown(); + } + } + + @Test + public void testPostAsInputStream() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody("fooPOST")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.postAsInputStream("", "foo"); + // Verify that the body is properly populated + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + assertEquals(result, "fooPOST"); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testPostAsInputStreamDoesNotRetryOnFailure() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(500), new MockResponse()); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + client.postAsInputStream("", "foo"); + fail("Request should have thrown an exception after a server error"); + } catch (Exception expected) { + assertEquals(server.getRequestCount(), 1); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testPostBinder() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody("fooPOSTJSON")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.postJson("", "foo"); + // Verify that the body is properly populated + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "{\"key\":\"foo\"}"); + assertEquals(result, "fooPOSTJSON"); + } finally { + close(client, true); + server.shutdown(); + } + } + + @Test + public void testPostContentDisposition() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().addHeader("x-Content-Disposition", + "attachment; filename=photo.jpg")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + Payload payload = null; + try { + ByteSource body = ByteSource.wrap("foo".getBytes()); + payload = newByteSourcePayload(body); + payload.getContentMetadata().setContentDisposition("attachment; filename=photo.jpg"); + payload.getContentMetadata().setContentLength(body.size()); + Multimap headers = client.postPayloadAndReturnHeaders("", payload); + RecordedRequest request = server.takeRequest(); + assertEquals(request.getHeader("Content-Disposition"), "attachment; filename=photo.jpg"); + assertEquals(headers.get("x-Content-Disposition"), ImmutableList.of("attachment; filename=photo.jpg")); + } finally { + if (payload != null) { + payload.release(); } - assertTrue(postFailures.get() > 0, "expected failures"); + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testPostBinder() { - assertEquals(client.postJson("", "foo").trim(), "{\"key\":\"foo\"}POST"); + @Test + public void testPostContentEncoding() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().addHeader("x-Content-Encoding", "gzip")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + Payload payload = null; + try { + ByteSource body = ByteSource.wrap("foo".getBytes()); + payload = newByteSourcePayload(body); + payload.getContentMetadata().setContentEncoding("gzip"); + payload.getContentMetadata().setContentLength(body.size()); + Multimap headers = client.postPayloadAndReturnHeaders("", payload); + RecordedRequest request = server.takeRequest(); + assertEquals(request.getHeader("Content-Encoding"), "gzip"); + assertEquals(headers.get("x-Content-Encoding"), ImmutableList.of("gzip")); + } finally { + if (payload != null) { + payload.release(); + } + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testPostContentDisposition() { - Payload payload = newStringPayload("foo"); - payload.getContentMetadata().setContentDisposition("attachment; filename=photo.jpg"); - Multimap headers = client.postPayloadAndReturnHeaders("", payload); - assertEquals(headers.get("x-Content-Disposition"), ImmutableList.of("attachment; filename=photo.jpg")); - payload.release(); + @Test + public void testPostContentLanguage() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().addHeader("x-Content-Language", "mi, en")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + Payload payload = null; + try { + ByteSource body = ByteSource.wrap("foo".getBytes()); + payload = newByteSourcePayload(body); + payload.getContentMetadata().setContentLanguage("mi, en"); + payload.getContentMetadata().setContentLength(body.size()); + Multimap headers = client.postPayloadAndReturnHeaders("", payload); + RecordedRequest request = server.takeRequest(); + assertEquals(request.getHeader("Content-Language"), "mi, en"); + assertEquals(headers.get("x-Content-Language"), ImmutableList.of("mi, en")); + } finally { + if (payload != null) { + payload.release(); + } + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testPostContentEncoding() { - Payload payload = newStringPayload("foo"); - payload.getContentMetadata().setContentEncoding("gzip"); - Multimap headers = client.postPayloadAndReturnHeaders("", payload); - assertEquals(headers.get("x-Content-Encoding"), ImmutableList.of("gzip")); - payload.release(); + @Test + public void testPut() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody("fooPUT")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.upload("", "foo"); + // Verify that the body is properly populated + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + assertEquals(result, "fooPUT"); + } finally { + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testPostContentLanguage() { - Payload payload = newStringPayload("foo"); - payload.getContentMetadata().setContentLanguage("mi, en"); - Multimap headers = client.postPayloadAndReturnHeaders("", payload); - assertEquals(headers.get("x-Content-Language"), ImmutableList.of("mi, en")); - payload.release(); + @Test + public void testPutRedirect() throws Exception { + MockWebServer redirectTarget = mockWebServer(new MockResponse().setBody("fooPUTREDIRECT")); + redirectTarget.useHttps(sslContext.getSocketFactory(), false); + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(302).setHeader("Location", + redirectTarget.getUrl("/").toString())); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.upload("redirect", "foo"); + assertEquals(result, "fooPUTREDIRECT"); + assertEquals(server.getRequestCount(), 1); + assertEquals(redirectTarget.getRequestCount(), 1); + // Verify that the body was populated after the redirect + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + request = redirectTarget.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + } finally { + close(client, true); + redirectTarget.shutdown(); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testPut() { - assertEquals(client.upload("", "foo").trim(), "fooPUT"); + @Test + public void testZeroLengthPut() throws Exception { + MockWebServer server = mockWebServer(new MockResponse()); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + client.putNothing(""); + assertEquals(server.getRequestCount(), 1); + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), ""); + } finally { + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testPutRedirect() { - assertEquals(client.upload("redirect", "foo").trim(), "fooPUTREDIRECT"); + @Test + public void testPutIsRetriedOnFailure() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(500), + new MockResponse().setBody("fooPUT")); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.upload("", "foo"); + assertEquals(server.getRequestCount(), 2); + assertEquals(result, "fooPUT"); + // Verify that the body was properly sent in the two requests + RecordedRequest request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + request = server.takeRequest(); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + } finally { + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testHead() { - assertTrue(client.exists(""), "head returned false"); + @Test + public void testHead() throws Exception { + MockWebServer server = mockWebServer(new MockResponse()); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + assertTrue(client.exists("")); + } finally { + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testGetAndParseSax() { - assertEquals(client.downloadAndParse(""), "whoppers"); + @Test + public void testHeadIsRetriedOnServerError() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(500), new MockResponse()); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + assertTrue(client.exists("")); + assertEquals(server.getRequestCount(), 2); + } finally { + close(client, true); + server.shutdown(); + } } - @Test(invocationCount = 5, timeOut = 5000) - public void testZeroLengthPut() { - client.putNothing(""); + @Test + public void testHeadFailure() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(404)); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + assertFalse(client.exists("")); + } finally { + close(client, true); + server.shutdown(); + } } + + @Test + public void testGetAndParseSax() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody(XML)); + IntegrationTestClient client = client(server.getUrl("/").toString()); + try { + String result = client.downloadAndParse(""); + assertEquals(result, "whoppers"); + } finally { + close(client, true); + server.shutdown(); + } + } + + @SuppressWarnings("unchecked") + private InputSupplier getTestDataSupplier() throws IOException { + byte[] oneConstitution = toByteArray(new GZIPInputStream( + BaseHttpCommandExecutorServiceIntegrationTest.class.getResourceAsStream("/const.txt.gz"))); + InputSupplier constitutionSupplier = newInputStreamSupplier(oneConstitution); + InputSupplier temp = join(constitutionSupplier); + for (int i = 0; i < 100; i++) { + temp = join(temp, constitutionSupplier); + } + constitutionsLength = oneConstitution.length * 101; + return temp; + } + } diff --git a/core/src/test/java/org/jclouds/http/BaseMockWebServerTest.java b/core/src/test/java/org/jclouds/http/BaseMockWebServerTest.java new file mode 100644 index 0000000000..6340da2118 --- /dev/null +++ b/core/src/test/java/org/jclouds/http/BaseMockWebServerTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.http; + +import static org.jclouds.Constants.PROPERTY_RELAX_HOSTNAME; +import static org.jclouds.Constants.PROPERTY_TRUST_ALL_CERTS; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.security.GeneralSecurityException; +import java.util.Properties; + +import javax.net.ssl.SSLContext; + +import org.jclouds.ContextBuilder; +import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; +import org.jclouds.providers.AnonymousProviderMetadata; +import org.testng.annotations.BeforeClass; + +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HttpHeaders; +import com.google.inject.Module; +import com.squareup.okhttp.internal.SslContextBuilder; +import com.squareup.okhttp.mockwebserver.Dispatcher; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; +import com.squareup.okhttp.mockwebserver.QueueDispatcher; +import com.squareup.okhttp.mockwebserver.RecordedRequest; + +/** + * Base class for integration tests that use {@link MockWebServer} to verify the + * behavior of the HTTP workflows. + * + * @author Ignasi Barrera + */ +public abstract class BaseMockWebServerTest { + + protected SSLContext sslContext; + + @BeforeClass(groups = "integration") + protected void setupSSL() { + try { + sslContext = new SslContextBuilder(InetAddress.getLocalHost().getHostName()).build(); + } catch (GeneralSecurityException ex) { + throw new RuntimeException(ex); + } catch (UnknownHostException ex) { + throw new RuntimeException(ex); + } + } + + protected static class GlobalChecksRequestDispatcher extends QueueDispatcher { + @Override + public MockResponse dispatch(RecordedRequest request) throws InterruptedException { + MockResponse response = responseQueue.take(); + if (!HttpRequest.NON_PAYLOAD_METHODS.contains(request.getMethod()) + && request.getHeader(HttpHeaders.CONTENT_LENGTH) == null) { + response.setResponseCode(500); + response.setBody("No content length!"); + } + return response; + } + } + + /** + * Creates a {@link MockWebServer} that uses the + * {@link GlobalChecksRequestDispatcher}. + */ + protected static MockWebServer mockWebServer(MockResponse... responses) throws IOException { + MockWebServer server = new MockWebServer(); + server.play(); + server.setDispatcher(new GlobalChecksRequestDispatcher()); + for (MockResponse response : responses) { + server.enqueue(response); + } + return server; + } + + /** + * Creates a {@link MockWebServer} that uses the given {@link Dispatcher}. + */ + protected static MockWebServer mockWebServer(Dispatcher dispatcher) throws IOException { + MockWebServer server = new MockWebServer(); + server.play(); + server.setDispatcher(dispatcher); + return server; + } + + /** + * Creates a test api for the given class and URL. + */ + protected T api(Class apiClass, String url) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_TRUST_ALL_CERTS, "true"); + properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true"); + addOverrideProperties(properties); + return ContextBuilder.newBuilder(AnonymousProviderMetadata.forApiOnEndpoint(apiClass, url)) + .modules(ImmutableSet. of(createConnectionModule())).overrides(properties).buildApi(apiClass); + } + + /** + * Add the connection properties used to configure the tests. + */ + protected abstract void addOverrideProperties(Properties props); + + /** + * Return the connection module that provides the HTTP driver to use in the + * tests. + *

+ * Unless a concrete HTTP is required, subclasses may want to use the + * {@link JavaUrlHttpCommandExecutorServiceModule}. + */ + protected abstract Module createConnectionModule(); + +} diff --git a/core/src/test/java/org/jclouds/http/IntegrationTestAsyncClient.java b/core/src/test/java/org/jclouds/http/IntegrationTestAsyncClient.java index 1ecf08b487..50eb3c93ed 100644 --- a/core/src/test/java/org/jclouds/http/IntegrationTestAsyncClient.java +++ b/core/src/test/java/org/jclouds/http/IntegrationTestAsyncClient.java @@ -196,6 +196,10 @@ public interface IntegrationTestAsyncClient extends Closeable { } + @POST + @Path("/objects/{id}") + ListenableFuture postNothing(@PathParam("id") String id); + @PUT @Path("/objects/{id}") ListenableFuture putNothing(@PathParam("id") String id); diff --git a/core/src/test/java/org/jclouds/http/IntegrationTestClient.java b/core/src/test/java/org/jclouds/http/IntegrationTestClient.java index f2ddcad42e..cadfa7c446 100644 --- a/core/src/test/java/org/jclouds/http/IntegrationTestClient.java +++ b/core/src/test/java/org/jclouds/http/IntegrationTestClient.java @@ -198,6 +198,10 @@ public interface IntegrationTestClient extends Closeable { @PUT @Path("/objects/{id}") void putNothing(@PathParam("id") String id); + + @POST + @Path("/objects/{id}") + void postNothing(@PathParam("id") String id); @Provides StringBuilder newStringBuilder(); diff --git a/core/src/test/java/org/jclouds/http/JavaUrlHttpCommandExecutorServiceIntegrationTest.java b/core/src/test/java/org/jclouds/http/JavaUrlHttpCommandExecutorServiceIntegrationTest.java index ef80193593..c35b1a0bc9 100644 --- a/core/src/test/java/org/jclouds/http/JavaUrlHttpCommandExecutorServiceIntegrationTest.java +++ b/core/src/test/java/org/jclouds/http/JavaUrlHttpCommandExecutorServiceIntegrationTest.java @@ -33,14 +33,14 @@ import com.google.inject.Module; * * @author Adrian Cole */ -@Test +@Test(groups = "integration") public class JavaUrlHttpCommandExecutorServiceIntegrationTest extends BaseHttpCommandExecutorServiceIntegrationTest { protected Module createConnectionModule() { return new JavaUrlHttpCommandExecutorServiceModule(); } - protected void addConnectionProperties(Properties props) { + protected void addOverrideProperties(Properties props) { props.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + ""); props.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 0 + ""); // IO workers not used in this executor diff --git a/core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java b/core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java index b19aeda409..56b0f9ddce 100644 --- a/core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java +++ b/core/src/test/java/org/jclouds/http/handlers/BackoffLimitedRetryHandlerTest.java @@ -21,14 +21,15 @@ import static org.testng.Assert.assertTrue; import java.io.IOException; import java.io.InputStream; -import java.util.Properties; -import org.jclouds.http.BaseJettyTest; +import org.jclouds.ContextBuilder; import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; import org.jclouds.http.IntegrationTestAsyncClient; +import org.jclouds.http.IntegrationTestClient; import org.jclouds.io.Payloads; +import org.jclouds.providers.AnonymousProviderMetadata; import org.jclouds.reflect.Invocation; import org.jclouds.rest.internal.RestAnnotationProcessor; import org.testng.annotations.Test; @@ -122,8 +123,10 @@ public class BackoffLimitedRetryHandlerTest { assertEquals(response.getPayload().getInput().read(), -1); } - private final Function processor = BaseJettyTest.newBuilder(8100, new Properties()).buildInjector() - .getInstance(RestAnnotationProcessor.class); + private final Function processor = ContextBuilder + .newBuilder(AnonymousProviderMetadata.forApiOnEndpoint(IntegrationTestClient.class, "http://localhost")) + .buildInjector().getInstance(RestAnnotationProcessor.class); + private HttpCommand createCommand() throws SecurityException, NoSuchMethodException { Invokable method = method(IntegrationTestAsyncClient.class, "download", String.class); diff --git a/drivers/apachehc/src/test/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorServiceTestDisabled.java b/drivers/apachehc/src/test/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorServiceTestDisabled.java index 1c05a81770..014586b977 100644 --- a/drivers/apachehc/src/test/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorServiceTestDisabled.java +++ b/drivers/apachehc/src/test/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorServiceTestDisabled.java @@ -46,7 +46,7 @@ public class ApacheHCHttpCommandExecutorServiceTestDisabled extends BaseHttpComm return new ApacheHCHttpCommandExecutorServiceModule(); } - protected void addConnectionProperties(Properties props) { + protected void addOverrideProperties(Properties props) { props.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 20 + ""); props.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 0 + ""); props.setProperty(PROPERTY_CONNECTION_TIMEOUT, 100 + ""); diff --git a/drivers/gae/pom.xml b/drivers/gae/pom.xml index 86c37d2f6b..06ec7024db 100644 --- a/drivers/gae/pom.xml +++ b/drivers/gae/pom.xml @@ -61,7 +61,6 @@ appengine-api-1.0-sdk 1.6.5 - org.apache.jclouds jclouds-core @@ -70,8 +69,8 @@ test - org.eclipse.jetty - jetty-security + com.squareup.okhttp + mockwebserver test diff --git a/drivers/gae/src/test/java/org/jclouds/gae/AsyncGaeHttpCommandExecutorServiceIntegrationTest.java b/drivers/gae/src/test/java/org/jclouds/gae/AsyncGaeHttpCommandExecutorServiceIntegrationTest.java index 115fc625ae..2596cbb1d9 100644 --- a/drivers/gae/src/test/java/org/jclouds/gae/AsyncGaeHttpCommandExecutorServiceIntegrationTest.java +++ b/drivers/gae/src/test/java/org/jclouds/gae/AsyncGaeHttpCommandExecutorServiceIntegrationTest.java @@ -19,16 +19,12 @@ package org.jclouds.gae; import java.io.IOException; import java.util.Properties; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - import org.jclouds.concurrent.SingleThreaded; import org.jclouds.concurrent.config.ConfiguresExecutorService; import org.jclouds.gae.config.GoogleAppEngineConfigurationModule; import org.jclouds.http.BaseHttpCommandExecutorServiceIntegrationTest; import org.jclouds.http.HttpCommandExecutorService; import org.jclouds.http.config.ConfiguresHttpCommandExecutorService; -import org.jclouds.logging.Logger; import org.testng.SkipException; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -46,121 +42,31 @@ import com.google.inject.Module; */ @Test public class AsyncGaeHttpCommandExecutorServiceIntegrationTest extends BaseHttpCommandExecutorServiceIntegrationTest { - Logger logger = Logger.CONSOLE; - - @Override - protected void setupAndStartSSLServer(final int testPort) throws Exception { - } - @Override - protected boolean redirectEveryTwentyRequests(HttpServletRequest request, HttpServletResponse response) - throws IOException { - return false; + @BeforeMethod + public void setupApiProxy() { + LocalServiceTestHelper helper = new LocalServiceTestHelper(new LocalURLFetchServiceTestConfig()); + helper.setUp(); } @Override public void testPostAsInputStream() { throw new SkipException("streams aren't supported"); } - + @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testPostBinder() { - setupApiProxy(); - super.testPostBinder(); + public void testPostAsInputStreamDoesNotRetryOnFailure() throws Exception { + throw new SkipException("streams aren't supported"); } - - @BeforeMethod - void setupApiProxy() { - LocalServiceTestHelper helper = new LocalServiceTestHelper(new LocalURLFetchServiceTestConfig()); - helper.setUp(); + + @Override + public void testGetBigFile() { + throw new SkipException("test data is too big for GAE"); } @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testGetAndParseSax() { - setupApiProxy(); - super.testGetAndParseSax(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testGetString() { - setupApiProxy(); - super.testGetString(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000, dataProvider = "gets") - public void testGetStringSynch(String path) { - setupApiProxy(); - super.testGetStringSynch(path); - } - - @Override - public void testGetStringRedirect() { - throw new SkipException("need to get redirects to operate"); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testGetException() { - setupApiProxy(); - super.testGetException(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testGetSynchException() { - setupApiProxy(); - super.testGetSynchException(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testPost() { - setupApiProxy(); - super.testPost(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testPut() { - setupApiProxy(); - super.testPut(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testGetStringViaRequest() throws IOException { - setupApiProxy(); - super.testGetStringViaRequest(); - } - - @Override - public void testPutRedirect() { - throw new SkipException("need to get redirects to operate"); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testGetStringWithHeader() { - setupApiProxy(); - super.testGetStringWithHeader(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testHead() { - setupApiProxy(); - super.testHead(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testRequestFilter() { - setupApiProxy(); - super.testRequestFilter(); + public void testUploadBigFile() throws IOException { + throw new SkipException("test data is too big for GAE"); } protected Module createConnectionModule() { @@ -184,45 +90,8 @@ public class AsyncGaeHttpCommandExecutorServiceIntegrationTest extends BaseHttpC } @Override - protected void addConnectionProperties(Properties props) { - } - - @Override - public void testGetBigFile() { - throw new SkipException("test data is too big for GAE"); - } - - @Override - public void testUploadBigFile() throws IOException { - throw new SkipException("test data is too big for GAE"); - } - - @Override - public void testPostContentDisposition() { - setupApiProxy(); - super.testPostContentDisposition(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testPostContentEncoding() { - setupApiProxy(); - super.testPostContentEncoding(); - } - - @Override - @Test(enabled = true, invocationCount = 5, timeOut = 3000) - public void testPostContentLanguage() { - setupApiProxy(); - super.testPostContentLanguage(); - } - - // http://code.google.com/p/googleappengine/issues/detail?id=3599 - @Override - @Test(enabled = true, expectedExceptions = IllegalArgumentException.class) - public void testAlternateMethod() { - setupApiProxy(); - super.testAlternateMethod(); + protected void addOverrideProperties(Properties props) { + } } diff --git a/drivers/okhttp/README.md b/drivers/okhttp/README.md new file mode 100644 index 0000000000..cf418493e3 --- /dev/null +++ b/drivers/okhttp/README.md @@ -0,0 +1,15 @@ +jclouds OkHttp driver +===================== + +A driver to use the OkHttp (http://square.github.io/okhttp/) client as an HTTP library in jclouds. + +This driver adds support for use of modern HTTP verbs such as PATCH in providers and APIs, and also supports SPDY. + +To use the driver, you just need to include the `OkHttpCommandExecutorServiceModule` when creating +the context: + + ContextBuilder.newBuilder("provider") + .endpoint("endpoint") + .credentials("identity", "credential") + .modules(ImmutableSet.of(new OkHttpCommandExecutorServiceModule())) + .build(); diff --git a/drivers/okhttp/pom.xml b/drivers/okhttp/pom.xml new file mode 100644 index 0000000000..9b7d3985c8 --- /dev/null +++ b/drivers/okhttp/pom.xml @@ -0,0 +1,64 @@ + + + + 4.0.0 + + org.apache.jclouds + jclouds-project + 1.8.0-SNAPSHOT + ../../project/pom.xml + + org.apache.jclouds.driver + jclouds-okhttp + jclouds OkHttp Driver + bundle + OkHttp Driver + + + org.jclouds.http.okhttp*;version="${project.version}" + org.jclouds*;version="${project.version}",* + + + + + org.apache.jclouds + jclouds-core + ${project.version} + + + org.apache.jclouds + jclouds-core + ${project.version} + test-jar + test + + + com.squareup.okhttp + okhttp + ${okhttp.version} + + + com.squareup.okhttp + mockwebserver + test + + + + diff --git a/drivers/okhttp/src/main/java/org/jclouds/http/okhttp/OkHttpCommandExecutorService.java b/drivers/okhttp/src/main/java/org/jclouds/http/okhttp/OkHttpCommandExecutorService.java new file mode 100644 index 0000000000..651969b814 --- /dev/null +++ b/drivers/okhttp/src/main/java/org/jclouds/http/okhttp/OkHttpCommandExecutorService.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.http.okhttp; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.Proxy; +import java.net.URI; +import java.net.URL; + +import javax.inject.Named; +import javax.inject.Singleton; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.ws.rs.core.HttpHeaders; + +import org.jclouds.Constants; +import org.jclouds.http.HttpRequest; +import org.jclouds.http.HttpUtils; +import org.jclouds.http.IOExceptionRetryHandler; +import org.jclouds.http.handlers.DelegatingErrorHandler; +import org.jclouds.http.handlers.DelegatingRetryHandler; +import org.jclouds.http.internal.HttpWire; +import org.jclouds.http.internal.JavaUrlHttpCommandExecutorService; +import org.jclouds.io.ContentMetadataCodec; + +import com.google.common.base.Function; +import com.google.common.base.Supplier; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.inject.Inject; +import com.squareup.okhttp.OkHttpClient; + +/** + * Implementation of the HttpCommandExecutorService that uses the + * OkHttp client to support modern HTTP methods such as PATCH. + * + * @author Ignasi Barrera + */ +@Singleton +public class OkHttpCommandExecutorService extends JavaUrlHttpCommandExecutorService { + + @Inject + public OkHttpCommandExecutorService(HttpUtils utils, ContentMetadataCodec contentMetadataCodec, + @Named(Constants.PROPERTY_IO_WORKER_THREADS) ListeningExecutorService ioExecutor, + DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler, + DelegatingErrorHandler errorHandler, HttpWire wire, @Named("untrusted") HostnameVerifier verifier, + @Named("untrusted") Supplier untrustedSSLContextProvider, Function proxyForURI) + throws SecurityException, NoSuchFieldException { + super(utils, contentMetadataCodec, ioExecutor, retryHandler, ioRetryHandler, errorHandler, wire, verifier, + untrustedSSLContextProvider, proxyForURI); + } + + @Override + protected HttpURLConnection initConnection(HttpRequest request) throws IOException { + OkHttpClient client = new OkHttpClient(); + URL url = request.getEndpoint().toURL(); + client.setProxy(proxyForURI.apply(request.getEndpoint())); + if (url.getProtocol().equalsIgnoreCase("https")) { + if (utils.relaxHostname()) { + client.setHostnameVerifier(verifier); + } + if (sslContextSupplier != null) { + // used for providers which e.g. use certs for authentication (like + // FGCP) Provider provides SSLContext impl (which inits context with + // key manager) + client.setSslSocketFactory(sslContextSupplier.get().getSocketFactory()); + } else if (utils.trustAllCerts()) { + client.setSslSocketFactory(untrustedSSLContextProvider.get().getSocketFactory()); + } + } + return client.open(url); + } + + @Override + protected void configureRequestHeaders(HttpURLConnection connection, HttpRequest request) { + super.configureRequestHeaders(connection, request); + // OkHttp does not set the Accept header if not present in the request. + // Make sure we send a flexible one. + if (request.getFirstHeaderOrNull(HttpHeaders.ACCEPT) == null) { + connection.setRequestProperty(HttpHeaders.ACCEPT, "*/*"); + } + } + +} diff --git a/drivers/okhttp/src/main/java/org/jclouds/http/okhttp/config/OkHttpCommandExecutorServiceModule.java b/drivers/okhttp/src/main/java/org/jclouds/http/okhttp/config/OkHttpCommandExecutorServiceModule.java new file mode 100644 index 0000000000..2e7c11683e --- /dev/null +++ b/drivers/okhttp/src/main/java/org/jclouds/http/okhttp/config/OkHttpCommandExecutorServiceModule.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.http.okhttp.config; + +import org.jclouds.http.HttpCommandExecutorService; +import org.jclouds.http.config.ConfiguresHttpCommandExecutorService; +import org.jclouds.http.config.SSLModule; +import org.jclouds.http.okhttp.OkHttpCommandExecutorService; + +import com.google.inject.AbstractModule; +import com.google.inject.Scopes; + +/** + * Configures the {@link OkHttpCommandExecutorService}. + * + * Note that this uses threads. + * + * @author Ignasi Barrera + */ +@ConfiguresHttpCommandExecutorService +public class OkHttpCommandExecutorServiceModule extends AbstractModule { + + @Override + protected void configure() { + install(new SSLModule()); + bind(HttpCommandExecutorService.class).to(OkHttpCommandExecutorService.class).in(Scopes.SINGLETON); + } + +} diff --git a/drivers/okhttp/src/test/java/org/jclouds/http/okhttp/OkHttpCommandExecutorServiceTest.java b/drivers/okhttp/src/test/java/org/jclouds/http/okhttp/OkHttpCommandExecutorServiceTest.java new file mode 100644 index 0000000000..e8f9005558 --- /dev/null +++ b/drivers/okhttp/src/test/java/org/jclouds/http/okhttp/OkHttpCommandExecutorServiceTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.http.okhttp; + +import static com.google.common.io.Closeables.close; +import static org.jclouds.Constants.PROPERTY_IO_WORKER_THREADS; +import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT; +import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST; +import static org.jclouds.Constants.PROPERTY_USER_THREADS; +import static org.testng.Assert.assertEquals; + +import java.io.Closeable; +import java.util.Properties; + +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; + +import org.jclouds.http.BaseHttpCommandExecutorServiceIntegrationTest; +import org.jclouds.http.okhttp.config.OkHttpCommandExecutorServiceModule; +import org.jclouds.rest.annotations.BinderParam; +import org.jclouds.rest.annotations.PATCH; +import org.jclouds.rest.binders.BindToStringPayload; +import org.testng.annotations.Test; + +import com.google.inject.Module; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; +import com.squareup.okhttp.mockwebserver.RecordedRequest; + +/** + * Tests the functionality of the {@link OkHttpCommandExecutorService} + * + * @author Ignasi Barrera + */ +@Test +public class OkHttpCommandExecutorServiceTest extends BaseHttpCommandExecutorServiceIntegrationTest { + + @Override + protected Module createConnectionModule() { + return new OkHttpCommandExecutorServiceModule(); + } + + @Override + protected void addOverrideProperties(final Properties props) { + props.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + ""); + props.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 0 + ""); + // IO workers not used in this executor + props.setProperty(PROPERTY_IO_WORKER_THREADS, 0 + ""); + props.setProperty(PROPERTY_USER_THREADS, 5 + ""); + } + + private interface PatchApi extends Closeable { + @PATCH + @Path("/objects/{id}") + String patch(@PathParam("id") String id, @BinderParam(BindToStringPayload.class) String body); + + @PATCH + @Path("/objects/{id}") + String patchNothing(@PathParam("id") String id); + } + + @Test + public void testPatch() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setBody("fooPATCH")); + PatchApi api = api(PatchApi.class, server.getUrl("/").toString()); + try { + String result = api.patch("", "foo"); + // Verify that the body is properly populated + RecordedRequest request = server.takeRequest(); + assertEquals(request.getMethod(), "PATCH"); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + assertEquals(result, "fooPATCH"); + } finally { + close(api, true); + server.shutdown(); + } + } + + @Test + public void testPatchIsRetriedOnFailure() throws Exception { + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(500), + new MockResponse().setBody("fooPATCH")); + PatchApi api = api(PatchApi.class, server.getUrl("/").toString()); + try { + String result = api.patch("", "foo"); + assertEquals(server.getRequestCount(), 2); + assertEquals(result, "fooPATCH"); + // Verify that the body was properly sent in the two requests + RecordedRequest request = server.takeRequest(); + assertEquals(request.getMethod(), "PATCH"); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + request = server.takeRequest(); + assertEquals(request.getMethod(), "PATCH"); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + } finally { + close(api, true); + server.shutdown(); + } + } + + @Test + public void testPatchRedirect() throws Exception { + MockWebServer redirectTarget = mockWebServer(new MockResponse().setBody("fooPATCHREDIRECT")); + redirectTarget.useHttps(sslContext.getSocketFactory(), false); + MockWebServer server = mockWebServer(new MockResponse().setResponseCode(302).setHeader("Location", + redirectTarget.getUrl("/").toString())); + PatchApi api = api(PatchApi.class, server.getUrl("/").toString()); + try { + String result = api.patch("", "foo"); + assertEquals(result, "fooPATCHREDIRECT"); + assertEquals(server.getRequestCount(), 1); + assertEquals(redirectTarget.getRequestCount(), 1); + // Verify that the body was populated after the redirect + RecordedRequest request = server.takeRequest(); + assertEquals(request.getMethod(), "PATCH"); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + request = redirectTarget.takeRequest(); + assertEquals(request.getMethod(), "PATCH"); + assertEquals(new String(request.getBody(), "UTF-8"), "foo"); + } finally { + close(api, true); + redirectTarget.shutdown(); + server.shutdown(); + } + } + + @Test + public void testZeroLengthPatch() throws Exception { + MockWebServer server = mockWebServer(new MockResponse()); + PatchApi api = api(PatchApi.class, server.getUrl("/").toString()); + try { + api.patchNothing(""); + assertEquals(server.getRequestCount(), 1); + RecordedRequest request = server.takeRequest(); + assertEquals(request.getMethod(), "PATCH"); + assertEquals(new String(request.getBody(), "UTF-8"), ""); + } finally { + close(api, true); + server.shutdown(); + } + } +} diff --git a/drivers/okhttp/src/test/resources/test.jks b/drivers/okhttp/src/test/resources/test.jks new file mode 100644 index 0000000000..e641fb5470 Binary files /dev/null and b/drivers/okhttp/src/test/resources/test.jks differ diff --git a/drivers/pom.xml b/drivers/pom.xml index f3357f707e..34ce8ce986 100644 --- a/drivers/pom.xml +++ b/drivers/pom.xml @@ -39,5 +39,6 @@ jsch netty enterprise + okhttp diff --git a/project/pom.xml b/project/pom.xml index fce64f0628..c94da03948 100644 --- a/project/pom.xml +++ b/project/pom.xml @@ -212,6 +212,7 @@ org.jclouds.test.testng.UnitTestStatusListener source-release-zip-tar + 1.3.0 @@ -251,6 +252,11 @@ jetty-server 8.1.8.v20121106 + + com.squareup.okhttp + mockwebserver + ${okhttp.version} + org.testng testng