Issue 207: increased ferocity of tests to do concurrent 5MB downloads w/checksum on both http and blobstore level.

aws/core/src/test/resources/log4j.xml

@@ -138,13 +138,13 @@
     <category name="jclouds.compute">
         <priority value="TRACE" />
         <appender-ref ref="ASYNCCOMPUTE" />
-    </category>
+    </category><!--
 
     <category name="jclouds.wire">
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCWIRE" />
     </category>
-    <!--  ======================= -->
+    --><!--  ======================= -->
     <!-- Setup the Root category -->
     <!-- ======================= -->
 

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java

@@ -23,17 +23,21 @@ import static org.jclouds.blobstore.options.GetOptions.Builder.ifETagMatches;
 import static org.jclouds.blobstore.options.GetOptions.Builder.ifModifiedSince;
 import static org.jclouds.blobstore.options.GetOptions.Builder.ifUnmodifiedSince;
 import static org.jclouds.blobstore.options.GetOptions.Builder.range;
+import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.util.Date;
+import java.util.Map;
 import java.util.Set;
+import java.util.zip.GZIPInputStream;
 
 import javax.ws.rs.core.MediaType;
 
@@ -43,17 +47,103 @@ import org.jclouds.blobstore.domain.BlobMetadata;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl;
+import org.jclouds.encryption.EncryptionService;
+import org.jclouds.encryption.EncryptionService.MD5InputStreamResult;
 import org.jclouds.encryption.internal.JCEEncryptionService;
+import org.jclouds.http.BaseJettyTest;
 import org.jclouds.http.HttpResponseException;
 import org.jclouds.http.Payloads;
+import org.jclouds.logging.Logger;
 import org.jclouds.util.Utils;
+import org.testng.ITestContext;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Maps;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.InputSupplier;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Guice;
+
 /**
  * @author Adrian Cole
  */
 public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
+   private byte[] oneHundredOneConstitutions;
+   private EncryptionService encryptionService;
+   private byte[] oneHundredOneConstitutionsMD5;
+   private long oneHundredOneConstitutionsLength;
+
+   @BeforeClass(groups = { "integration", "live" })
+   @Override
+   public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
+      encryptionService = Guice.createInjector().getInstance(EncryptionService.class);
+      MD5InputStreamResult result = encryptionService.generateMD5Result(getTestDataSupplier()
+               .getInput());
+      oneHundredOneConstitutions = result.data;
+      oneHundredOneConstitutionsMD5 = result.md5;
+      oneHundredOneConstitutionsLength = result.length;
+      super.setUpResourcesOnThisThread(testContext);
+   }
+
+   @SuppressWarnings("unchecked")
+   public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
+      byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
+               .getResourceAsStream("/const.txt.gz")));
+      InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams
+               .newInputStreamSupplier(oneConstitution);
+
+      InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
+
+      for (int i = 0; i < 100; i++) {
+         temp = ByteStreams.join(temp, constitutionSupplier);
+      }
+      return temp;
+   }
+
+   @Test(groups = { "integration", "live" })
+   public void testBigFileGets() throws InterruptedException, IOException {
+      String containerName = getContainerName();
+      try {
+         String key = "constitution.txt";
+
+         uploadConstitution(containerName, key);
+         Map<Integer, ListenableFuture<?>> responses = Maps.newHashMap();
+         for (int i = 0; i < 10; i++) {
+
+            responses.put(i, Futures.compose(context.getAsyncBlobStore()
+                     .getBlob(containerName, key), new Function<Blob, Void>() {
+
+               @Override
+               public Void apply(Blob from) {
+                  assertEquals(encryptionService.md5(from.getContent()),
+                           oneHundredOneConstitutionsMD5);
+                  return null;
+               }
+
+            }));
+         }
+         Map<Integer, Exception> exceptions = awaitCompletion(responses, exec, 30000l,
+                  Logger.CONSOLE, "get constitution");
+         assert exceptions.size() == 0 : exceptions;
+
+      } finally {
+         returnContainer(containerName);
+      }
+
+   }
+
+   private void uploadConstitution(String containerName, String key) throws IOException {
+      Blob sourceObject = context.getBlobStore().newBlob(key);
+      sourceObject.getMetadata().setContentType("text/plain");
+      sourceObject.getMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
+      sourceObject.setContentLength(oneHundredOneConstitutionsLength);
+      sourceObject.setPayload(oneHundredOneConstitutions);
+      context.getBlobStore().putBlob(containerName, sourceObject);
+   }
 
    @Test(groups = { "integration", "live" })
    public void testGetIfModifiedSince() throws InterruptedException {

core/src/main/java/org/jclouds/encryption/EncryptionService.java

@@ -21,7 +21,6 @@ package org.jclouds.encryption;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import java.io.File;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.security.InvalidKeyException;
@@ -57,7 +56,7 @@ public interface EncryptionService {
 
    byte[] md5(byte[] plainBytes);
 
-   byte[] md5(File toEncode);
+   byte[] md5(InputStream toEncode);
 
    String toBase64String(byte[] resBuf);
 

core/src/main/java/org/jclouds/encryption/internal/BaseEncryptionService.java

@@ -21,6 +21,8 @@ package org.jclouds.encryption.internal;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
@@ -30,6 +32,8 @@ import java.security.NoSuchProviderException;
 
 import org.jclouds.encryption.EncryptionService;
 
+import com.google.common.base.Throwables;
+
 /**
  * 
  * @author Adrian Cole
@@ -90,9 +94,13 @@ public abstract class BaseEncryptionService implements EncryptionService {
       } else if (data instanceof String) {
          md5 = md5(((String) data).getBytes());
       } else if (data instanceof File) {
-         md5 = md5(((File) data));
+         try {
+            md5 = md5(new FileInputStream((File) data));
+         } catch (FileNotFoundException e) {
+            Throwables.propagate(e);
+         }
       } else if (data instanceof InputStream) {
-         md5 = generateMD5Result(((InputStream) data)).md5;
+         md5 = md5(((InputStream) data));
       } else {
          throw new UnsupportedOperationException("Content not supported " + data.getClass());
       }

core/src/main/java/org/jclouds/encryption/internal/JCEEncryptionService.java

@@ -18,8 +18,7 @@
  */
 package org.jclouds.encryption.internal;
 
-import java.io.File;
-import java.io.FileInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.security.InvalidKeyException;
@@ -30,8 +29,6 @@ import java.security.NoSuchProviderException;
 import javax.crypto.Mac;
 import javax.crypto.spec.SecretKeySpec;
 
-import java.io.ByteArrayOutputStream;
-
 import com.google.common.io.Closeables;
 
 /**
@@ -78,15 +75,13 @@ public class JCEEncryptionService extends BaseEncryptionService {
       return eTag.digest();
    }
 
-   public byte[] md5(File toEncode) {
+   public byte[] md5(InputStream toEncode) {
       MessageDigest eTag = getDigest();
       byte[] buffer = new byte[1024];
       int numRead = -1;
-      InputStream i = null;
       try {
-         i = new FileInputStream(toEncode);
          do {
-            numRead = i.read(buffer);
+            numRead = toEncode.read(buffer);
             if (numRead > 0) {
                eTag.update(buffer, 0, numRead);
             }
@@ -94,7 +89,7 @@ public class JCEEncryptionService extends BaseEncryptionService {
       } catch (IOException e) {
          throw new RuntimeException(e);
       } finally {
-         Closeables.closeQuietly(i);
+         Closeables.closeQuietly(toEncode);
       }
       return eTag.digest();
    }

core/src/test/java/org/jclouds/http/BackoffLimitedRetryJavaTest.java

@@ -44,7 +44,7 @@ import com.google.inject.Module;
  * @author James Murty
  */
 @Test(sequential = true)
-public class BackoffLimitedRetryJavaIntegrationTest extends BaseJettyTest {
+public class BackoffLimitedRetryJavaTest extends BaseJettyTest {
    private int beginToFailOnRequestNumber = 0;
    private int endFailuresOnRequestNumber = 0;
    private int requestCount = 0;

core/src/test/java/org/jclouds/http/BaseHttpCommandExecutorServiceIntegrationTest.java

@@ -38,8 +38,8 @@ import com.google.common.collect.ImmutableMap;
  * 
  * @author Adrian Cole
  */
-@Test(threadPoolSize = 10, sequential = true)
-public abstract class BaseHttpCommandExecutorServiceTest extends BaseJettyTest {
+@Test(threadPoolSize = 10, groups = "integration", sequential = true)
+public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends BaseJettyTest {
 
    @Test(invocationCount = 25, timeOut = 5000)
    public void testRequestFilter() throws MalformedURLException, ExecutionException,
@@ -92,6 +92,13 @@ public abstract class BaseHttpCommandExecutorServiceTest extends BaseJettyTest {
       assertEquals(client.download("redirect").trim(), XML2);
    }
 
+   @Test(invocationCount = 100, timeOut = 5000)
+   public void testGetBigFile() throws MalformedURLException, ExecutionException,
+            InterruptedException, TimeoutException {
+      assertEquals(encryptionService.toBase64String(encryptionService.md5(client
+               .downloadStream("101constitutions"))), md5);
+   }
+
    @Test(enabled = false, invocationCount = 25, timeOut = 5000)
    public void testGetStringPermanentRedirect() throws MalformedURLException, ExecutionException,
             InterruptedException, TimeoutException {

core/src/test/java/org/jclouds/http/BaseJettyTest.java

@@ -18,7 +18,9 @@
  */
 package org.jclouds.http;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
 import java.util.List;
 import java.util.Map;
@@ -26,6 +28,7 @@ import java.util.Properties;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
 
 import javax.inject.Singleton;
 import javax.servlet.ServletException;
@@ -35,6 +38,7 @@ import javax.ws.rs.core.HttpHeaders;
 
 import org.jclouds.PropertiesBuilder;
 import org.jclouds.concurrent.internal.SyncProxy;
+import org.jclouds.encryption.EncryptionService;
 import org.jclouds.lifecycle.Closer;
 import org.jclouds.rest.ConfiguresRestClient;
 import org.jclouds.rest.RestClientFactory;
@@ -53,7 +57,10 @@ import org.testng.annotations.Optional;
 import org.testng.annotations.Parameters;
 
 import com.google.common.collect.Maps;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.InputSupplier;
 import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Module;
 import com.google.inject.Provides;
@@ -139,12 +146,22 @@ public abstract class BaseJettyTest {
    private Server server2;
    protected RestContext<IntegrationTestAsyncClient, IntegrationTestClient> context;
    private int testPort;
+   protected EncryptionService encryptionService;
+   protected String md5;
    static final Pattern actionPattern = Pattern.compile("/objects/(.*)/action/([a-z]*);?(.*)");
 
    @BeforeTest
    @Parameters( { "test-jetty-port" })
    public void setUpJetty(@Optional("8123") final int testPort) throws Exception {
       this.testPort = testPort;
+
+      final InputSupplier<InputStream> oneHundredOneConstitutions = getTestDataSupplier();
+
+      encryptionService = Guice.createInjector().getInstance(EncryptionService.class);
+
+      md5 = encryptionService.md5Base64(ByteStreams.toByteArray(oneHundredOneConstitutions
+               .getInput()));
+
       Handler server1Handler = new AbstractHandler() {
          public void handle(String target, HttpServletRequest request,
                   HttpServletResponse response, int dispatch) throws IOException, ServletException {
@@ -152,6 +169,11 @@ public abstract class BaseJettyTest {
                return;
             } else if (target.indexOf("redirect") > 0) {
                response.sendRedirect("http://localhost:" + (testPort + 1));
+            } else if (target.indexOf("101constitutions") > 0) {
+               response.setContentType("text/plain");
+               response.setHeader("Content-MD5", md5);
+               response.setStatus(HttpServletResponse.SC_OK);
+               ByteStreams.copy(oneHundredOneConstitutions.getInput(), response.getOutputStream());
             } else if (request.getMethod().equals("PUT")) {
                if (request.getContentLength() > 0) {
                   response.setStatus(HttpServletResponse.SC_OK);
@@ -245,6 +267,21 @@ public abstract class BaseJettyTest {
       assert client.newStringBuffer() != null;
    }
 
+   @SuppressWarnings("unchecked")
+   public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
+      byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
+               .getResourceAsStream("/const.txt.gz")));
+      InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams
+               .newInputStreamSupplier(oneConstitution);
+
+      InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
+
+      for (int i = 0; i < 100; i++) {
+         temp = ByteStreams.join(temp, constitutionSupplier);
+      }
+      return temp;
+   }
+
    public static RestContextBuilder<IntegrationTestAsyncClient, IntegrationTestClient> newBuilder(
             final int testPort, final Properties properties, Module connectionModule) {
       return new IntegrationContextBuilder(properties, testPort).withModules(connectionModule);

core/src/test/java/org/jclouds/http/IntegrationTestAsyncClient.java

@@ -18,6 +18,7 @@
  */
 package org.jclouds.http;
 
+import java.io.InputStream;
 import java.util.Map;
 
 import javax.ws.rs.GET;
@@ -62,6 +63,10 @@ public interface IntegrationTestAsyncClient {
    @Path("objects/{id}")
    ListenableFuture<String> download(@PathParam("id") String id);
 
+   @GET
+   @Path("{path}")
+   ListenableFuture<InputStream> downloadStream(@PathParam("path") String path);
+
    @GET
    @Path("{path}")
    ListenableFuture<String> synch(@PathParam("path") String id);

core/src/test/java/org/jclouds/http/IntegrationTestClient.java

@@ -18,6 +18,7 @@
  */
 package org.jclouds.http;
 
+import java.io.InputStream;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -38,6 +39,8 @@ public interface IntegrationTestClient {
 
    String download(String id);
 
+   InputStream downloadStream(String id);
+
    String downloadException(String id, HttpRequestOptions options);
 
    String synchException(String id, String header);

core/src/test/java/org/jclouds/http/JavaUrlHttpCommandExecutorServiceIntegrationTest.java

@@ -37,7 +37,7 @@ import com.google.inject.Module;
  * @author Adrian Cole
  */
 @Test
-public class JavaUrlHttpCommandExecutorServiceTest extends BaseHttpCommandExecutorServiceTest {
+public class JavaUrlHttpCommandExecutorServiceIntegrationTest extends BaseHttpCommandExecutorServiceIntegrationTest {
 
    protected Module createConnectionModule() {
       return new JavaUrlHttpCommandExecutorServiceModule();

extensions/apachehc/src/test/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorServiceTest.java

@@ -25,7 +25,7 @@ import static org.jclouds.Constants.PROPERTY_USER_THREADS;
 
 import java.util.Properties;
 
-import org.jclouds.http.BaseHttpCommandExecutorServiceTest;
+import org.jclouds.http.BaseHttpCommandExecutorServiceIntegrationTest;
 import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
 import org.testng.annotations.Test;
 
@@ -37,7 +37,7 @@ import com.google.inject.Module;
  * @author Adrian Cole
  */
 @Test
-public class ApacheHCHttpCommandExecutorServiceTest extends BaseHttpCommandExecutorServiceTest {
+public class ApacheHCHttpCommandExecutorServiceTest extends BaseHttpCommandExecutorServiceIntegrationTest {
    static {
       System.setProperty("http.conn-manager.timeout", 1000 + "");
    }

extensions/bouncycastle/src/main/java/org/jclouds/encryption/bouncycastle/BouncyCastleEncryptionService.java

@@ -19,8 +19,6 @@
 package org.jclouds.encryption.bouncycastle;
 
 import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.security.InvalidKeyException;
@@ -83,16 +81,14 @@ public class BouncyCastleEncryptionService extends BaseEncryptionService {
       return resBuf;
    }
 
-   public byte[] md5(File toEncode) {
+   public byte[] md5(InputStream toEncode) {
       MD5Digest eTag = new MD5Digest();
       byte[] resBuf = new byte[eTag.getDigestSize()];
       byte[] buffer = new byte[1024];
       int numRead = -1;
-      InputStream i = null;
       try {
-         i = new FileInputStream(toEncode);
          do {
-            numRead = i.read(buffer);
+            numRead = toEncode.read(buffer);
             if (numRead > 0) {
                eTag.update(buffer, 0, numRead);
             }
@@ -100,7 +96,7 @@ public class BouncyCastleEncryptionService extends BaseEncryptionService {
       } catch (IOException e) {
          throw new RuntimeException(e);
       } finally {
-         Closeables.closeQuietly(i);
+         Closeables.closeQuietly(toEncode);
       }
       eTag.doFinal(resBuf, 0);
       return resBuf;

extensions/enterprise/src/test/java/org/jclouds/enterprise/config/EnterpriseConfigurationModuleTest.java

@@ -25,7 +25,7 @@ import static org.jclouds.Constants.PROPERTY_USER_THREADS;
 
 import java.util.Properties;
 
-import org.jclouds.http.BaseHttpCommandExecutorServiceTest;
+import org.jclouds.http.BaseHttpCommandExecutorServiceIntegrationTest;
 import org.testng.annotations.Test;
 
 import com.google.inject.Module;
@@ -36,7 +36,7 @@ import com.google.inject.Module;
  * @author Adrian Cole
  */
 @Test
-public class EnterpriseConfigurationModuleTest extends BaseHttpCommandExecutorServiceTest {
+public class EnterpriseConfigurationModuleTest extends BaseHttpCommandExecutorServiceIntegrationTest {
 
    protected Module createConnectionModule() {
       return new EnterpriseConfigurationModule();

extensions/gae/src/test/java/org/jclouds/gae/GaeHttpCommandExecutorServiceIntegrationTest.java

@@ -28,7 +28,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
 
 import org.jclouds.gae.config.GoogleAppEngineConfigurationModule;
-import org.jclouds.http.BaseHttpCommandExecutorServiceTest;
+import org.jclouds.http.BaseHttpCommandExecutorServiceIntegrationTest;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
@@ -46,7 +46,7 @@ import com.google.inject.Module;
  */
 @Test
 public class GaeHttpCommandExecutorServiceIntegrationTest extends
-         BaseHttpCommandExecutorServiceTest {
+         BaseHttpCommandExecutorServiceIntegrationTest {
 
    @Override
    @Test(invocationCount = 50, timeOut = 3000)
@@ -248,4 +248,11 @@ public class GaeHttpCommandExecutorServiceIntegrationTest extends
    protected void addConnectionProperties(Properties props) {
    }
 
+   @Override
+   @Test(invocationCount = 50, timeOut = 3000)
+   public void testGetBigFile() throws MalformedURLException, ExecutionException,
+            InterruptedException, TimeoutException {
+      // disabled since test data is too big
+   }
+
 }

extensions/httpnio/src/test/java/org/jclouds/http/httpnio/pool/NioBackoffLimitedRetryJavaIntegrationTest.java

@@ -18,7 +18,7 @@
  */
 package org.jclouds.http.httpnio.pool;
 
-import org.jclouds.http.BackoffLimitedRetryJavaIntegrationTest;
+import org.jclouds.http.BackoffLimitedRetryJavaTest;
 import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
 import org.jclouds.http.httpnio.config.NioTransformingHttpCommandExecutorServiceModule;
 import org.testng.annotations.Test;
@@ -35,7 +35,7 @@ import com.google.inject.Module;
  */
 @Test(sequential = true)
 public class NioBackoffLimitedRetryJavaIntegrationTest extends
-         BackoffLimitedRetryJavaIntegrationTest {
+         BackoffLimitedRetryJavaTest {
 
    protected Module createConnectionModule() {
       return new NioTransformingHttpCommandExecutorServiceModule();

extensions/httpnio/src/test/java/org/jclouds/http/httpnio/pool/NioTransformingHttpCommandExecutorServiceTest.java

@@ -27,7 +27,7 @@ import java.util.Properties;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
 
-import org.jclouds.http.BaseHttpCommandExecutorServiceTest;
+import org.jclouds.http.BaseHttpCommandExecutorServiceIntegrationTest;
 import org.jclouds.http.httpnio.config.NioTransformingHttpCommandExecutorServiceModule;
 import org.testng.annotations.Test;
 
@@ -40,7 +40,7 @@ import com.google.inject.Module;
  */
 @Test(threadPoolSize = 10, sequential = true)
 public class NioTransformingHttpCommandExecutorServiceTest extends
-         BaseHttpCommandExecutorServiceTest {
+         BaseHttpCommandExecutorServiceIntegrationTest {
 
    @Override
    @Test(enabled = false)
@@ -66,4 +66,10 @@ public class NioTransformingHttpCommandExecutorServiceTest extends
       props.setProperty(PROPERTY_USER_THREADS, 0 + "");
    }
 
+   @Override
+   @Test(enabled = false)
+   public void testGetBigFile() throws MalformedURLException, ExecutionException,
+            InterruptedException, TimeoutException {
+      // disabled since test data is too big
+   }
 }