Improve use of ByteSource with test resources

This commit ensures proper resource cleanup, simplifies
initialization, and paves the way for purely synthetic inputs in a
subsequent commit.

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java

@@ -35,6 +35,7 @@ import java.io.InputStream;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.Map;
 import java.util.Random;
@@ -43,7 +44,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.zip.GZIPInputStream;
 
 import javax.ws.rs.core.MediaType;
 
@@ -79,43 +79,31 @@ import com.google.common.hash.HashCode;
 import com.google.common.io.ByteSource;
 import com.google.common.io.ByteStreams;
 import com.google.common.io.Files;
+import com.google.common.io.Resources;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.Uninterruptibles;
 
 public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
-   private ByteSource oneHundredOneConstitutions;
+   private static ByteSource oneHundredOneConstitutions;
-   private byte[] oneHundredOneConstitutionsMD5;
-   private static long oneHundredOneConstitutionsLength;
 
    @BeforeClass(groups = { "integration", "live" }, dependsOnMethods = "setupContext")
    @Override
    public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
       super.setUpResourcesOnThisThread(testContext);
       oneHundredOneConstitutions = getTestDataSupplier();
-      oneHundredOneConstitutionsMD5 = oneHundredOneConstitutions.hash(md5()).asBytes();
    }
 
-   @SuppressWarnings("unchecked")
    public static ByteSource getTestDataSupplier() throws IOException {
-      byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
+      return ByteSource.concat(Collections.nCopies(
-               .getResourceAsStream("/const.txt.gz")));
+            101, Resources.asByteSource(BaseJettyTest.class.getResource("/const.txt"))));
-      ByteSource constitutionSupplier = ByteSource.wrap(oneConstitution);
-
-      ByteSource temp = ByteSource.concat(constitutionSupplier);
-
-      for (int i = 0; i < 100; i++) {
-         temp = ByteSource.concat(temp, constitutionSupplier);
-      }
-      oneHundredOneConstitutionsLength = oneConstitution.length * 101l;
-      return temp;
    }
 
    public static long getOneHundredOneConstitutionsLength() throws IOException {
-      if (oneHundredOneConstitutionsLength == 0) {
+      if (oneHundredOneConstitutions == null) {
          getTestDataSupplier();
       }
-      return oneHundredOneConstitutionsLength;
+      return oneHundredOneConstitutions.size();
    }
 
    /**

core/src/test/java/org/jclouds/http/BaseHttpCommandExecutorServiceIntegrationTest.java

@@ -18,7 +18,6 @@ package org.jclouds.http;
 
 import static com.google.common.hash.Hashing.md5;
 import static com.google.common.io.BaseEncoding.base64;
-import static com.google.common.io.ByteStreams.toByteArray;
 import static com.google.common.io.Closeables.close;
 import static com.google.common.io.Files.asByteSource;
 import static org.jclouds.http.options.GetOptions.Builder.tail;
@@ -34,9 +33,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URLDecoder;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
-import java.util.zip.GZIPInputStream;
 
 import org.jclouds.io.ByteSources;
 import org.jclouds.io.Payload;
@@ -50,6 +49,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Multimap;
 import com.google.common.io.ByteSource;
 import com.google.common.io.Files;
+import com.google.common.io.Resources;
 import com.squareup.okhttp.mockwebserver.Dispatcher;
 import com.squareup.okhttp.mockwebserver.MockResponse;
 import com.squareup.okhttp.mockwebserver.MockWebServer;
@@ -67,7 +67,6 @@ public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends Base
    private static final String XML2 = "<foo><bar>chubbs</bar></foo>";
 
    private String constitutionsMd5;
-   private long constitutionsLength;
    private ByteSource oneHundredOneConstitutions;
 
    @BeforeClass(groups = "integration")
@@ -222,7 +221,7 @@ public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends Base
    public void testGetBigFile() throws Exception {
       MockResponse response = new MockResponse().addHeader("Content-MD5", constitutionsMd5)
             .addHeader("Content-type", "text/plain")
-            .setBody(oneHundredOneConstitutions.openStream(), constitutionsLength);
+            .setBody(oneHundredOneConstitutions.openStream(), oneHundredOneConstitutions.size());
 
       MockWebServer server = mockWebServer(response, response);
       InputStream input = server.getUrl("/101constitutions").openStream();
@@ -640,15 +639,8 @@ public abstract class BaseHttpCommandExecutorServiceIntegrationTest extends Base
 
    @SuppressWarnings("unchecked")
    private ByteSource getTestDataSupplier() throws IOException {
-      byte[] oneConstitution = toByteArray(new GZIPInputStream(
+      return ByteSource.concat(Collections.nCopies(
-            BaseHttpCommandExecutorServiceIntegrationTest.class.getResourceAsStream("/const.txt.gz")));
+            101, Resources.asByteSource(BaseHttpCommandExecutorServiceIntegrationTest.class.getResource("/const.txt"))));
-      ByteSource constitutionSupplier = ByteSource.wrap(oneConstitution);
-      ByteSource temp = ByteSource.concat(constitutionSupplier);
-      for (int i = 0; i < 100; i++) {
-         temp = ByteSource.concat(temp, constitutionSupplier);
-      }
-      constitutionsLength = oneConstitution.length * 101;
-      return temp;
    }
 
 }

core/src/test/java/org/jclouds/http/BaseJettyTest.java

@@ -19,7 +19,6 @@ package org.jclouds.http;
 import static com.google.common.base.Throwables.getStackTraceAsString;
 import static com.google.common.hash.Hashing.md5;
 import static com.google.common.io.BaseEncoding.base64;
-import static com.google.common.io.ByteStreams.toByteArray;
 import static com.google.common.net.HttpHeaders.CONTENT_DISPOSITION;
 import static com.google.common.net.HttpHeaders.CONTENT_ENCODING;
 import static com.google.common.net.HttpHeaders.CONTENT_LANGUAGE;
@@ -34,13 +33,13 @@ import static org.jclouds.util.Strings2.toStringAndClose;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.Enumeration;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.zip.GZIPInputStream;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -67,6 +66,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Multimap;
 import com.google.common.io.ByteSource;
+import com.google.common.io.Resources;
 import com.google.inject.Injector;
 import com.google.inject.Module;
 
@@ -245,17 +245,9 @@ public abstract class BaseJettyTest {
       server2.start();
    }
 
-   @SuppressWarnings("unchecked")
    public static ByteSource getTestDataSupplier() throws IOException {
-      byte[] oneConstitution = toByteArray(new GZIPInputStream(BaseJettyTest.class.getResourceAsStream("/const.txt.gz")));
+      return ByteSource.concat(Collections.nCopies(
-      ByteSource constitutionSupplier = ByteSource.wrap(oneConstitution);
+            101, Resources.asByteSource(BaseJettyTest.class.getResource("/const.txt"))));
-
-      ByteSource temp = ByteSource.concat(constitutionSupplier);
-
-      for (int i = 0; i < 100; i++) {
-         temp = ByteSource.concat(temp, constitutionSupplier);
-      }
-      return temp;
    }
 
    public static ContextBuilder newBuilder(int testPort, Properties properties, Module... connectionModules) {

providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java

@@ -28,9 +28,9 @@ import static org.testng.Assert.fail;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Set;
 import java.util.UUID;
-import java.util.zip.GZIPInputStream;
 
 import org.jclouds.aws.AWSResponseException;
 import org.jclouds.aws.domain.Region;
@@ -59,6 +59,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.io.ByteSource;
 import com.google.common.io.Files;
+import com.google.common.io.Resources;
 
 /**
  * Tests behavior of {@code S3Client}
@@ -70,7 +71,6 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
    }
    private ByteSource oneHundredOneConstitutions;
    private byte[] oneHundredOneConstitutionsMD5;
-   private static long oneHundredOneConstitutionsLength;
 
    @Override
    public AWSS3Client getApi() {
@@ -85,17 +85,11 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
       oneHundredOneConstitutionsMD5 = oneHundredOneConstitutions.hash(md5()).asBytes();
    }
 
-   @SuppressWarnings("unchecked")
    public static ByteSource getTestDataSupplier() throws IOException {
-      byte[] oneConstitution = toByteArray(new GZIPInputStream(BaseJettyTest.class.getResourceAsStream("/const.txt.gz")));
+      ByteSource byteSource = Resources.asByteSource(BaseJettyTest.class.getResource("/const.txt"));
-      ByteSource constitutionSupplier = ByteSource.wrap(oneConstitution);
-
-      ByteSource temp = ByteSource.concat(constitutionSupplier);
       // we have to go beyond 5MB per part
-      for (oneHundredOneConstitutionsLength = oneConstitution.length; oneHundredOneConstitutionsLength < 5 * 1024 * 1024; oneHundredOneConstitutionsLength += oneConstitution.length) {
+      int nCopies = (int) ((5 * 1024 * 1024 + 1) / byteSource.size());
-         temp = ByteSource.concat(temp, constitutionSupplier);
+      return ByteSource.concat(Collections.nCopies(nCopies, byteSource));
-      }
-      return temp;
    }
 
    public void testMultipartSynchronously() throws InterruptedException, IOException {
@@ -106,7 +100,7 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
          String uploadId = getApi().initiateMultipartUpload(containerName,
                   ObjectMetadataBuilder.create().key(key).contentMD5(oneHundredOneConstitutionsMD5).build());
          byte[] buffer = oneHundredOneConstitutions.read();
-         assertEquals(oneHundredOneConstitutionsLength, (long) buffer.length);
+         assertEquals(oneHundredOneConstitutions.size(), (long) buffer.length);
 
          Payload part1 = newByteArrayPayload(buffer);
          part1.getContentMetadata().setContentLength((long) buffer.length);