From a8b106c2bb3e3e7e292cd80f3e56a5f56a51742f Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Wed, 9 Jul 2014 17:55:31 -0700 Subject: [PATCH] JCLOUDS-622: remove calls to InputSupplier methods Guava 16 deprecated these methods and Guava 18 will remove them. --- .../FilesystemAsyncBlobStoreTest.java | 4 +- .../SwiftBlobIntegrationLiveTest.java | 8 ++- .../blobstore/LocalAsyncBlobStore.java | 4 +- .../internal/BaseBlobIntegrationTest.java | 8 +-- .../JavaUrlHttpCommandExecutorService.java | 6 +- .../java/org/jclouds/io/ByteStreams2.java | 26 +++++++++ .../jclouds/io/payloads/MultipartForm.java | 57 +++++++++++-------- .../main/java/org/jclouds/util/Strings2.java | 3 +- .../io/internal/BasePayloadSlicerTest.java | 3 +- .../ApacheHCHttpCommandExecutorService.java | 4 +- .../jclouds/aws/s3/AWSS3ClientLiveTest.java | 4 +- 11 files changed, 85 insertions(+), 42 deletions(-) diff --git a/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java b/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java index a3845f8a2f..082e13c944 100644 --- a/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java +++ b/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java @@ -49,6 +49,7 @@ import org.jclouds.filesystem.reference.FilesystemConstants; import org.jclouds.filesystem.util.Utils; import org.jclouds.filesystem.utils.TestUtils; import org.jclouds.http.HttpRequest; +import org.jclouds.io.ByteStreams2; import org.jclouds.io.Payload; import org.jclouds.io.payloads.PhantomPayload; import org.jclouds.io.payloads.StringPayload; @@ -60,7 +61,6 @@ import org.testng.annotations.Test; import com.google.common.collect.Sets; import com.google.common.io.ByteSource; -import com.google.common.io.ByteStreams; import com.google.common.io.Files; import com.google.inject.CreationException; @@ -583,7 +583,7 @@ public class FilesystemAsyncBlobStoreTest { assertNotNull(resultBlob, "Blob exists"); // checks file content ByteSource expectedFile = Files.asByteSource(new File(TARGET_CONTAINER_NAME, blobKey)); - assertTrue(ByteStreams.equal(expectedFile, resultBlob.getPayload()), + assertEquals(expectedFile.read(), ByteStreams2.toByteArrayAndClose(resultBlob.getPayload().openStream()), "Blob payload differs from file content"); // metadata are verified in the test for blobMetadata, so no need to // perform a complete test here diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java index ecd237008d..9882efae68 100644 --- a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java @@ -43,6 +43,7 @@ import org.testng.annotations.Test; import com.google.common.hash.Hashing; import com.google.common.io.ByteSource; +import com.google.common.io.Closeables; import com.google.common.io.Files; @Test(groups = "live") @@ -178,7 +179,12 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { blobStore.putBlob(container, write, PutOptions.Builder.multipart()); Blob read = blobStore.getBlob(container, "const.txt"); - Files.copy(read.getPayload(), outFile); + InputStream is = read.getPayload().openStream(); + try { + Files.asByteSink(outFile).writeFrom(is); + } finally { + Closeables.closeQuietly(is); + } assertEquals(Files.hash(outFile, Hashing.md5()), Files.hash(inFile, Hashing.md5())); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java index ba987ab0af..88117e42fb 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java @@ -25,7 +25,6 @@ import static com.google.common.collect.Iterables.size; import static com.google.common.collect.Iterables.transform; import static com.google.common.collect.Sets.filter; import static com.google.common.collect.Sets.newTreeSet; -import static com.google.common.io.ByteStreams.toByteArray; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; @@ -66,6 +65,7 @@ import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpResponseException; import org.jclouds.http.HttpUtils; +import org.jclouds.io.ByteStreams2; import org.jclouds.io.ContentMetadata; import org.jclouds.io.ContentMetadataCodec; import org.jclouds.io.Payload; @@ -467,7 +467,7 @@ public class LocalAsyncBlobStore extends BaseAsyncBlobStore { if (options.getRanges() != null && options.getRanges().size() > 0) { byte[] data; try { - data = toByteArray(blob.getPayload()); + data = ByteStreams2.toByteArrayAndClose(blob.getPayload().openStream()); } catch (IOException e) { return immediateFailedFuture(new RuntimeException(e)); } diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index 8d8f379b6a..e563d7bc69 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -57,6 +57,7 @@ import org.jclouds.crypto.Crypto; import org.jclouds.encryption.internal.JCECrypto; import org.jclouds.http.BaseJettyTest; import org.jclouds.http.HttpResponseException; +import org.jclouds.io.ByteStreams2; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; import org.jclouds.io.payloads.ByteSourcePayload; @@ -75,7 +76,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.hash.HashCode; import com.google.common.io.ByteSource; -import com.google.common.io.ByteStreams; import com.google.common.io.Files; import com.google.common.io.Resources; import com.google.common.util.concurrent.Futures; @@ -115,7 +115,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { createTestInput(32 * 1024).copyTo(Files.asByteSink(payloadFile)); final Payload testPayload = Payloads.newFilePayload(payloadFile); - final HashCode md5 = ByteStreams.hash(testPayload, md5()); + final HashCode md5 = ByteStreams2.hashAndClose(testPayload.openStream(), md5()); testPayload.getContentMetadata().setContentType("image/png"); final AtomicInteger blobCount = new AtomicInteger(); @@ -134,7 +134,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { assertConsistencyAwareBlobExists(container, name); blob = view.getBlobStore().getBlob(container, name); - assertEquals(ByteStreams.hash(blob.getPayload(), md5()), md5, + assertEquals(ByteStreams2.hashAndClose(blob.getPayload().openStream(), md5()), md5, String.format("md5 didn't match on %s/%s", container, name)); view.getBlobStore().removeBlob(container, name); @@ -173,7 +173,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { public Void apply(Blob from) { try { validateMetadata(from.getMetadata(), container, name); - assertEquals(ByteStreams.hash(from.getPayload(), md5()), supplier.hash(md5())); + assertEquals(ByteStreams2.hashAndClose(from.getPayload().openStream(), md5()), supplier.hash(md5())); checkContentDisposition(from, expectedContentDisposition); } catch (IOException e) { Throwables.propagate(e); diff --git a/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java b/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java index 8d665fe8fc..7a34c31334 100644 --- a/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java +++ b/core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java @@ -61,6 +61,7 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableMultimap.Builder; import com.google.common.io.ByteStreams; +import com.google.common.io.Closeables; import com.google.common.io.CountingOutputStream; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.inject.Inject; @@ -302,11 +303,14 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe void writePayloadToConnection(Payload payload, Object lengthDesc, HttpURLConnection connection) throws IOException { connection.setDoOutput(true); CountingOutputStream out = new CountingOutputStream(connection.getOutputStream()); + InputStream is = payload.openStream(); try { - ByteStreams.copy(payload, out); + ByteStreams.copy(is, out); } catch (IOException e) { logger.error(e, "error after writing %d/%s bytes to %s", out.getCount(), lengthDesc, connection.getURL()); throw e; + } finally { + Closeables.closeQuietly(is); } } diff --git a/core/src/main/java/org/jclouds/io/ByteStreams2.java b/core/src/main/java/org/jclouds/io/ByteStreams2.java index be3a67ff5d..4f4f123889 100644 --- a/core/src/main/java/org/jclouds/io/ByteStreams2.java +++ b/core/src/main/java/org/jclouds/io/ByteStreams2.java @@ -22,11 +22,37 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.InputStream; import com.google.common.annotations.Beta; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.HashingInputStream; import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; +import com.google.common.io.Closeables; import com.google.common.io.InputSupplier; @Beta public class ByteStreams2 { + public static HashCode hashAndClose(InputStream input, HashFunction hashFunction) throws IOException { + checkNotNull(input, "input"); + checkNotNull(hashFunction, "hashFunction"); + try { + HashingInputStream his = new HashingInputStream(hashFunction, input); + ByteStreams.copy(his, ByteStreams.nullOutputStream()); + return his.hash(); + } finally { + Closeables.closeQuietly(input); + } + } + + public static byte[] toByteArrayAndClose(InputStream input) throws IOException { + checkNotNull(input, "input"); + try { + return ByteStreams.toByteArray(input); + } finally { + Closeables.closeQuietly(input); + } + } + @Deprecated public static ByteSource asByteSource(final InputSupplier supplier) { checkNotNull(supplier, "supplier"); diff --git a/core/src/main/java/org/jclouds/io/payloads/MultipartForm.java b/core/src/main/java/org/jclouds/io/payloads/MultipartForm.java index 61b678c26f..94d4112cf4 100644 --- a/core/src/main/java/org/jclouds/io/payloads/MultipartForm.java +++ b/core/src/main/java/org/jclouds/io/payloads/MultipartForm.java @@ -17,40 +17,45 @@ package org.jclouds.io.payloads; import static com.google.common.collect.Lists.newArrayList; -import static com.google.common.io.ByteStreams.join; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.Collections; import java.util.Map.Entry; -import com.google.common.io.ByteSource; -import com.google.common.io.InputSupplier; +import com.google.common.collect.ImmutableList; public class MultipartForm extends BasePayload> { public static final String BOUNDARY = "--JCLOUDS--"; private static final String rn = "\r\n"; private static final String dd = "--"; - private boolean isRepeatable; - private final InputSupplier chain; + private final String boundary; + private final Iterable content; + private final boolean isRepeatable; @SuppressWarnings("unchecked") public MultipartForm(String boundary, Iterable content) { super(content); + this.boundary = boundary; + this.content = content; + getContentMetadata().setContentType("multipart/form-data; boundary=" + boundary); - getContentMetadata().setContentLength(0l); String boundaryrn = boundary + rn; - isRepeatable = true; - InputSupplier chain = join(); + boolean isRepeatable = true; + long contentLength = 0; for (Part part : content) { if (!part.isRepeatable()) isRepeatable = false; - getContentMetadata().setContentLength( - getContentMetadata().getContentLength() + part.getContentMetadata().getContentLength()); - chain = join(chain, addLengthAndReturnHeaders(boundaryrn, part), part, addLengthAndReturnRn()); + contentLength += part.getContentMetadata().getContentLength() + + createHeaders(boundaryrn, part).length() + + createRn().length(); } - chain = join(chain, addLengthAndReturnFooter(boundary)); - this.chain = chain; + contentLength += createFooter(boundary).length(); + getContentMetadata().setContentLength(contentLength); + this.isRepeatable = isRepeatable; } public MultipartForm(String boundary, Part... parts) { @@ -61,31 +66,35 @@ public class MultipartForm extends BasePayload> { this(BOUNDARY, parts); } - private ByteSource addLengthAndReturnRn() { - getContentMetadata().setContentLength(getContentMetadata().getContentLength() + rn.length()); - return ByteSource.wrap(rn.getBytes()); + private String createRn() { + return rn; } - private ByteSource addLengthAndReturnHeaders(String boundaryrn, Part part) { + private String createHeaders(String boundaryrn, Part part) { StringBuilder builder = new StringBuilder(dd).append(boundaryrn); for (Entry entry : part.getHeaders().entries()) { String header = String.format("%s: %s%s", entry.getKey(), entry.getValue(), rn); builder.append(header); } builder.append(rn); - getContentMetadata().setContentLength(getContentMetadata().getContentLength() + builder.length()); - return ByteSource.wrap(builder.toString().getBytes()); + return builder.toString(); } - private ByteSource addLengthAndReturnFooter(String boundary) { - String end = dd + boundary + dd + rn; - getContentMetadata().setContentLength(getContentMetadata().getContentLength() + end.length()); - return ByteSource.wrap(end.getBytes()); + private static String createFooter(String boundary) { + return dd + boundary + dd + rn; } @Override public InputStream openStream() throws IOException { - return chain.getInput(); + String boundaryrn = boundary + rn; + ImmutableList.Builder builder = ImmutableList.builder(); + for (Part part : content) { + builder.add(new ByteArrayInputStream(createHeaders(boundaryrn, part).getBytes())) + .add(part.openStream()) + .add(new ByteArrayInputStream(createRn().getBytes())); + } + builder.add(new ByteArrayInputStream(createFooter(boundary).getBytes())); + return new SequenceInputStream(Collections.enumeration(builder.build())); } @Override diff --git a/core/src/main/java/org/jclouds/util/Strings2.java b/core/src/main/java/org/jclouds/util/Strings2.java index 82c26312b5..afaac305cd 100644 --- a/core/src/main/java/org/jclouds/util/Strings2.java +++ b/core/src/main/java/org/jclouds/util/Strings2.java @@ -126,8 +126,7 @@ public class Strings2 { @Deprecated public static String toString(InputSupplier supplier) throws IOException { - return CharStreams.toString(CharStreams.newReaderSupplier(supplier, - Charsets.UTF_8)); + return toStringAndClose(supplier.getInput()); } public static String toStringAndClose(InputStream input) throws IOException { diff --git a/core/src/test/java/org/jclouds/io/internal/BasePayloadSlicerTest.java b/core/src/test/java/org/jclouds/io/internal/BasePayloadSlicerTest.java index 17e4076ac7..8f42296a75 100644 --- a/core/src/test/java/org/jclouds/io/internal/BasePayloadSlicerTest.java +++ b/core/src/test/java/org/jclouds/io/internal/BasePayloadSlicerTest.java @@ -35,7 +35,6 @@ import org.testng.annotations.Test; import com.google.common.base.Charsets; import com.google.common.collect.Iterables; import com.google.common.io.ByteSource; -import com.google.common.io.ByteStreams; @Test public class BasePayloadSlicerTest { @@ -49,7 +48,7 @@ public class BasePayloadSlicerTest { Iterator iter = slicer.slice(payload, 25).iterator(); assertTrue(iter.hasNext(), "Not enough results"); - assertEquals(new String(ByteStreams.toByteArray(iter.next())), contents); + assertEquals(Strings2.toStringAndClose(iter.next().openStream()), contents); assertFalse(iter.hasNext()); } diff --git a/drivers/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorService.java b/drivers/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorService.java index fac5592123..8448a55643 100644 --- a/drivers/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorService.java +++ b/drivers/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorService.java @@ -39,12 +39,12 @@ import org.jclouds.http.handlers.DelegatingRetryHandler; import org.jclouds.http.internal.BaseHttpCommandExecutorService; import org.jclouds.http.internal.HttpWire; import org.jclouds.io.ContentMetadataCodec; +import org.jclouds.io.ByteStreams2; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; -import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.inject.Inject; @@ -69,7 +69,7 @@ public class ApacheHCHttpCommandExecutorService extends BaseHttpCommandExecutorS protected HttpUriRequest convert(HttpRequest request) throws IOException { HttpUriRequest returnVal = apacheHCUtils.convertToApacheRequest(request); if (request.getPayload() != null && request.getPayload().getContentMetadata().getContentMD5() != null) { - String md5 = base64().encode(ByteStreams.hash(request.getPayload(), md5()).asBytes()); + String md5 = base64().encode(ByteStreams2.hashAndClose(request.getPayload().openStream(), md5()).asBytes()); returnVal.addHeader("Content-MD5", md5); } diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java index 8744ea1e2f..dbf2fe4b69 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java @@ -17,7 +17,6 @@ package org.jclouds.aws.s3; import static com.google.common.hash.Hashing.md5; -import static com.google.common.io.ByteStreams.toByteArray; import static org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions.Builder.storageClass; import static org.jclouds.io.Payloads.newByteArrayPayload; import static org.jclouds.s3.options.ListBucketOptions.Builder.withPrefix; @@ -43,6 +42,7 @@ import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.domain.Location; import org.jclouds.http.BaseJettyTest; +import org.jclouds.io.ByteStreams2; import org.jclouds.io.Payload; import org.jclouds.s3.S3Client; import org.jclouds.s3.S3ClientLiveTest; @@ -123,7 +123,7 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest { assert !eTagOf1.equals(eTag); object = getApi().getObject(containerName, key); - assertEquals(toByteArray(object.getPayload()), buffer); + assertEquals(ByteStreams2.toByteArrayAndClose(object.getPayload().openStream()), buffer); // noticing amazon does not return content-md5 header or a parsable ETag after a multi-part // upload is complete: