From 91c47bfd92c7fa20ee10c5b6fa8df74acf128c00 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Wed, 7 Jan 2015 15:07:56 -0800 Subject: [PATCH] JCLOUDS-801: Add portable multipart upload tests Exercise both repeatable and non-repeatable payloads. Tested against AWS-S3, Azure, and legacy Swift. Skipped on all other providers. --- .../SwiftBlobIntegrationLiveTest.java | 5 ++ .../internal/BaseBlobIntegrationTest.java | 77 +++++++++++++++++++ .../AWSS3BlobIntegrationLiveTest.java | 15 ++++ .../AzureBlobIntegrationLiveTest.java | 5 ++ 4 files changed, 102 insertions(+) diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java index 36dba5643e..a0e42411f7 100644 --- a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java @@ -65,6 +65,11 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { provider = System.getProperty("test.swift.provider", "swift"); } + @Override + protected long getMinimumMultipartBlobSize() { + return PART_SIZE + 1; + } + @Override @Test(enabled = false) public void testGetTwoRanges() { diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index dca61a1113..d890a17d33 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -18,6 +18,7 @@ package org.jclouds.blobstore.integration.internal; import static com.google.common.base.Charsets.UTF_8; import static com.google.common.hash.Hashing.md5; +import static org.assertj.core.api.Assertions.assertThat; import static org.jclouds.blobstore.options.GetOptions.Builder.ifETagDoesntMatch; import static org.jclouds.blobstore.options.GetOptions.Builder.ifETagMatches; import static org.jclouds.blobstore.options.GetOptions.Builder.ifModifiedSince; @@ -32,6 +33,7 @@ import static org.testng.Assert.fail; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.Date; @@ -53,15 +55,19 @@ import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageType; +import org.jclouds.blobstore.options.PutOptions; import org.jclouds.crypto.Crypto; import org.jclouds.encryption.internal.JCECrypto; import org.jclouds.http.HttpResponseException; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; import org.jclouds.io.payloads.ByteSourcePayload; +import org.jclouds.io.payloads.InputStreamPayload; import org.jclouds.logging.Logger; +import org.jclouds.util.Closeables2; import org.jclouds.utils.TestUtils; import org.testng.ITestContext; +import org.testng.SkipException; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -517,6 +523,77 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { } } + @Test(groups = { "integration", "live" }) + public void testPutByteSource() throws Exception { + long length = 42; + ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); + Payload payload = new ByteSourcePayload(byteSource); + testPut(payload, payload, length, new PutOptions()); + } + + @Test(groups = { "integration", "live" }) + public void testPutInputStream() throws Exception { + long length = 42; + ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); + Payload payload = new InputStreamPayload(byteSource.openStream()); + testPut(payload, new ByteSourcePayload(byteSource), length, new PutOptions()); + } + + @Test(groups = { "integration", "live" }) + public void testPutMultipartByteSource() throws Exception { + long length = getMinimumMultipartBlobSize(); + ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); + Payload payload = new ByteSourcePayload(byteSource); + testPut(payload, payload, length, new PutOptions().multipart(true)); + } + + @Test(groups = { "integration", "live" }) + public void testPutMultipartInputStream() throws Exception { + long length = getMinimumMultipartBlobSize(); + ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); + Payload payload = new InputStreamPayload(byteSource.openStream()); + testPut(payload, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true)); + } + + private void testPut(Payload payload, Payload expectedPayload, long length, PutOptions options) + throws IOException, InterruptedException { + BlobStore blobStore = view.getBlobStore(); + String blobName = "multipart-upload"; + Map userMetadata = ImmutableMap.of("key1", "value1", "key2", "value2"); + PayloadBlobBuilder blobBuilder = blobStore.blobBuilder(blobName) + .userMetadata(userMetadata) + .payload(payload) + .contentLength(length); + addContentMetadata(blobBuilder); + + String container = getContainerName(); + try { + String etag = blobStore.putBlob(container, blobBuilder.build(), options); + assertThat(etag).isNotNull(); + + Blob blob = blobStore.getBlob(container, blobName); + InputStream is = null; + try { + is = blob.getPayload().openStream(); + assertThat(is).hasContentEqualTo(expectedPayload.openStream()); + } finally { + Closeables2.closeQuietly(is); + } + validateMetadata(blob.getMetadata(), container, blob.getMetadata().getName()); + checkContentMetadata(blob); + assertThat(blob.getMetadata().getUserMetadata()).isEqualTo(userMetadata); + + PageSet set = blobStore.list(container); + assertThat(set).hasSize(1); + } finally { + returnContainer(container); + } + } + + protected long getMinimumMultipartBlobSize() { + throw new SkipException("multipart upload not supported"); + } + protected void checkContentMetadata(Blob blob) { checkContentType(blob, "text/csv"); checkContentDisposition(blob, "attachment; filename=photo.jpg"); diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3BlobIntegrationLiveTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3BlobIntegrationLiveTest.java index cba1f72536..775926897b 100644 --- a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3BlobIntegrationLiveTest.java +++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/integration/AWSS3BlobIntegrationLiveTest.java @@ -16,6 +16,9 @@ */ package org.jclouds.aws.s3.blobstore.integration; +import java.util.Properties; + +import org.jclouds.aws.s3.blobstore.strategy.MultipartUpload; import org.jclouds.s3.blobstore.integration.S3BlobIntegrationLiveTest; import org.testng.annotations.Test; @@ -24,4 +27,16 @@ public class AWSS3BlobIntegrationLiveTest extends S3BlobIntegrationLiveTest { public AWSS3BlobIntegrationLiveTest() { provider = "aws-s3"; } + + @Override + protected Properties setupProperties() { + Properties props = super.setupProperties(); + props.setProperty("jclouds.mpu.parts.size", String.valueOf(MultipartUpload.MIN_PART_SIZE)); + return props; + } + + @Override + protected long getMinimumMultipartBlobSize() { + return MultipartUpload.MIN_PART_SIZE + 1; + } } diff --git a/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/integration/AzureBlobIntegrationLiveTest.java b/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/integration/AzureBlobIntegrationLiveTest.java index dea1a57c85..eabd81b648 100644 --- a/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/integration/AzureBlobIntegrationLiveTest.java +++ b/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/integration/AzureBlobIntegrationLiveTest.java @@ -45,6 +45,11 @@ public class AzureBlobIntegrationLiveTest extends BaseBlobIntegrationTest { private ByteSource oneHundredOneConstitutions; private byte[] oneHundredOneConstitutionsMD5; + @Override + protected long getMinimumMultipartBlobSize() { + return MultipartUploadStrategy.MAX_BLOCK_SIZE + 1; + } + public AzureBlobIntegrationLiveTest() { provider = "azureblob"; }