diff --git a/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java b/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java index a56e9f8a21..3d7f42b3a1 100644 --- a/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java +++ b/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java @@ -16,10 +16,15 @@ */ package org.jclouds.cloudfiles.blobstore.integration; +import java.io.IOException; + +import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.openstack.swift.blobstore.integration.SwiftBlobIntegrationLiveTest; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; + /** * * @author Adrian Cole @@ -38,4 +43,22 @@ public class CloudFilesBlobIntegrationLiveTest extends SwiftBlobIntegrationLiveT .getMetadata().getContentMetadata().getContentDisposition(); } + @Test(groups = { "integration", "live" }) + public void testChunksAreDeletedWhenMultipartBlobIsDeleted() throws IOException, InterruptedException { + String containerName = getContainerName(); + try { + BlobStore blobStore = view.getBlobStore(); + + long countBefore = blobStore.countBlobs(containerName); + String blobName = "deleteme.txt"; + addMultipartBlobToContainer(containerName, blobName); + + blobStore.removeBlob(containerName, blobName); + long countAfter = blobStore.countBlobs(containerName); + + assertEquals(countAfter, countBefore); + } finally { + returnContainer(containerName); + } + } } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java index 8c0ac705f3..2940d6f07a 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java @@ -16,8 +16,10 @@ */ package org.jclouds.openstack.swift.blobstore; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static org.jclouds.blobstore.util.BlobStoreUtils.createParentIfNeededAsync; +import static org.jclouds.openstack.swift.options.ListContainerOptions.Builder.withPrefix; import java.util.Set; @@ -25,6 +27,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import com.google.common.annotations.VisibleForTesting; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; @@ -50,8 +53,11 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; import org.jclouds.openstack.swift.domain.ContainerMetadata; +import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata; +import org.jclouds.openstack.swift.domain.ObjectInfo; import com.google.common.base.Function; +import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Iterables; @@ -118,7 +124,7 @@ public class SwiftBlobStore extends BaseBlobStore { } /** - * This implementation invokes {@link CommonSwiftClient#putBucketInRegion} + * This implementation invokes {@link CommonSwiftClient#createContainer} * * @param location * currently ignored @@ -145,7 +151,7 @@ public class SwiftBlobStore extends BaseBlobStore { } /** - * This implementation invokes {@link CommonSwiftClient#blobExists} + * This implementation invokes {@link CommonSwiftClient#objectExists} * * @param container * container name @@ -225,7 +231,53 @@ public class SwiftBlobStore extends BaseBlobStore { */ @Override public void removeBlob(String container, String key) { + String objectManifest = getObjectManifestOrNull(container, key); + sync.removeObject(container, key); + + if (!Strings.isNullOrEmpty(objectManifest)) { + removeObjectsWithPrefix(objectManifest); + } + } + + private String getObjectManifestOrNull(String container, String key) { + MutableObjectInfoWithMetadata objectInfo = sync.getObjectInfo(container, key); + return objectInfo == null ? null : objectInfo.getObjectManifest(); + } + + private void removeObjectsWithPrefix(String containerAndPrefix) { + String[] parts = splitContainerAndKey(containerAndPrefix); + + String container = parts[0]; + String prefix = parts[1]; + + removeObjectsWithPrefix(container, prefix); + } + + @VisibleForTesting + static String[] splitContainerAndKey(String containerAndKey) { + String[] parts = containerAndKey.split("/", 2); + checkArgument(parts.length == 2, + "No / separator found in \"%s\"", + containerAndKey); + return parts; + } + + private void removeObjectsWithPrefix(String container, String prefix) { + String nextMarker = null; + do { + org.jclouds.openstack.swift.options.ListContainerOptions listContainerOptions = + withPrefix(prefix); + if (nextMarker != null) { + listContainerOptions = listContainerOptions.afterMarker(nextMarker); + } + + PageSet chunks = sync.listObjects(container, listContainerOptions); + for (ObjectInfo chunk : chunks) { + sync.removeObject(container, chunk.getName()); + } + nextMarker = chunks.getNextMarker(); + } while (nextMarker != null); } @Override diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java index ee326a4d17..103292fcd7 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java @@ -48,4 +48,7 @@ public interface MutableObjectInfoWithMetadata extends ObjectInfo { Map getMetadata(); + String getObjectManifest(); + + void setObjectManifest(String objectManifest); } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java index 57d344483d..9b5768684d 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java @@ -147,4 +147,14 @@ public class DelegatingMutableObjectInfoWithMetadata extends BaseMutableContentM public URI getUri() { return delegate.getUri(); } + + @Override + public void setObjectManifest(String objectManifest) { + delegate.setObjectManifest(objectManifest); + } + + @Override + public String getObjectManifest() { + return delegate.getObjectManifest(); + } } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java index dc09819626..484888dae2 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java @@ -41,6 +41,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM private byte[] hash; private String contentType = MediaType.APPLICATION_OCTET_STREAM; private Date lastModified; + private String objectManifest; private final Map metadata = Maps.newLinkedHashMap(); /** @@ -121,6 +122,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM int result = 1; result = prime * result + ((container == null) ? 0 : container.hashCode()); result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((objectManifest == null) ? 0 : objectManifest.hashCode()); return result; } @@ -143,6 +145,11 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM return false; } else if (!name.equals(other.name)) return false; + if (objectManifest == null) { + if (other.objectManifest != null) + return false; + } else if (!objectManifest.equals(other.objectManifest)) + return false; return true; } @@ -196,10 +203,20 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM return uri; } + @Override + public String getObjectManifest() { + return objectManifest; + } + + @Override + public void setObjectManifest(String objectManifest) { + this.objectManifest = objectManifest; + } + @Override public String toString() { - return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s]", name, - container, uri, bytes, contentType, lastModified, Arrays.toString(hash)); + return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s, objectManifest=%s]", + name, container, uri, bytes, contentType, lastModified, Arrays.toString(hash), objectManifest); } } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java index bd082feb74..d9f02978a3 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java @@ -65,6 +65,8 @@ public class ParseObjectInfoFromHeaders implements Function oneHundredOneConstitutions; - private byte[] oneHundredOneConstitutionsMD5; public SwiftBlobIntegrationLiveTest() { provider = System.getProperty("test.swift.provider", "swift"); @@ -66,7 +77,6 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { super.setUpResourcesOnThisThread(testContext); oneHundredOneConstitutions = getTestDataSupplier(); - oneHundredOneConstitutionsMD5 = md5Supplier(oneHundredOneConstitutions); } @Override @@ -91,19 +101,52 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { { "asteri*k" }, { "{greaten" }, { "p|pe" } }; } + @Test(groups = { "integration", "live" }) public void testMultipartChunkedFileStream() throws IOException, InterruptedException { - Files.copy(oneHundredOneConstitutions, new File("target/const.txt")); - String containerName = getContainerName(); + String containerName = getContainerName(); + try { + BlobStore blobStore = view.getBlobStore(); + long countBefore = blobStore.countBlobs(containerName); - try { - BlobStore blobStore = view.getBlobStore(); - blobStore.createContainerInLocation(null, containerName); - Blob blob = blobStore.blobBuilder("const.txt") - .payload(new File("target/const.txt")).contentMD5(oneHundredOneConstitutionsMD5).build(); - blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart()); - } finally { - returnContainer(containerName); - } + addMultipartBlobToContainer(containerName, "const.txt"); + + long countAfter = blobStore.countBlobs(containerName); + assertNotEquals(countBefore, countAfter, + "No blob was created"); + assertTrue(countAfter - countBefore > 1, + "A multipart blob wasn't actually created - " + + "there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs"); + } finally { + returnContainer(containerName); + } + } + + protected void addMultipartBlobToContainer(String containerName, String key) throws IOException { + File fileToUpload = createFileBiggerThan(PART_SIZE); + + BlobStore blobStore = view.getBlobStore(); + blobStore.createContainerInLocation(null, containerName); + Blob blob = blobStore.blobBuilder(key) + .payload(fileToUpload) + .build(); + blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart()); + } + + @SuppressWarnings("unchecked") + private File createFileBiggerThan(long partSize) throws IOException { + long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1; + + InputSupplier temp = ByteStreams.join(oneHundredOneConstitutions); + + for (int i = 0; i < copiesNeeded; i++) { + temp = ByteStreams.join(temp, oneHundredOneConstitutions); + } + + File fileToUpload = new File("target/lots-of-const.txt"); + Files.copy(temp, fileToUpload); + + assertTrue(fileToUpload.length() > partSize); + return fileToUpload; } @Override diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index d96393efef..5e5d3c9ec9 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -122,6 +122,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { return temp; } + public static long getOneHundredOneConstitutionsLength() throws IOException { + if (oneHundredOneConstitutionsLength == 0) { + getTestDataSupplier(); + } + return oneHundredOneConstitutionsLength; + } + /** * Attempt to capture the issue detailed in * http://groups.google.com/group/jclouds/browse_thread/thread/4a7c8d58530b287f