JCLOUDS-251: Swift: Delete chunks when deleting a multipart blob

Also:
- Make SwiftBlobIntegrationLiveTest.testMultipartChunkedFileStream more realistic by uploading a file large enough to be split into parts.
- JavaDoc fixes for SwiftBlobStore: don't reference nonexistent methods.

apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java

@@ -16,10 +16,15 @@
  */
 package org.jclouds.cloudfiles.blobstore.integration;
 
+import java.io.IOException;
+
+import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.openstack.swift.blobstore.integration.SwiftBlobIntegrationLiveTest;
 import org.testng.annotations.Test;
 
+import static org.testng.Assert.assertEquals;
+
 /**
  * 
  * @author Adrian Cole
@@ -38,4 +43,22 @@ public class CloudFilesBlobIntegrationLiveTest extends SwiftBlobIntegrationLiveT
                .getMetadata().getContentMetadata().getContentDisposition();
    }
 
+   @Test(groups = { "integration", "live" })
+   public void testChunksAreDeletedWhenMultipartBlobIsDeleted() throws IOException, InterruptedException {
+      String containerName = getContainerName();
+      try {
+         BlobStore blobStore = view.getBlobStore();
+
+         long countBefore = blobStore.countBlobs(containerName);
+         String blobName = "deleteme.txt";
+         addMultipartBlobToContainer(containerName, blobName);
+
+         blobStore.removeBlob(containerName, blobName);
+         long countAfter = blobStore.countBlobs(containerName);
+
+         assertEquals(countAfter, countBefore);
+      } finally {
+          returnContainer(containerName);
+      }
+   }
 }

apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java

@@ -16,8 +16,10 @@
  */
 package org.jclouds.openstack.swift.blobstore;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.jclouds.blobstore.util.BlobStoreUtils.createParentIfNeededAsync;
+import static org.jclouds.openstack.swift.options.ListContainerOptions.Builder.withPrefix;
 
 import java.util.Set;
 
@@ -25,6 +27,7 @@ import javax.inject.Inject;
 import javax.inject.Provider;
 import javax.inject.Singleton;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
@@ -50,8 +53,11 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
 import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy;
 import org.jclouds.openstack.swift.domain.ContainerMetadata;
+import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata;
+import org.jclouds.openstack.swift.domain.ObjectInfo;
 
 import com.google.common.base.Function;
+import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
 import com.google.common.collect.Iterables;
 
@@ -118,7 +124,7 @@ public class SwiftBlobStore extends BaseBlobStore {
    }
 
    /**
-    * This implementation invokes {@link CommonSwiftClient#putBucketInRegion}
+    * This implementation invokes {@link CommonSwiftClient#createContainer}
     * 
     * @param location
     *           currently ignored
@@ -145,7 +151,7 @@ public class SwiftBlobStore extends BaseBlobStore {
    }
 
    /**
-    * This implementation invokes {@link CommonSwiftClient#blobExists}
+    * This implementation invokes {@link CommonSwiftClient#objectExists}
     * 
     * @param container
     *           container name
@@ -225,7 +231,53 @@ public class SwiftBlobStore extends BaseBlobStore {
     */
    @Override
    public void removeBlob(String container, String key) {
+      String objectManifest = getObjectManifestOrNull(container, key);
+
       sync.removeObject(container, key);
+
+      if (!Strings.isNullOrEmpty(objectManifest)) {
+         removeObjectsWithPrefix(objectManifest);
+      }
+   }
+
+   private String getObjectManifestOrNull(String container, String key) {
+      MutableObjectInfoWithMetadata objectInfo = sync.getObjectInfo(container, key);
+      return objectInfo == null ? null : objectInfo.getObjectManifest();
+   }
+
+   private void removeObjectsWithPrefix(String containerAndPrefix) {
+      String[] parts = splitContainerAndKey(containerAndPrefix);
+
+      String container = parts[0];
+      String prefix = parts[1];
+
+      removeObjectsWithPrefix(container, prefix);
+   }
+
+   @VisibleForTesting
+   static String[] splitContainerAndKey(String containerAndKey) {
+      String[] parts = containerAndKey.split("/", 2);
+      checkArgument(parts.length == 2,
+                    "No / separator found in \"%s\"",
+                    containerAndKey);
+      return parts;
+   }
+
+   private void removeObjectsWithPrefix(String container, String prefix) {
+      String nextMarker = null;
+      do {
+         org.jclouds.openstack.swift.options.ListContainerOptions listContainerOptions =
+            withPrefix(prefix);
+         if (nextMarker != null) {
+            listContainerOptions = listContainerOptions.afterMarker(nextMarker);
+         }
+
+         PageSet<ObjectInfo> chunks = sync.listObjects(container, listContainerOptions);
+         for (ObjectInfo chunk : chunks) {
+            sync.removeObject(container, chunk.getName());
+         }
+         nextMarker = chunks.getNextMarker();
+      } while (nextMarker != null);
    }
 
    @Override

apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java

@@ -48,4 +48,7 @@ public interface MutableObjectInfoWithMetadata extends ObjectInfo {
 
    Map<String, String> getMetadata();
 
+   String getObjectManifest();
+
+   void setObjectManifest(String objectManifest);
 }

apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java

@@ -147,4 +147,14 @@ public class DelegatingMutableObjectInfoWithMetadata extends BaseMutableContentM
    public URI getUri() {
       return delegate.getUri();
    }
+
+   @Override
+   public void setObjectManifest(String objectManifest) {
+      delegate.setObjectManifest(objectManifest);
+   }
+
+   @Override
+   public String getObjectManifest() {
+      return delegate.getObjectManifest();
+   }
 }

apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java

@@ -41,6 +41,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
    private byte[] hash;
    private String contentType = MediaType.APPLICATION_OCTET_STREAM;
    private Date lastModified;
+   private String objectManifest;
    private final Map<String, String> metadata = Maps.newLinkedHashMap();
 
    /**
@@ -121,6 +122,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
       int result = 1;
       result = prime * result + ((container == null) ? 0 : container.hashCode());
       result = prime * result + ((name == null) ? 0 : name.hashCode());
+      result = prime * result + ((objectManifest == null) ? 0 : objectManifest.hashCode());
       return result;
    }
 
@@ -143,6 +145,11 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
             return false;
       } else if (!name.equals(other.name))
          return false;
+      if (objectManifest == null) {
+         if (other.objectManifest != null)
+            return false;
+      } else if (!objectManifest.equals(other.objectManifest))
+         return false;
       return true;
    }
 
@@ -196,10 +203,20 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
       return uri;
    }
 
+   @Override
+   public String getObjectManifest() {
+      return objectManifest;
+   }
+
+   @Override
+   public void setObjectManifest(String objectManifest) {
+      this.objectManifest = objectManifest;
+   }
+
    @Override
    public String toString() {
-      return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s]", name,
+      return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s, objectManifest=%s]",
-               container, uri, bytes, contentType, lastModified, Arrays.toString(hash));
+               name, container, uri, bytes, contentType, lastModified, Arrays.toString(hash), objectManifest);
    }
 
 }

apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java

@@ -65,6 +65,8 @@ public class ParseObjectInfoFromHeaders implements Function<HttpResponse, Mutabl
       if (eTagHeader != null) {
          to.setHash(ETagUtils.convertHexETagToByteArray(eTagHeader));
       }
+      to.setObjectManifest(from.getFirstHeaderOrNull("X-Object-Manifest"));
+
       return to;
    }
 

apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.blobstore;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+@Test(groups = "unit")
+public class SwiftBlobStoreTest {
+   @Test
+   public void testSplitContainerAndKey() {
+      String container = "test-container";
+      String key = "key/with/some/slashes/in/it/and/a/trailing/slash/";
+
+      String containerAndKey = container + "/" + key;
+
+      String[] split = SwiftBlobStore.splitContainerAndKey(containerAndKey);
+      String actualContainer = split[0];
+      String actualKey = split[1];
+
+      assertEquals(actualContainer, container);
+      assertEquals(actualKey, key);
+   }
+
+   @Test(expectedExceptions = IllegalArgumentException.class,
+         expectedExceptionsMessageRegExp = "No / separator found in \"not-a-container-and-key\"")
+   public void testSplitContainerAndKeyWithNoSeparator() {
+      SwiftBlobStore.splitContainerAndKey("not-a-container-and-key");
+   }
+}

apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java

@@ -21,11 +21,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.Properties;
 
+import com.google.common.io.ByteStreams;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
 import org.jclouds.blobstore.options.PutOptions;
 import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
+import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
 import org.testng.ITestContext;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
@@ -34,6 +36,9 @@ import org.testng.annotations.Test;
 import com.google.common.io.Files;
 import com.google.common.io.InputSupplier;
 
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertTrue;
+
 /**
  * 
  * @author James Murty
@@ -41,15 +46,21 @@ import com.google.common.io.InputSupplier;
  */
 @Test(groups = "live")
 public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
+   /**
+    * Use the minimum part size to minimise the file size that we have to
+    * upload to get a multipart blob thereby make the test run faster
+    */
+   private static final long PART_SIZE = MultipartUpload.MIN_PART_SIZE;
+
    @Override
    protected Properties setupProperties() {
       Properties props = super.setupProperties();
       setIfTestSystemPropertyPresent(props, KeystoneProperties.CREDENTIAL_TYPE);
+      props.setProperty("jclouds.mpu.parts.size", String.valueOf(PART_SIZE));
       return props;
    }
    
    private InputSupplier<InputStream> oneHundredOneConstitutions;
-   private byte[] oneHundredOneConstitutionsMD5;
 
    public SwiftBlobIntegrationLiveTest() {
       provider = System.getProperty("test.swift.provider", "swift");
@@ -66,7 +77,6 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
     public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
         super.setUpResourcesOnThisThread(testContext);
         oneHundredOneConstitutions = getTestDataSupplier();
-        oneHundredOneConstitutionsMD5 = md5Supplier(oneHundredOneConstitutions);
     }
 
    @Override
@@ -91,19 +101,52 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
                { "asteri*k" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
    }
     
+   @Test(groups = { "integration", "live" })
    public void testMultipartChunkedFileStream() throws IOException, InterruptedException {
-       Files.copy(oneHundredOneConstitutions, new File("target/const.txt"));
+      String containerName = getContainerName();
-       String containerName = getContainerName();
+      try {
+         BlobStore blobStore = view.getBlobStore();
+         long countBefore = blobStore.countBlobs(containerName);
 
-       try {
+         addMultipartBlobToContainer(containerName, "const.txt");
-           BlobStore blobStore = view.getBlobStore();
+
-           blobStore.createContainerInLocation(null, containerName);
+         long countAfter = blobStore.countBlobs(containerName);
-           Blob blob = blobStore.blobBuilder("const.txt")
+         assertNotEquals(countBefore, countAfter,
-                   .payload(new File("target/const.txt")).contentMD5(oneHundredOneConstitutionsMD5).build();
+                         "No blob was created");
-           blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
+         assertTrue(countAfter - countBefore > 1,
-       } finally {
+                    "A multipart blob wasn't actually created - " +
-           returnContainer(containerName);
+                    "there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs");
-       }
+      } finally {
+          returnContainer(containerName);
+      }
+   }
+
+   protected void addMultipartBlobToContainer(String containerName, String key) throws IOException {
+      File fileToUpload = createFileBiggerThan(PART_SIZE);
+
+      BlobStore blobStore = view.getBlobStore();
+      blobStore.createContainerInLocation(null, containerName);
+      Blob blob = blobStore.blobBuilder(key)
+         .payload(fileToUpload)
+         .build();
+      blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
+   }
+
+   @SuppressWarnings("unchecked")
+   private File createFileBiggerThan(long partSize) throws IOException {
+      long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1;
+
+      InputSupplier<InputStream> temp = ByteStreams.join(oneHundredOneConstitutions);
+
+      for (int i = 0; i < copiesNeeded; i++) {
+         temp = ByteStreams.join(temp, oneHundredOneConstitutions);
+      }
+
+      File fileToUpload = new File("target/lots-of-const.txt");
+      Files.copy(temp, fileToUpload);
+
+      assertTrue(fileToUpload.length() > partSize);
+      return fileToUpload;
    }
 
    @Override

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java

@@ -122,6 +122,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
       return temp;
    }
 
+   public static long getOneHundredOneConstitutionsLength() throws IOException {
+      if (oneHundredOneConstitutionsLength == 0) {
+         getTestDataSupplier();
+      }
+      return oneHundredOneConstitutionsLength;
+   }
+
    /**
     * Attempt to capture the issue detailed in
     * http://groups.google.com/group/jclouds/browse_thread/thread/4a7c8d58530b287f