Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-251: Swift: Delete chunks when deleting a multipart blob 

Also:
- Make SwiftBlobIntegrationLiveTest.testMultipartChunkedFileStream more realistic by uploading a file large enough to be split into parts.
- JavaDoc fixes for SwiftBlobStore: don't reference nonexistent methods.
This commit is contained in:
Francis Devereux 2013-08-22 15:27:25 +01:00 committed by Everett Toews
parent d05e77b8b6
commit d60d2681d1
9 changed files with 219 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java
View File

@ -16,10 +16,15 @@
*/
package org.jclouds.cloudfiles.blobstore.integration;
import java.io.IOException;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.openstack.swift.blobstore.integration.SwiftBlobIntegrationLiveTest;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
/**
*
* @author Adrian Cole
@ -38,4 +43,22 @@ public class CloudFilesBlobIntegrationLiveTest extends SwiftBlobIntegrationLiveT
.getMetadata().getContentMetadata().getContentDisposition();
}
@Test(groups = { "integration", "live" })
public void testChunksAreDeletedWhenMultipartBlobIsDeleted() throws IOException, InterruptedException {
String containerName = getContainerName();
try {
BlobStore blobStore = view.getBlobStore();
long countBefore = blobStore.countBlobs(containerName);
String blobName = "deleteme.txt";
addMultipartBlobToContainer(containerName, blobName);
blobStore.removeBlob(containerName, blobName);
long countAfter = blobStore.countBlobs(containerName);
assertEquals(countAfter, countBefore);
} finally {
returnContainer(containerName);
}
}
}

56
apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
View File

@ -16,8 +16,10 @@
*/
package org.jclouds.openstack.swift.blobstore;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.blobstore.util.BlobStoreUtils.createParentIfNeededAsync;
import static org.jclouds.openstack.swift.options.ListContainerOptions.Builder.withPrefix;
import java.util.Set;
@ -25,6 +27,7 @@ import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
import com.google.common.annotations.VisibleForTesting;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
@ -50,8 +53,11 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy;
import org.jclouds.openstack.swift.domain.ContainerMetadata;
import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata;
import org.jclouds.openstack.swift.domain.ObjectInfo;
import com.google.common.base.Function;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.Iterables;
@ -118,7 +124,7 @@ public class SwiftBlobStore extends BaseBlobStore {
}
/**
* This implementation invokes {@link CommonSwiftClient#putBucketInRegion}
* This implementation invokes {@link CommonSwiftClient#createContainer}
*
* @param location
* currently ignored
@ -145,7 +151,7 @@ public class SwiftBlobStore extends BaseBlobStore {
}
/**
* This implementation invokes {@link CommonSwiftClient#blobExists}
* This implementation invokes {@link CommonSwiftClient#objectExists}
*
* @param container
* container name
@ -225,7 +231,53 @@ public class SwiftBlobStore extends BaseBlobStore {
*/
@Override
public void removeBlob(String container, String key) {
String objectManifest = getObjectManifestOrNull(container, key);
sync.removeObject(container, key);
if (!Strings.isNullOrEmpty(objectManifest)) {
removeObjectsWithPrefix(objectManifest);
}
}
private String getObjectManifestOrNull(String container, String key) {
MutableObjectInfoWithMetadata objectInfo = sync.getObjectInfo(container, key);
return objectInfo == null ? null : objectInfo.getObjectManifest();
}
private void removeObjectsWithPrefix(String containerAndPrefix) {
String[] parts = splitContainerAndKey(containerAndPrefix);
String container = parts[0];
String prefix = parts[1];
removeObjectsWithPrefix(container, prefix);
}
@VisibleForTesting
static String[] splitContainerAndKey(String containerAndKey) {
String[] parts = containerAndKey.split("/", 2);
checkArgument(parts.length == 2,
"No / separator found in \"%s\"",
containerAndKey);
return parts;
}
private void removeObjectsWithPrefix(String container, String prefix) {
String nextMarker = null;
do {
org.jclouds.openstack.swift.options.ListContainerOptions listContainerOptions =
withPrefix(prefix);
if (nextMarker != null) {
listContainerOptions = listContainerOptions.afterMarker(nextMarker);
}
PageSet<ObjectInfo> chunks = sync.listObjects(container, listContainerOptions);
for (ObjectInfo chunk : chunks) {
sync.removeObject(container, chunk.getName());
}
nextMarker = chunks.getNextMarker();
} while (nextMarker != null);
}
@Override

3
apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java
View File

@ -48,4 +48,7 @@ public interface MutableObjectInfoWithMetadata extends ObjectInfo {
Map<String, String> getMetadata();
String getObjectManifest();
void setObjectManifest(String objectManifest);
}

10
apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java
View File

@ -147,4 +147,14 @@ public class DelegatingMutableObjectInfoWithMetadata extends BaseMutableContentM
public URI getUri() {
return delegate.getUri();
}
@Override
public void setObjectManifest(String objectManifest) {
delegate.setObjectManifest(objectManifest);
}
@Override
public String getObjectManifest() {
return delegate.getObjectManifest();
}
}

21
apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java
View File

@ -41,6 +41,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
private byte[] hash;
private String contentType = MediaType.APPLICATION_OCTET_STREAM;
private Date lastModified;
private String objectManifest;
private final Map<String, String> metadata = Maps.newLinkedHashMap();
/**
@ -121,6 +122,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
int result = 1;
result = prime * result + ((container == null) ? 0 : container.hashCode());
result = prime * result + ((name == null) ? 0 : name.hashCode());
result = prime * result + ((objectManifest == null) ? 0 : objectManifest.hashCode());
return result;
}
@ -143,6 +145,11 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
return false;
} else if (!name.equals(other.name))
return false;
if (objectManifest == null) {
if (other.objectManifest != null)
return false;
} else if (!objectManifest.equals(other.objectManifest))
return false;
return true;
}
@ -196,10 +203,20 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM
return uri;
}
@Override
public String getObjectManifest() {
return objectManifest;
}
@Override
public void setObjectManifest(String objectManifest) {
this.objectManifest = objectManifest;
}
@Override
public String toString() {
return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s]", name,
container, uri, bytes, contentType, lastModified, Arrays.toString(hash));
return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s, objectManifest=%s]",
name, container, uri, bytes, contentType, lastModified, Arrays.toString(hash), objectManifest);
}
}

2
apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java
View File

@ -65,6 +65,8 @@ public class ParseObjectInfoFromHeaders implements Function<HttpResponse, Mutabl
if (eTagHeader != null) {
to.setHash(ETagUtils.convertHexETagToByteArray(eTagHeader));
}
to.setObjectManifest(from.getFirstHeaderOrNull("X-Object-Manifest"));
return to;
}

45
apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.openstack.swift.blobstore;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@Test(groups = "unit")
public class SwiftBlobStoreTest {
@Test
public void testSplitContainerAndKey() {
String container = "test-container";
String key = "key/with/some/slashes/in/it/and/a/trailing/slash/";
String containerAndKey = container + "/" + key;
String[] split = SwiftBlobStore.splitContainerAndKey(containerAndKey);
String actualContainer = split[0];
String actualKey = split[1];
assertEquals(actualContainer, container);
assertEquals(actualKey, key);
}
@Test(expectedExceptions = IllegalArgumentException.class,
expectedExceptionsMessageRegExp = "No / separator found in \"not-a-container-and-key\"")
public void testSplitContainerAndKeyWithNoSeparator() {
SwiftBlobStore.splitContainerAndKey("not-a-container-and-key");
}
}

69
apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
View File

@ -21,11 +21,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import com.google.common.io.ByteStreams;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
import org.testng.ITestContext;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
@ -34,6 +36,9 @@ import org.testng.annotations.Test;
import com.google.common.io.Files;
import com.google.common.io.InputSupplier;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertTrue;
/**
*
* @author James Murty
@ -41,15 +46,21 @@ import com.google.common.io.InputSupplier;
*/
@Test(groups = "live")
public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
/**
* Use the minimum part size to minimise the file size that we have to
* upload to get a multipart blob thereby make the test run faster
*/
private static final long PART_SIZE = MultipartUpload.MIN_PART_SIZE;
@Override
protected Properties setupProperties() {
Properties props = super.setupProperties();
setIfTestSystemPropertyPresent(props, KeystoneProperties.CREDENTIAL_TYPE);
props.setProperty("jclouds.mpu.parts.size", String.valueOf(PART_SIZE));
return props;
}
private InputSupplier<InputStream> oneHundredOneConstitutions;
private byte[] oneHundredOneConstitutionsMD5;
public SwiftBlobIntegrationLiveTest() {
provider = System.getProperty("test.swift.provider", "swift");
@ -66,7 +77,6 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
oneHundredOneConstitutions = getTestDataSupplier();
oneHundredOneConstitutionsMD5 = md5Supplier(oneHundredOneConstitutions);
}
@Override
@ -91,18 +101,51 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
{ "asteri*k" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
}
@Test(groups = { "integration", "live" })
public void testMultipartChunkedFileStream() throws IOException, InterruptedException {
Files.copy(oneHundredOneConstitutions, new File("target/const.txt"));
String containerName = getContainerName();
String containerName = getContainerName();
try {
BlobStore blobStore = view.getBlobStore();
long countBefore = blobStore.countBlobs(containerName);
try {
BlobStore blobStore = view.getBlobStore();
blobStore.createContainerInLocation(null, containerName);
Blob blob = blobStore.blobBuilder("const.txt")
.payload(new File("target/const.txt")).contentMD5(oneHundredOneConstitutionsMD5).build();
blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
} finally {
returnContainer(containerName);
}
addMultipartBlobToContainer(containerName, "const.txt");
long countAfter = blobStore.countBlobs(containerName);
assertNotEquals(countBefore, countAfter,
"No blob was created");
assertTrue(countAfter - countBefore > 1,
"A multipart blob wasn't actually created - " +
"there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs");
} finally {
returnContainer(containerName);
}
}
protected void addMultipartBlobToContainer(String containerName, String key) throws IOException {
File fileToUpload = createFileBiggerThan(PART_SIZE);
BlobStore blobStore = view.getBlobStore();
blobStore.createContainerInLocation(null, containerName);
Blob blob = blobStore.blobBuilder(key)
.payload(fileToUpload)
.build();
blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
}
@SuppressWarnings("unchecked")
private File createFileBiggerThan(long partSize) throws IOException {
long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1;
InputSupplier<InputStream> temp = ByteStreams.join(oneHundredOneConstitutions);
for (int i = 0; i < copiesNeeded; i++) {
temp = ByteStreams.join(temp, oneHundredOneConstitutions);
}
File fileToUpload = new File("target/lots-of-const.txt");
Files.copy(temp, fileToUpload);
assertTrue(fileToUpload.length() > partSize);
return fileToUpload;
}
}

7
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
View File

@ -119,6 +119,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
return temp;
}
public static long getOneHundredOneConstitutionsLength() throws IOException {
if (oneHundredOneConstitutionsLength == 0) {
getTestDataSupplier();
}
return oneHundredOneConstitutionsLength;
}
/**
* Attempt to capture the issue detailed in
* http://groups.google.com/group/jclouds/browse_thread/thread/4a7c8d58530b287f