From 197801204e77b07d3c6d2f925cd30cd20b6f9c61 Mon Sep 17 00:00:00 2001 From: "adrian.f.cole" Date: Tue, 3 Nov 2009 01:53:42 +0000 Subject: [PATCH] Issue 111: fixed blobstore related bugs and accomodated eventual consistency git-svn-id: http://jclouds.googlecode.com/svn/trunk@2040 3d8758e0-26b5-11de-8745-db77d3ebf521 --- .../saas/blobstore/AtmosBlobStore.java | 40 +++++++++------- .../functions/ObjectToBlobMetadata.java | 22 ++++++++- .../strategy/FindMD5InUserMetadata.java | 7 +-- .../saas/AtmosStorageClientLiveTest.java | 2 +- .../AtmosStorageIntegrationTest.java | 47 +++++++++++++++++++ 5 files changed, 96 insertions(+), 22 deletions(-) diff --git a/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java b/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java index 8d1d0f496f..b10cb80385 100644 --- a/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java +++ b/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java @@ -77,11 +77,11 @@ public class AtmosBlobStore implements BlobStore { private final BlobToHttpGetOptions blob2ObjectGetOptions; private final DirectoryEntryListToResourceMetadataList container2ResourceList; private final ExecutorService service; - + @Inject(optional = true) @Named(BlobStoreConstants.PROPERTY_BLOBSTORE_TIMEOUT) protected long requestTimeoutMilliseconds = 30000; - + @Inject private AtmosBlobStore(AtmosStorageClient connection, Blob.Factory blobFactory, LoggerFactory logFactory, ClearListStrategy clearContainerStrategy, @@ -125,7 +125,7 @@ public class AtmosBlobStore implements BlobStore { }); } - + public Future createContainer(String container) { return wrapFuture(connection.createDirectory(container), new Function() { @@ -173,7 +173,7 @@ public class AtmosBlobStore implements BlobStore { public Future> list( String container, org.jclouds.blobstore.options.ListContainerOptions... optionsList) { if (optionsList.length == 1) { - if (!optionsList[0].isRecursive()) { + if (optionsList[0].isRecursive()) { throw new UnsupportedOperationException("recursive not currently supported in emcsaas"); } if (optionsList[0].getPath() != null) { @@ -184,6 +184,9 @@ public class AtmosBlobStore implements BlobStore { return wrapFuture(connection.listDirectory(container, nativeOptions), container2ResourceList); } + /** + * Since there is no etag support in atmos, we just return the path. + */ public Future putBlob(final String container, final Blob blob) { final String path = container + "/" + blob.getMetadata().getName(); @@ -191,19 +194,24 @@ public class AtmosBlobStore implements BlobStore { .deletePath(path), new Function() { public String apply(Void from) { - boolean exists = connection.pathExists(path); - if (!exists) - try { - if (blob.getMetadata().getContentMD5() != null) - blob.getMetadata().getUserMetadata().put("content-md5", - HttpUtils.toHexString(blob.getMetadata().getContentMD5())); - connection.createFile(container, blob2Object.apply(blob)).get(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (ExecutionException e) { - throw new RuntimeException(e); + try { + if (!Utils.enventuallyTrue(new Supplier() { + public Boolean get() { + return !connection.pathExists(path); + } + }, requestTimeoutMilliseconds)) { + throw new IllegalStateException(path + " still exists after deleting!"); } - return null; + if (blob.getMetadata().getContentMD5() != null) + blob.getMetadata().getUserMetadata().put("content-md5", + HttpUtils.toHexString(blob.getMetadata().getContentMD5())); + connection.createFile(container, blob2Object.apply(blob)).get(); + return path; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } } }); diff --git a/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java b/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java index 2fd97f589a..d149b13b16 100644 --- a/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java +++ b/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java @@ -1,5 +1,9 @@ package org.jclouds.atmosonline.saas.blobstore.functions; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + import javax.inject.Inject; import javax.inject.Singleton; @@ -8,8 +12,11 @@ import org.jclouds.atmosonline.saas.functions.AtmosObjectName; import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.domain.ResourceType; import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; +import org.jclouds.http.HttpUtils; import com.google.common.base.Function; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; /** * @author Adrian Cole @@ -17,6 +24,9 @@ import com.google.common.base.Function; @Singleton public class ObjectToBlobMetadata implements Function { private final AtmosObjectName objectName; + private static final Set systemMetadata = ImmutableSet.of("atime", "mtime", "ctime", + "itime", "type", "uid", "gid", "objectid", "objname", "size", "nlink", "policyname", + "content-md5"); @Inject protected ObjectToBlobMetadata(AtmosObjectName objectName) { @@ -27,13 +37,21 @@ public class ObjectToBlobMetadata implements Function lowerKeyMetadata = Maps.newHashMap(); + for (Entry entry : from.getUserMetadata().getMetadata().entrySet()) { + String key = entry.getKey().toLowerCase(); + if (!systemMetadata.contains(key)) + lowerKeyMetadata.put(key, entry.getValue()); + } + to.setUserMetadata(lowerKeyMetadata); return to; } } \ No newline at end of file diff --git a/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java b/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java index 3872088592..8d1b4a74c1 100644 --- a/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java +++ b/atmosonline/saas/core/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java @@ -27,8 +27,8 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy { private final AtmosStorageClient client; @Inject - private FindMD5InUserMetadata(ObjectMD5 objectMD5, - ListBlobMetadataStrategy getAllBlobMetadata, AtmosStorageClient client) { + private FindMD5InUserMetadata(ObjectMD5 objectMD5, ListBlobMetadataStrategy getAllBlobMetadata, + AtmosStorageClient client) { this.objectMD5 = objectMD5; this.getAllBlobMetadata = getAllBlobMetadata; this.client = client; @@ -39,7 +39,8 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy { byte[] toSearch = objectMD5.apply(value); String hex = HttpUtils.toHexString(toSearch); for (BlobMetadata metadata : getAllBlobMetadata.execute(containerName, options)) { - UserMetadata properties = client.getUserMetadata(containerName+"/"+metadata.getName()); + UserMetadata properties = client.headFile(containerName + "/" + metadata.getName()) + .getUserMetadata(); if (hex.equals(properties.getMetadata().get("content-md5"))) return true; } diff --git a/atmosonline/saas/core/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java b/atmosonline/saas/core/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java index 268ab67c46..8686d1f7b6 100644 --- a/atmosonline/saas/core/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java +++ b/atmosonline/saas/core/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java @@ -195,7 +195,7 @@ public class AtmosStorageClientLiveTest { // loop to gather metrics for (boolean stream : new Boolean[] { true, false }) { - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 10; i++) { System.err.printf("upload/delete/create attempt %d type %s%n", i + 1, stream ? "stream" : "string"); // try updating diff --git a/atmosonline/saas/core/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageIntegrationTest.java b/atmosonline/saas/core/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageIntegrationTest.java index 88532cf2d2..dc6a35bd78 100755 --- a/atmosonline/saas/core/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageIntegrationTest.java +++ b/atmosonline/saas/core/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageIntegrationTest.java @@ -23,8 +23,13 @@ */ package org.jclouds.atmosonline.saas.blobstore.integration; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; + import org.jclouds.atmosonline.saas.AtmosStorageClient; import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; /** @@ -34,4 +39,46 @@ import org.testng.annotations.Test; @Test(groups = { "integration", "live" }, testName = "emcsaas.AtmosStorageIntegrationTest") public class AtmosStorageIntegrationTest extends BaseBlobIntegrationTest { + @DataProvider(name = "delete") + // no unicode support + @Override + public Object[][] createData() { + return new Object[][] { { "normal" } }; + } + + @Override + @Test(enabled = false) + public void testGetIfMatch() throws InterruptedException, ExecutionException, TimeoutException, + IOException { + // no etag support + } + + @Override + @Test(enabled = false) + public void testGetIfModifiedSince() throws InterruptedException, ExecutionException, + TimeoutException, IOException { + // not supported + } + + @Override + @Test(enabled = false) + public void testGetIfNoneMatch() throws InterruptedException, ExecutionException, + TimeoutException, IOException { + // no etag support + } + + @Override + @Test(enabled = false) + public void testGetIfUnmodifiedSince() throws InterruptedException, ExecutionException, + TimeoutException, IOException { + // not supported + } + + @Override + @Test(enabled = false) + public void testGetTwoRanges() throws InterruptedException, ExecutionException, + TimeoutException, IOException { + // not supported + } + } \ No newline at end of file