From 0684dc2344f71538adcc46cedee5f6bbffdc4fae Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Mon, 24 May 2010 12:29:39 -0700 Subject: [PATCH] Issue 264: fixed blob stuff in rackspace --- .../util/internal/BlobStoreUtilsImpl.java | 13 +++ .../internal/BaseBlobIntegrationTest.java | 20 +++-- .../BaseContainerIntegrationTest.java | 7 +- .../blobstore/util/BlobStoreUtilsTest.java | 79 ++++++++++++++++++- .../blobstore/CloudFilesAsyncBlobStore.java | 7 +- .../blobstore/CloudFilesBlobStore.java | 6 +- ...ontainerOptionsToListContainerOptions.java | 2 +- 7 files changed, 118 insertions(+), 16 deletions(-) diff --git a/blobstore/src/main/java/org/jclouds/blobstore/util/internal/BlobStoreUtilsImpl.java b/blobstore/src/main/java/org/jclouds/blobstore/util/internal/BlobStoreUtilsImpl.java index e3eeed79d2..6c4edfd10f 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/util/internal/BlobStoreUtilsImpl.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/util/internal/BlobStoreUtilsImpl.java @@ -28,6 +28,7 @@ import java.util.regex.Pattern; import javax.inject.Inject; import javax.inject.Singleton; +import org.jclouds.blobstore.AsyncBlobStore; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; @@ -135,6 +136,10 @@ public class BlobStoreUtilsImpl implements BlobStoreUtils { return "".equals(prefix) ? null : prefix; } + public static String parseDirectoryFromPath(String path) { + return path.substring(0, path.lastIndexOf('/')); + } + private static Pattern keyFromContainer = Pattern.compile("/?[^/]+/(.*)"); public static String getKeyFor(GeneratedHttpRequest request, HttpResponse from) { @@ -168,4 +173,12 @@ public class BlobStoreUtilsImpl implements BlobStoreUtils { throw new IllegalArgumentException("Object type not supported: " + o.getClass().getName()); } } + + public static void createParentIfNeededAsync(AsyncBlobStore asyncBlobStore, String container, + Blob blob) { + String name = blob.getMetadata().getName(); + if (name.indexOf('/') > 0) { + asyncBlobStore.createDirectory(container, parseDirectoryFromPath(name)); + } + } } \ No newline at end of file diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index 0ffed1d6b6..75a99fbcf1 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -36,7 +36,6 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.util.Date; import java.util.Map; -import java.util.Set; import java.util.zip.GZIPInputStream; import javax.ws.rs.core.MediaType; @@ -46,6 +45,7 @@ import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl; import org.jclouds.encryption.EncryptionService; import org.jclouds.encryption.EncryptionService.MD5InputStreamResult; @@ -61,6 +61,8 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.io.ByteStreams; import com.google.common.io.InputSupplier; @@ -371,10 +373,18 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { } private void assertContainerEmptyDeleting(String containerName, String key) { - Set listing = context.getBlobStore().list(containerName); - assertEquals(listing.size(), 0, String.format( - "deleting %s, we still have %s left in container %s, using encoding %s", key, - listing.size(), containerName, LOCAL_ENCODING)); + Iterable listing = Iterables.filter(context.getBlobStore().list( + containerName), new Predicate() { + + @Override + public boolean apply(StorageMetadata input) { + return input.getType() == StorageType.BLOB; + } + + }); + assertEquals(Iterables.size(listing), 0, String.format( + "deleting %s, we still have %s blobs left in container %s, using encoding %s", key, + Iterables.size(listing), containerName, LOCAL_ENCODING)); } @Test(groups = { "integration", "live" }) diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java index d382d2a513..f6aecaedbc 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java @@ -69,7 +69,6 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest { } } - @Test(groups = { "integration", "live" }) public void testWithDetails() throws InterruptedException { String key = "hello"; @@ -202,6 +201,12 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest { assert container.getNextMarker() == null; assert container.size() == 1 : container; + context.getBlobStore().createDirectory(containerName, directory + "/" + directory); + + container = context.getBlobStore().list(containerName, inDirectory(directory).recursive()); + assert container.getNextMarker() == null; + assert container.size() == 1 : container; + context.getBlobStore().clearContainer(containerName, inDirectory(directory).recursive()); // should no longer have the 2 level-deep directory above diff --git a/blobstore/src/test/java/org/jclouds/blobstore/util/BlobStoreUtilsTest.java b/blobstore/src/test/java/org/jclouds/blobstore/util/BlobStoreUtilsTest.java index f027030112..3f21a5873a 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/util/BlobStoreUtilsTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/util/BlobStoreUtilsTest.java @@ -21,10 +21,14 @@ package org.jclouds.blobstore.util; import static org.easymock.EasyMock.expect; import static org.easymock.classextension.EasyMock.createMock; import static org.easymock.classextension.EasyMock.replay; +import static org.easymock.classextension.EasyMock.verify; import static org.testng.Assert.assertEquals; import java.net.URI; +import org.jclouds.blobstore.AsyncBlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl; import org.jclouds.http.HttpResponse; import org.jclouds.rest.internal.GeneratedHttpRequest; @@ -38,6 +42,68 @@ import org.testng.annotations.Test; @Test(groups = "unit", testName = "blobstore.BlobStoreUtilsTest") public class BlobStoreUtilsTest { + public void testCreateParentIfNeededAsyncNoPath() { + AsyncBlobStore asyncBlobStore = createMock(AsyncBlobStore.class); + String container = "container"; + Blob blob = createMock(Blob.class); + MutableBlobMetadata md = createMock(MutableBlobMetadata.class); + + expect(blob.getMetadata()).andReturn(md).atLeastOnce(); + expect(md.getName()).andReturn("hello").atLeastOnce(); + + replay(asyncBlobStore); + replay(blob); + replay(md); + + BlobStoreUtilsImpl.createParentIfNeededAsync(asyncBlobStore, container, blob); + + verify(asyncBlobStore); + verify(blob); + verify(md); + } + + public void testCreateParentIfNeededAsyncSinglePath() { + AsyncBlobStore asyncBlobStore = createMock(AsyncBlobStore.class); + String container = "container"; + Blob blob = createMock(Blob.class); + MutableBlobMetadata md = createMock(MutableBlobMetadata.class); + + expect(blob.getMetadata()).andReturn(md).atLeastOnce(); + expect(md.getName()).andReturn("rootpath/hello").atLeastOnce(); + expect(asyncBlobStore.createDirectory("container", "rootpath")).andReturn(null); + + replay(asyncBlobStore); + replay(blob); + replay(md); + + BlobStoreUtilsImpl.createParentIfNeededAsync(asyncBlobStore, container, blob); + + verify(asyncBlobStore); + verify(blob); + verify(md); + } + + public void testCreateParentIfNeededAsyncNestedPath() { + AsyncBlobStore asyncBlobStore = createMock(AsyncBlobStore.class); + String container = "container"; + Blob blob = createMock(Blob.class); + MutableBlobMetadata md = createMock(MutableBlobMetadata.class); + + expect(blob.getMetadata()).andReturn(md).atLeastOnce(); + expect(md.getName()).andReturn("rootpath/subpath/hello").atLeastOnce(); + expect(asyncBlobStore.createDirectory("container", "rootpath/subpath")).andReturn(null); + + replay(asyncBlobStore); + replay(blob); + replay(md); + + BlobStoreUtilsImpl.createParentIfNeededAsync(asyncBlobStore, container, blob); + + verify(asyncBlobStore); + verify(blob); + verify(md); + } + public void testGetKeyForAzureS3AndRackspace() { GeneratedHttpRequest request = createMock(GeneratedHttpRequest.class); @@ -45,7 +111,8 @@ public class BlobStoreUtilsTest { HttpResponse from = createMock(HttpResponse.class); expect(request.getEndpoint()).andReturn( URI.create("https://jclouds.blob.core.windows.net/adriancole-blobstore0/five")); - expect(request.getArgs()).andReturn(new Object[] { "adriancole-blobstore0", "five" }).atLeastOnce(); + expect(request.getArgs()).andReturn(new Object[] { "adriancole-blobstore0", "five" }) + .atLeastOnce(); replay(request); replay(from); @@ -58,15 +125,19 @@ public class BlobStoreUtilsTest { GeneratedHttpRequest request = createMock(GeneratedHttpRequest.class); HttpResponse from = createMock(HttpResponse.class); - expect(request.getEndpoint()).andReturn( - URI.create("https://storage4.clouddrive.com/v1/MossoCloudFS_dc1f419c-5059-4c87-a389-3f2e33a77b22/adriancole-blobstore0/four")); - expect(request.getArgs()).andReturn(new Object[] { "adriancole-blobstore0/four" }).atLeastOnce(); + expect(request.getEndpoint()) + .andReturn( + URI + .create("https://storage4.clouddrive.com/v1/MossoCloudFS_dc1f419c-5059-4c87-a389-3f2e33a77b22/adriancole-blobstore0/four")); + expect(request.getArgs()).andReturn(new Object[] { "adriancole-blobstore0/four" }) + .atLeastOnce(); replay(request); replay(from); assertEquals(BlobStoreUtilsImpl.getKeyFor(request, from), "four"); } + public void testGetContainer() { String container = BlobStoreUtilsImpl.parseContainerFromPath("foo"); assertEquals(container, "foo"); diff --git a/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java b/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java index 3868a9a56a..f5a4c888d7 100644 --- a/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java +++ b/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java @@ -20,6 +20,7 @@ package org.jclouds.rackspace.cloudfiles.blobstore; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.util.concurrent.Futures.compose; +import static org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl.createParentIfNeededAsync; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -80,9 +81,8 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore { @Inject CloudFilesAsyncBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, - Location defaultLocation, Set locations, - CloudFilesClient sync, CloudFilesAsyncClient async, - ContainerToResourceMetadata container2ResourceMd, + Location defaultLocation, Set locations, CloudFilesClient sync, + CloudFilesAsyncClient async, ContainerToResourceMetadata container2ResourceMd, BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, @@ -216,6 +216,7 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore { */ @Override public ListenableFuture putBlob(String container, Blob blob) { + createParentIfNeededAsync(this, container, blob); return async.putObject(container, blob2Object.apply(blob)); } diff --git a/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesBlobStore.java b/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesBlobStore.java index 9ea106ca36..693f0a9f50 100644 --- a/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesBlobStore.java +++ b/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesBlobStore.java @@ -19,6 +19,7 @@ package org.jclouds.rackspace.cloudfiles.blobstore; import static com.google.common.base.Preconditions.checkNotNull; +import static org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl.createParentIfNeededAsync; import java.util.Set; @@ -69,8 +70,8 @@ public class CloudFilesBlobStore extends BaseBlobStore { @Inject CloudFilesBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils, - Location defaultLocation, Set locations, - CloudFilesClient sync, ContainerToResourceMetadata container2ResourceMd, + Location defaultLocation, Set locations, CloudFilesClient sync, + ContainerToResourceMetadata container2ResourceMd, BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, @@ -194,6 +195,7 @@ public class CloudFilesBlobStore extends BaseBlobStore { */ @Override public String putBlob(String container, Blob blob) { + createParentIfNeededAsync(context.getAsyncBlobStore(), container, blob); return sync.putObject(container, blob2Object.apply(blob)); } diff --git a/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/BlobStoreListContainerOptionsToListContainerOptions.java b/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/BlobStoreListContainerOptionsToListContainerOptions.java index cb01b433c1..3f4e8e8bbb 100644 --- a/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/BlobStoreListContainerOptionsToListContainerOptions.java +++ b/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/BlobStoreListContainerOptionsToListContainerOptions.java @@ -44,7 +44,7 @@ public class BlobStoreListContainerOptionsToListContainerOptions options.underPath(""); } if ((from.getDir() != null) && (from.isRecursive())) { - options.withPrefix(from.getDir()); + options.withPrefix(from.getDir().endsWith("/") ? from.getDir() : from.getDir() + "/"); } if ((from.getDir() != null) && (!from.isRecursive())) { options.underPath(from.getDir());