diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java index 112d15bc33..7035c38cda 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java @@ -45,8 +45,6 @@ import org.jclouds.atmosonline.saas.functions.ReturnEndpointIfAlreadyExists; import org.jclouds.atmosonline.saas.options.ListOptions; import org.jclouds.blobstore.attr.ConsistencyModel; import org.jclouds.blobstore.attr.ConsistencyModels; -import org.jclouds.blobstore.functions.ReturnFalseOnKeyNotFound; -import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound; import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404; import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404; import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404; @@ -59,6 +57,8 @@ import org.jclouds.rest.annotations.QueryParams; import org.jclouds.rest.annotations.RequestFilters; import org.jclouds.rest.annotations.ResponseParser; import org.jclouds.rest.annotations.SkipEncoding; +import org.jclouds.rest.functions.ReturnFalseOnResourceNotFound; +import org.jclouds.rest.functions.ReturnNullOnResourceNotFound; import com.google.common.util.concurrent.ListenableFuture; @@ -75,7 +75,9 @@ import com.google.common.util.concurrent.ListenableFuture; @SkipEncoding( { '/' }) @ConsistencyModel(ConsistencyModels.EVENTUAL) public interface AtmosStorageAsyncClient { - + /** + * Creates a default implementation of AtmosObject + */ AtmosObject newObject(); /** @@ -134,7 +136,7 @@ public interface AtmosStorageAsyncClient { */ @GET @ResponseParser(ParseObjectFromHeadersAndHttpContent.class) - @ExceptionParser(ReturnNullOnKeyNotFound.class) + @ExceptionParser(ReturnNullOnResourceNotFound.class) @Path("/rest/namespace/{path}") @Consumes(MediaType.WILDCARD) ListenableFuture readFile(@PathParam("path") String path, GetOptions... options); @@ -144,7 +146,7 @@ public interface AtmosStorageAsyncClient { */ @HEAD @ResponseParser(ParseObjectFromHeadersAndHttpContent.class) - @ExceptionParser(ReturnNullOnKeyNotFound.class) + @ExceptionParser(ReturnNullOnResourceNotFound.class) @Path("/rest/namespace/{path}") @Consumes(MediaType.WILDCARD) ListenableFuture headFile(@PathParam("path") String path); @@ -154,7 +156,7 @@ public interface AtmosStorageAsyncClient { */ @HEAD @ResponseParser(ParseSystemMetadataFromHeaders.class) - @ExceptionParser(ReturnNullOnKeyNotFound.class) + @ExceptionParser(ReturnNullOnResourceNotFound.class) // currently throws 403 errors @QueryParams(keys = "metadata/system") @Path("/rest/namespace/{path}") @Consumes(MediaType.WILDCARD) @@ -165,7 +167,7 @@ public interface AtmosStorageAsyncClient { */ @HEAD @ResponseParser(ParseSystemMetadataFromHeaders.class) - @ExceptionParser(ReturnNullOnKeyNotFound.class) + @ExceptionParser(ReturnNullOnResourceNotFound.class) @Path("/rest/namespace/{path}") @QueryParams(keys = "metadata/user") @Consumes(MediaType.WILDCARD) @@ -184,7 +186,7 @@ public interface AtmosStorageAsyncClient { * @see AtmosStorageClient#pathExists */ @HEAD - @ExceptionParser(ReturnFalseOnKeyNotFound.class) + @ExceptionParser(ReturnFalseOnResourceNotFound.class) @Path("/rest/namespace/{path}") @Consumes(MediaType.WILDCARD) ListenableFuture pathExists(@PathParam("path") String path); diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageClient.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageClient.java index 37d9d806d8..5c08d803f4 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageClient.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageClient.java @@ -40,13 +40,14 @@ import org.jclouds.http.options.GetOptions; */ @Timeout(duration = 120, timeUnit = TimeUnit.SECONDS) public interface AtmosStorageClient { - + /** + * Creates a default implementation of AtmosObject + */ AtmosObject newObject(); BoundedSet listDirectories(ListOptions... options); - BoundedSet listDirectory(String directoryName, - ListOptions... options); + BoundedSet listDirectory(String directoryName, ListOptions... options); URI createDirectory(String directoryName); @@ -65,23 +66,8 @@ public interface AtmosStorageClient { UserMetadata getUserMetadata(String path); - /** - * - * @param path - * @return - * @throws AtmosStorageResponseException - * , if the path is a directory and not empty - */ void deletePath(String path); boolean pathExists(String path); - // signature currently doesn't work - // @POST - // @QueryParams(keys = "acl") - // @Headers(keys = { "x-emc-useracl", "x-emc-groupacl" }, values = { "root=FULL_CONTROL", - // "other=READ" }) - // @Consumes(MediaType.WILDCARD) - // void makePublic(@Endpoint URI url); - } diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java index 25f0ed9e2d..b6200d1cab 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java @@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.util.concurrent.Futures.compose; import java.net.URI; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import javax.inject.Inject; @@ -46,13 +47,11 @@ import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.internal.BaseAsyncBlobStore; import org.jclouds.blobstore.util.BlobStoreUtils; +import org.jclouds.concurrent.ConcurrentUtils; import org.jclouds.encryption.EncryptionService; import org.jclouds.http.options.GetOptions; -import org.jclouds.util.Utils; import com.google.common.base.Function; -import com.google.common.base.Supplier; -import com.google.common.base.Throwables; import com.google.common.util.concurrent.ListenableFuture; /** @@ -140,8 +139,8 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { * {@link AtmosStorageAsyncClient#pathExists} until it is true. */ protected boolean deleteAndVerifyContainerGone(final String container) { - sync.deletePath(container); - return !sync.pathExists(container); + sync.deletePath(container + "/"); + return !sync.pathExists(container + "/"); } /** @@ -149,7 +148,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { */ @Override public ListenableFuture containerExists(String container) { - return async.pathExists(container); + return async.pathExists(container + "/"); } /** @@ -160,6 +159,14 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { return async.pathExists(container + "/" + directory + "/"); } + /** + * This implementation invokes {@link #removeBlob} + */ + @Override + public ListenableFuture deleteDirectory(String containerName, String directory) { + return removeBlob(containerName, directory + "/"); + } + /** * This implementation invokes {@link AtmosStorageAsyncClient#pathExists} * @@ -210,31 +217,15 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { */ @Override public ListenableFuture putBlob(final String container, final Blob blob) { - final String path = container + "/" + blob.getMetadata().getName(); - return compose(async.deletePath(path), new Function() { + return ConcurrentUtils.makeListenable(service.submit(new Callable() { + + @Override + public String call() throws Exception { + return AtmosStorageUtils.putBlob(sync, encryptionService, blob2Object, container, blob); - public String apply(Void from) { - try { - if (!Utils.enventuallyTrue(new Supplier() { - public Boolean get() { - return !sync.pathExists(path); - } - }, 300)) { - throw new IllegalStateException(path + " still exists after deleting!"); - } - if (blob.getMetadata().getContentMD5() != null) - blob.getMetadata().getUserMetadata().put("content-md5", - encryptionService.toHexString(blob.getMetadata().getContentMD5())); - sync.createFile(container, blob2Object.apply(blob)); - return path; - } catch (InterruptedException e) { - Throwables.propagate(e); - } - assert false : " should have propagated error"; - return null; } - }, service); + }), service); } diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java index deebc8af86..8e13c712f3 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java @@ -87,8 +87,8 @@ public class AtmosBlobStore extends BaseBlobStore { * {@link AtmosStorageAsyncClient#pathExists} until it is true. */ protected boolean deleteAndVerifyContainerGone(final String container) { - sync.deletePath(container); - return !sync.pathExists(container); + sync.deletePath(container + "/"); + return !sync.pathExists(container + "/"); } /** @@ -116,12 +116,20 @@ public class AtmosBlobStore extends BaseBlobStore { sync.createDirectory(container + "/" + directory); } + /** + * This implementation invokes {@link #removeBlob} + */ + @Override + public void deleteDirectory(String containerName, String directory) { + removeBlob(containerName, directory + "/"); + } + /** * This implementation invokes {@link AtmosStorageClient#pathExists} */ @Override public boolean containerExists(String container) { - return sync.pathExists(container); + return sync.pathExists(container + "/"); } /** @@ -181,13 +189,7 @@ public class AtmosBlobStore extends BaseBlobStore { */ @Override public String putBlob(final String container, final Blob blob) { - final String path = container + "/" + blob.getMetadata().getName(); - deleteAndEnsurePathGone(path); - if (blob.getMetadata().getContentMD5() != null) - blob.getMetadata().getUserMetadata().put("content-md5", - encryptionService.toHexString(blob.getMetadata().getContentMD5())); - sync.createFile(container, blob2Object.apply(blob)); - return path; + return AtmosStorageUtils.putBlob(sync, encryptionService, blob2Object, container, blob); } /** diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/config/AtmosBlobStoreContextModule.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/config/AtmosBlobStoreContextModule.java index d069eb55fd..2e7959f955 100755 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/config/AtmosBlobStoreContextModule.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/config/AtmosBlobStoreContextModule.java @@ -25,7 +25,6 @@ import org.jclouds.atmosonline.saas.AtmosStorageClient; import org.jclouds.atmosonline.saas.blobstore.AtmosAsyncBlobStore; import org.jclouds.atmosonline.saas.blobstore.AtmosBlobStore; import org.jclouds.atmosonline.saas.blobstore.strategy.FindMD5InUserMetadata; -import org.jclouds.atmosonline.saas.blobstore.strategy.RecursiveRemove; import org.jclouds.atmosonline.saas.config.AtmosStorageContextModule; import org.jclouds.blobstore.AsyncBlobStore; import org.jclouds.blobstore.BlobMap; @@ -34,8 +33,6 @@ import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.InputStreamMap; import org.jclouds.blobstore.config.BlobStoreMapModule; import org.jclouds.blobstore.internal.BlobStoreContextImpl; -import org.jclouds.blobstore.strategy.ClearContainerStrategy; -import org.jclouds.blobstore.strategy.ClearListStrategy; import org.jclouds.blobstore.strategy.ContainsValueInListStrategy; import org.jclouds.lifecycle.Closer; import org.jclouds.rest.RestContext; @@ -56,8 +53,6 @@ public class AtmosBlobStoreContextModule extends AtmosStorageContextModule { bind(AsyncBlobStore.class).to(AtmosAsyncBlobStore.class).asEagerSingleton(); bind(BlobStore.class).to(AtmosBlobStore.class).asEagerSingleton(); bind(ContainsValueInListStrategy.class).to(FindMD5InUserMetadata.class); - bind(ClearListStrategy.class).to(RecursiveRemove.class); - bind(ClearContainerStrategy.class).to(RecursiveRemove.class); } @Provides diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobToSystemMetadata.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobToSystemMetadata.java index 4ba4ed6958..e6ca441756 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobToSystemMetadata.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobToSystemMetadata.java @@ -32,9 +32,9 @@ import com.google.common.base.Function; @Singleton public class BlobToSystemMetadata implements Function { public SystemMetadata apply(BlobMetadata base) { - return new SystemMetadata(null, base.getLastModified(), null, null, null, 1, null, base - .getName(), null, (base.getSize() != null) ? base.getSize() : 0, FileType.REGULAR, - "root"); + return new SystemMetadata(base.getContentMD5(), null, base.getLastModified(), null, null, + null, 1, null, base.getName(), null, (base.getSize() != null) ? base.getSize() : 0, + FileType.REGULAR, "root"); } } \ No newline at end of file diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlob.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlob.java index 0672a72156..5a7fb4bdc2 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlob.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlob.java @@ -49,6 +49,7 @@ public class ObjectToBlob implements Function { Blob blob = blobFactory.create(object2BlobMd.apply(from)); if (from.getContentMetadata().getContentLength() != null) blob.setContentLength(from.getContentMetadata().getContentLength()); + blob.getMetadata().setContentMD5(from.getSystemMetadata().getContentMD5()); blob.setPayload(checkNotNull(from.getPayload(), "payload: " + from)); blob.setAllHeaders(from.getAllHeaders()); return blob; diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java index 8a7229c5da..a16a5cc236 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java @@ -31,7 +31,6 @@ import org.jclouds.atmosonline.saas.functions.AtmosObjectName; import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; -import org.jclouds.encryption.EncryptionService; import com.google.common.base.Function; import com.google.common.collect.ImmutableSet; @@ -46,12 +45,10 @@ public class ObjectToBlobMetadata implements Function systemMetadata = ImmutableSet.of("atime", "mtime", "ctime", "itime", "type", "uid", "gid", "objectid", "objname", "size", "nlink", "policyname", "content-md5"); - private final EncryptionService encryptionService; @Inject - protected ObjectToBlobMetadata(AtmosObjectName objectName, EncryptionService encryptionService) { + protected ObjectToBlobMetadata(AtmosObjectName objectName) { this.objectName = objectName; - this.encryptionService = encryptionService; } public MutableBlobMetadata apply(AtmosObject from) { @@ -60,9 +57,7 @@ public class ObjectToBlobMetadata implements Function - * - * ==================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ==================================================================== - */ -package org.jclouds.atmosonline.saas.blobstore.strategy; - -import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion; - -import java.util.Map; -import java.util.concurrent.ExecutorService; - -import javax.annotation.Resource; -import javax.inject.Named; -import javax.inject.Singleton; - -import org.jclouds.Constants; -import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient; -import org.jclouds.atmosonline.saas.AtmosStorageClient; -import org.jclouds.atmosonline.saas.domain.DirectoryEntry; -import org.jclouds.atmosonline.saas.domain.FileType; -import org.jclouds.blobstore.internal.BlobRuntimeException; -import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.blobstore.strategy.ClearContainerStrategy; -import org.jclouds.blobstore.strategy.ClearListStrategy; -import org.jclouds.logging.Logger; -import org.jclouds.util.Utils; - -import com.google.common.base.Function; -import com.google.common.base.Supplier; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.inject.Inject; - -/** - * Recursively remove a path. - * - * @author Adrian Cole - */ -@Singleton -public class RecursiveRemove implements ClearListStrategy, ClearContainerStrategy { - /** - * maximum duration of an blob Request - */ - @Inject(optional = true) - @Named(Constants.PROPERTY_HTTP_REQUEST_TIMEOUT) - protected Long maxTime; - protected final AtmosStorageAsyncClient async; - protected final AtmosStorageClient sync; - private final ExecutorService userExecutor; - - @Resource - protected Logger logger = Logger.NULL; - - @Inject - public RecursiveRemove(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, - AtmosStorageAsyncClient connection, AtmosStorageClient sync) { - this.async = connection; - this.sync = sync; - this.userExecutor = userExecutor; - } - - public void execute(String containerName) { - logger.debug("clearing container ", containerName); - execute(containerName, new ListContainerOptions().recursive()); - logger.trace("cleared container " + containerName); - } - - private ListenableFuture rm(final String fullPath, FileType type, boolean recursive) { - Map> responses = Maps.newHashMap(); - if ((type == FileType.DIRECTORY) && recursive) { - for (DirectoryEntry child : sync.listDirectory(fullPath)) { - responses.put(fullPath + "/" + child.getObjectName(), rm(fullPath + "/" - + child.getObjectName(), child.getType(), true)); - } - } - Map exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, - String.format("deleting from path: %s", fullPath)); - if (exceptions.size() > 0) - throw new BlobRuntimeException(String.format("deleting from path %s: %s", fullPath, - exceptions)); - - return Futures.compose(async.deletePath(fullPath), new Function() { - - public Void apply(Void from) { - try { - if (!Utils.enventuallyTrue(new Supplier() { - public Boolean get() { - return !sync.pathExists(fullPath); - } - }, maxTime != null ? maxTime : 1000)) { - throw new IllegalStateException(fullPath + " still exists after deleting!"); - } - return null; - } catch (InterruptedException e) { - throw new IllegalStateException(fullPath + " still exists after deleting!", e); - } - } - - }); - } - - public void execute(String path, ListContainerOptions options) { - if (options.getDir() != null) - path += "/" + options.getDir(); - Map> responses = Maps.newHashMap(); - for (DirectoryEntry md : sync.listDirectory(path)) { - responses.put(path + "/" + md.getObjectName(), rm(path + "/" + md.getObjectName(), md - .getType(), options.isRecursive())); - } - Map exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, - String.format("deleting from path: %s", path)); - if (exceptions.size() > 0) - throw new BlobRuntimeException(String - .format("deleting from path %s: %s", path, exceptions)); - } - -} \ No newline at end of file diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/DirectoryEntry.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/DirectoryEntry.java index 8a3b5d9899..5be767f585 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/DirectoryEntry.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/DirectoryEntry.java @@ -88,4 +88,10 @@ public class DirectoryEntry implements Comparable { return false; return true; } + + @Override + public String toString() { + return "DirectoryEntry [type=" + type + ", objectid=" + objectid + ", objname=" + objname + + "]"; + } } \ No newline at end of file diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/SystemMetadata.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/SystemMetadata.java index de1517e536..a30d05f783 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/SystemMetadata.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/SystemMetadata.java @@ -20,6 +20,8 @@ package org.jclouds.atmosonline.saas.domain; import java.util.Date; +import javax.annotation.Nullable; + /** * Metadata of a Atmos Online object * @@ -36,10 +38,12 @@ public class SystemMetadata extends DirectoryEntry { private final String policyname; private final long size; private final String uid; + private final byte[] contentmd5; - public SystemMetadata(Date atime, Date ctime, String gid, Date itime, Date mtime, int nlink, + public SystemMetadata(@Nullable byte [] contentmd5, Date atime, Date ctime, String gid, Date itime, Date mtime, int nlink, String objectid, String objname, String policyname, long size, FileType type, String uid) { super(objectid, type, objname); + this.contentmd5 = contentmd5; this.atime = atime; this.ctime = ctime; this.gid = gid; @@ -87,6 +91,10 @@ public class SystemMetadata extends DirectoryEntry { return uid; } + public byte[] getContentMD5() { + return contentmd5; + } + @Override public int hashCode() { final int prime = 31; @@ -154,4 +162,9 @@ public class SystemMetadata extends DirectoryEntry { return true; } + @Override + public String toString() { + return "[type=" + getType() + ", id=" + getObjectID() + ", name=" + getObjectName() + "]"; + } + } \ No newline at end of file diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/internal/AtmosObjectImpl.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/internal/AtmosObjectImpl.java index c20f700913..4f636bb69d 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/internal/AtmosObjectImpl.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/domain/internal/AtmosObjectImpl.java @@ -78,6 +78,27 @@ public class AtmosObjectImpl extends BasePayloadEnclosingImpl implements AtmosOb this.allHeaders = checkNotNull(allHeaders, "allHeaders"); } + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return systemMetadata != null ? systemMetadata.hashCode() : super.hashCode(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object obj) { + return systemMetadata != null ? systemMetadata.equals(obj) : super.equals(obj); + } + + @Override + public String toString() { + return "[metadata=" + systemMetadata + "]"; + } + /** * {@inheritDoc} */ diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeaders.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeaders.java index 95df08dc60..658db72326 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeaders.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeaders.java @@ -29,6 +29,7 @@ import org.jclouds.atmosonline.saas.domain.FileType; import org.jclouds.atmosonline.saas.domain.SystemMetadata; import org.jclouds.atmosonline.saas.reference.AtmosStorageHeaders; import org.jclouds.date.DateService; +import org.jclouds.encryption.EncryptionService; import org.jclouds.http.HttpResponse; import com.google.common.base.Function; @@ -40,10 +41,13 @@ import com.google.common.collect.Maps; @Singleton public class ParseSystemMetadataFromHeaders implements Function { private final DateService dateService; + private final EncryptionService encryptionService; @Inject - public ParseSystemMetadataFromHeaders(DateService dateService) { + public ParseSystemMetadataFromHeaders(DateService dateService, + EncryptionService encryptionService) { this.dateService = dateService; + this.encryptionService = encryptionService; } public SystemMetadata apply(HttpResponse from) { @@ -56,8 +60,9 @@ public class ParseSystemMetadataFromHeaders implements Function= 12 : String.format("Should be 12 entries in %s", metaMap); - - return new SystemMetadata(dateService.iso8601SecondsDateParse(checkNotNull(metaMap + byte[] md5 = metaMap.containsKey("content-md5") ? encryptionService.fromHexString(metaMap + .get("content-md5")) : null; + return new SystemMetadata(md5, dateService.iso8601SecondsDateParse(checkNotNull(metaMap .get("atime"), "atime")), dateService.iso8601SecondsDateParse(checkNotNull(metaMap .get("ctime"), "ctime")), checkNotNull(metaMap.get("gid"), "gid"), dateService .iso8601SecondsDateParse(checkNotNull(metaMap.get("itime"), "itime")), dateService diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/handlers/ParseAtmosStorageErrorFromXmlContent.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/handlers/ParseAtmosStorageErrorFromXmlContent.java index e7a879f430..8a3c8cf8aa 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/handlers/ParseAtmosStorageErrorFromXmlContent.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/handlers/ParseAtmosStorageErrorFromXmlContent.java @@ -60,8 +60,8 @@ public class ParseAtmosStorageErrorFromXmlContent implements HttpErrorHandler { this.utils = utils; } - public static final Pattern CONTAINER_PATH = Pattern.compile("^/rest/namespace/?([^/]+)[/]?$"); - public static final Pattern CONTAINER_KEY_PATH = Pattern + public static final Pattern DIRECTORY_PATH = Pattern.compile("^/rest/namespace/?([^/]+)/$"); + public static final Pattern DIRECTORY_KEY_PATH = Pattern .compile("^/rest/namespace/?([^/]+)/(.*)"); public void handleError(HttpCommand command, HttpResponse response) { @@ -84,11 +84,11 @@ public class ParseAtmosStorageErrorFromXmlContent implements HttpErrorHandler { "%s -> %s", command.getRequest().getRequestLine(), response .getStatusLine()); String path = command.getRequest().getEndpoint().getPath(); - Matcher matcher = CONTAINER_PATH.matcher(path); + Matcher matcher = DIRECTORY_PATH.matcher(path); if (matcher.find()) { exception = new ContainerNotFoundException(matcher.group(1), message); } else { - matcher = CONTAINER_KEY_PATH.matcher(path); + matcher = DIRECTORY_KEY_PATH.matcher(path); if (matcher.find()) { exception = new KeyNotFoundException(matcher.group(1), matcher.group(2), message); diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/options/ListOptions.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/options/ListOptions.java index 726e7943e5..4da0e5608b 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/options/ListOptions.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/options/ListOptions.java @@ -51,7 +51,7 @@ public class ListOptions extends BaseHttpRequestOptions { */ public ListOptions limit(int maxresults) { checkState(maxresults >= 0, "maxresults must be >= 0"); - checkState(maxresults <= 10000, "maxresults must be <= 5000"); + checkState(maxresults <= 10000, "maxresults must be <= 10000"); headers.put("x-emc-limit", Integer.toString(maxresults)); return this; } diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/util/AtmosStorageUtils.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/util/AtmosStorageUtils.java index 110453e037..5058ed3910 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/util/AtmosStorageUtils.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/util/AtmosStorageUtils.java @@ -24,13 +24,20 @@ import java.io.InputStream; import javax.inject.Inject; import javax.inject.Provider; +import org.jclouds.atmosonline.saas.AtmosStorageClient; +import org.jclouds.atmosonline.saas.blobstore.functions.BlobToObject; import org.jclouds.atmosonline.saas.domain.AtmosStorageError; import org.jclouds.atmosonline.saas.filters.SignRequest; import org.jclouds.atmosonline.saas.xml.ErrorHandler; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.encryption.EncryptionService; import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpException; import org.jclouds.http.HttpResponse; import org.jclouds.http.functions.ParseSax; +import org.jclouds.util.Utils; + +import com.google.common.base.Supplier; /** * Encryption, Hashing, and IO Utilities needed to sign and verify Atmos Storage requests and @@ -60,18 +67,44 @@ public class AtmosStorageUtils { } + public static String putBlob(final AtmosStorageClient sync, EncryptionService encryptionService, + BlobToObject blob2Object, String container, Blob blob) { + final String path = container + "/" + blob.getMetadata().getName(); + deleteAndEnsureGone(sync, path); + if (blob.getMetadata().getContentMD5() != null) + blob.getMetadata().getUserMetadata().put("content-md5", + encryptionService.toHexString(blob.getMetadata().getContentMD5())); + sync.createFile(container, blob2Object.apply(blob)); + return path; + } + + public static void deleteAndEnsureGone(final AtmosStorageClient sync, final String path) { + try { + if (!Utils.enventuallyTrue(new Supplier() { + public Boolean get() { + sync.deletePath(path); + return !sync.pathExists(path); + } + }, 3000)) { + throw new IllegalStateException(path + " still exists after deleting!"); + } + } catch (InterruptedException e) { + new IllegalStateException(path + " interrupted during deletion!", e); + } + } + public AtmosStorageError parseAtmosStorageErrorFromContent(HttpCommand command, HttpResponse response, String content) throws HttpException { return parseAtmosStorageErrorFromContent(command, response, new ByteArrayInputStream(content .getBytes())); } - + public static String adjustContainerIfDirOptionPresent(String container, org.jclouds.blobstore.options.ListContainerOptions options) { if (options != org.jclouds.blobstore.options.ListContainerOptions.NONE) { - if (options.isRecursive()) { - throw new UnsupportedOperationException("recursive not currently supported in emcsaas"); - } + // if (options.isRecursive()) { + // throw new UnsupportedOperationException("recursive not currently supported in emcsaas"); + // } if (options.getDir() != null) { container = container + "/" + options.getDir(); } diff --git a/atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java b/atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java index d6ee323183..efbff92f6a 100644 --- a/atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java +++ b/atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java @@ -26,22 +26,18 @@ import java.lang.reflect.UndeclaredThrowableException; import java.net.URI; import java.security.SecureRandom; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; -import org.jclouds.atmosonline.saas.blobstore.strategy.RecursiveRemove; import org.jclouds.atmosonline.saas.domain.AtmosObject; import org.jclouds.atmosonline.saas.domain.BoundedSet; import org.jclouds.atmosonline.saas.domain.DirectoryEntry; import org.jclouds.atmosonline.saas.domain.FileType; import org.jclouds.atmosonline.saas.domain.SystemMetadata; import org.jclouds.atmosonline.saas.options.ListOptions; -import org.jclouds.blobstore.ContainerNotFoundException; +import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.KeyAlreadyExistsException; import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; -import org.jclouds.blobstore.strategy.ClearContainerStrategy; import org.jclouds.http.HttpResponseException; import org.jclouds.http.Payloads; import org.jclouds.http.payloads.InputStreamPayload; @@ -107,7 +103,7 @@ public class AtmosStorageClientLiveTest { private static final int INCONSISTENCY_WINDOW = 5000; protected AtmosStorageClient connection; - private String containerPrefix = BaseBlobStoreIntegrationTest.CONTAINER_PREFIX+"live"; + private String containerPrefix = BaseBlobStoreIntegrationTest.CONTAINER_PREFIX + "live"; URI container1; URI container2; @@ -116,23 +112,15 @@ public class AtmosStorageClientLiveTest { public void setupClient() throws InterruptedException, ExecutionException, TimeoutException { String uid = checkNotNull(System.getProperty("jclouds.test.user"), "jclouds.test.user"); String key = checkNotNull(System.getProperty("jclouds.test.key"), "jclouds.test.key"); - - RestContext context = new AtmosStorageContextBuilder( + BlobStoreContext blobStoreContext = new AtmosStorageContextBuilder( new AtmosStoragePropertiesBuilder(uid, key).build()).withModules( - new Log4JLoggingModule()).buildContext(); - ExecutorService service = Executors.newCachedThreadPool(); + new Log4JLoggingModule()).buildBlobStoreContext(); + RestContext context = blobStoreContext + .getProviderSpecificContext(); connection = context.getApi(); - ClearContainerStrategy clearer = new RecursiveRemove(service, context.getAsyncApi(), - connection); for (DirectoryEntry entry : connection.listDirectories()) { - try { - if (entry.getObjectName().startsWith(containerPrefix)) { - clearer.execute(entry.getObjectName()); - deleteConfirmed(entry.getObjectName()); - } - } catch (ContainerNotFoundException e) { - if (entry.getType() != FileType.DIRECTORY) - throw e; + if (entry.getObjectName().startsWith(containerPrefix)) { + blobStoreContext.getBlobStore().deleteContainer(entry.getObjectName()); } } } diff --git a/atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientTest.java b/atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientTest.java index 2400e15442..e9d914220a 100644 --- a/atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientTest.java +++ b/atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientTest.java @@ -39,7 +39,6 @@ import org.jclouds.atmosonline.saas.options.ListOptions; import org.jclouds.atmosonline.saas.reference.AtmosStorageConstants; import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest; import org.jclouds.blobstore.config.BlobStoreObjectModule; -import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound; import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404; import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404; import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404; @@ -51,6 +50,7 @@ import org.jclouds.http.options.GetOptions; import org.jclouds.logging.Logger; import org.jclouds.logging.Logger.LoggerFactory; import org.jclouds.rest.RestClientTest; +import org.jclouds.rest.functions.ReturnNullOnResourceNotFound; import org.jclouds.rest.internal.GeneratedHttpRequest; import org.jclouds.rest.internal.RestAnnotationProcessor; import org.jclouds.util.Jsr330; @@ -221,7 +221,7 @@ public class AtmosStorageClientTest extends RestClientTest deletePath(String path) { - if (path.indexOf('/') == -1) - return compose(blobStore.deleteContainerImpl(path), new Function() { + if (path.indexOf('/') == path.length() - 1) { + // chop off the trailing slash + return compose(blobStore.deleteContainerImpl(path.substring(0, path.length() - 1)), + new Function() { - public Void apply(Boolean from) { - return null; - } + public Void apply(Boolean from) { + return null; + } - }); - else { + }); + } else { String container = path.substring(0, path.indexOf('/')); path = path.substring(path.indexOf('/') + 1); return blobStore.removeBlob(container, path); @@ -205,9 +207,10 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { } public ListenableFuture pathExists(final String path) { - if (path.indexOf('/') == -1 ) - return blobStore.containerExists(path); - else { + if (path.indexOf('/') == path.length() - 1) { + // chop off the trailing slash + return blobStore.containerExists(path.substring(0, path.length() - 1)); + } else { String container = path.substring(0, path.indexOf('/')); String blobName = path.substring(path.indexOf('/') + 1); try { diff --git a/atmos/src/test/resources/log4j.xml b/atmos/src/test/resources/log4j.xml index 1515121fb3..5dee9ab38d 100755 --- a/atmos/src/test/resources/log4j.xml +++ b/atmos/src/test/resources/log4j.xml @@ -1,4 +1,29 @@ + + + + @@ -45,10 +70,18 @@ --> - + + + + + + + + + - - + + @@ -61,28 +94,23 @@ - - - - - - + + - - - - - + + + + @@ -92,20 +120,15 @@ - + + + + --> diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java b/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java index 59b690418e..4a87577c98 100644 --- a/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java @@ -143,14 +143,14 @@ public class S3BlobStore extends BaseBlobStore { */ @Override public void deleteContainer(String container) { - deleteAndEnsurePathGone(container); + clearAndDeleteContainer(container); } /** * This implementation invokes {@link #clearContainer} then {@link S3Client#deleteBucketIfEmpty} * until it is true. */ - public void deleteAndEnsurePathGone(final String container) { + public void clearAndDeleteContainer(final String container) { try { if (!Utils.enventuallyTrue(new Supplier() { public Boolean get() { diff --git a/aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3BlobMapIntegrationTest.java b/aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3BlobMapIntegrationTest.java index c6aab6bd79..8ea3c1c4ac 100644 --- a/aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3BlobMapIntegrationTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3BlobMapIntegrationTest.java @@ -27,9 +27,4 @@ import org.testng.annotations.Test; @Test(groups = { "integration", "live" }, testName = "s3.S3BlobMapIntegrationTest") public class S3BlobMapIntegrationTest extends BaseBlobMapIntegrationTest { - @Override - protected int maxList() { - return 1000; - } - } \ No newline at end of file diff --git a/aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3InputStreamMapIntegrationTest.java b/aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3InputStreamMapIntegrationTest.java index de74637a24..b2c22248d3 100644 --- a/aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3InputStreamMapIntegrationTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3InputStreamMapIntegrationTest.java @@ -26,8 +26,5 @@ import org.testng.annotations.Test; */ @Test(groups = { "integration", "live" }, testName = "s3.S3InputStreamMapIntegrationTest") public class S3InputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest { - @Override - protected int maxList() { - return 1000; - } + } \ No newline at end of file diff --git a/azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/strategy/FindMD5InBlobProperties.java b/azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/strategy/FindMD5InBlobProperties.java index 797a9de916..42e0fab8dc 100644 --- a/azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/strategy/FindMD5InBlobProperties.java +++ b/azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/strategy/FindMD5InBlobProperties.java @@ -30,7 +30,7 @@ import org.jclouds.blobstore.functions.ObjectMD5; import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.ContainsValueInListStrategy; -import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy; +import org.jclouds.blobstore.strategy.ListBlobsInContainer; import com.google.common.base.Throwables; @@ -43,12 +43,12 @@ import com.google.common.base.Throwables; public class FindMD5InBlobProperties implements ContainsValueInListStrategy { protected final ObjectMD5 objectMD5; - protected final ListBlobMetadataStrategy getAllBlobMetadata; + protected final ListBlobsInContainer getAllBlobMetadata; private final AzureBlobClient client; @Inject private FindMD5InBlobProperties(ObjectMD5 objectMD5, - ListBlobMetadataStrategy getAllBlobMetadata, AzureBlobClient client) { + ListBlobsInContainer getAllBlobMetadata, AzureBlobClient client) { this.objectMD5 = objectMD5; this.getAllBlobMetadata = getAllBlobMetadata; this.client = client; diff --git a/azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobClientLiveTest.java b/azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobClientLiveTest.java index a661463b30..0a1f8c64e0 100644 --- a/azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobClientLiveTest.java +++ b/azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobClientLiveTest.java @@ -18,6 +18,8 @@ */ package org.jclouds.azure.storage.blob; +import static org.jclouds.azure.storage.blob.options.CreateContainerOptions.Builder.withMetadata; +import static org.jclouds.azure.storage.blob.options.CreateContainerOptions.Builder.withPublicAcl; import static org.jclouds.azure.storage.options.ListOptions.Builder.includeMetadata; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -33,7 +35,6 @@ import org.jclouds.azure.storage.blob.domain.AzureBlob; import org.jclouds.azure.storage.blob.domain.BlobProperties; import org.jclouds.azure.storage.blob.domain.ContainerProperties; import org.jclouds.azure.storage.blob.domain.ListBlobsResponse; -import org.jclouds.azure.storage.blob.options.CreateContainerOptions; import org.jclouds.azure.storage.blob.options.ListBlobsOptions; import org.jclouds.azure.storage.domain.BoundedSet; import org.jclouds.azure.storage.options.ListOptions; @@ -59,7 +60,7 @@ import com.google.inject.internal.Iterables; @Test(groups = "live", sequential = true, testName = "azureblob.AzureBlobClientLiveTest") public class AzureBlobClientLiveTest { - protected AzureBlobClient connection; + protected AzureBlobClient client; private String containerPrefix = System.getProperty("user.name") + "-azureblob"; private EncryptionService encryptionService = new JCEEncryptionService(); @@ -68,13 +69,13 @@ public class AzureBlobClientLiveTest { public void setupClient() { account = System.getProperty("jclouds.test.user"); String key = System.getProperty("jclouds.test.key"); - connection = (AzureBlobClient) AzureBlobContextFactory.createContext(account, key, + client = (AzureBlobClient) AzureBlobContextFactory.createContext(account, key, new Log4JLoggingModule()).getProviderSpecificContext().getApi(); } @Test public void testListContainers() throws Exception { - Set response = connection.listContainers(); + Set response = client.listContainers(); assert null != response; long initialContainerCount = response.size(); assertTrue(initialContainerCount >= 0); @@ -91,8 +92,8 @@ public class AzureBlobClientLiveTest { while (!created) { privateContainer = containerPrefix + new SecureRandom().nextInt(); try { - created = connection.createContainer(privateContainer, CreateContainerOptions.Builder - .withMetadata(ImmutableMultimap.of("foo", "bar"))); + created = client.createContainer(privateContainer, withMetadata(ImmutableMultimap + .of("foo", "bar"))); } catch (UndeclaredThrowableException e) { HttpResponseException htpe = (HttpResponseException) e.getCause().getCause(); if (htpe.getResponse().getStatusCode() == 409) @@ -100,11 +101,11 @@ public class AzureBlobClientLiveTest { throw e; } } - Set response = connection.listContainers(includeMetadata()); + Set response = client.listContainers(includeMetadata()); assert null != response; long containerCount = response.size(); assertTrue(containerCount >= 1); - ListBlobsResponse list = connection.listBlobs(privateContainer); + ListBlobsResponse list = client.listBlobs(privateContainer); assertEquals(list.getUrl(), URI.create(String.format("https://%s.blob.core.windows.net/%s", account, privateContainer))); // TODO ... check to see the container actually exists @@ -116,8 +117,7 @@ public class AzureBlobClientLiveTest { while (!created) { publicContainer = containerPrefix + new SecureRandom().nextInt(); try { - created = connection.createContainer(publicContainer, CreateContainerOptions.Builder - .withPublicAcl()); + created = client.createContainer(publicContainer, withPublicAcl()); } catch (UndeclaredThrowableException e) { HttpResponseException htpe = (HttpResponseException) e.getCause().getCause(); if (htpe.getResponse().getStatusCode() == 409) @@ -134,7 +134,7 @@ public class AzureBlobClientLiveTest { @Test(timeOut = 5 * 60 * 1000) public void testCreatePublicRootContainer() throws Exception { try { - connection.deleteRootContainer(); + client.deleteRootContainer(); } catch (ContainerNotFoundException e) { Thread.sleep(5000); } catch (AzureStorageResponseException htpe) { @@ -148,7 +148,7 @@ public class AzureBlobClientLiveTest { boolean created = false; while (!created) { try { - created = connection.createRootContainer(); + created = client.createRootContainer(); } catch (AzureStorageResponseException htpe) { if (htpe.getResponse().getStatusCode() == 409) {// TODO look for specific message Thread.sleep(5000); @@ -158,7 +158,7 @@ public class AzureBlobClientLiveTest { } } } - ListBlobsResponse list = connection.listBlobs(); + ListBlobsResponse list = client.listBlobs(); assertEquals(list.getUrl(), URI.create(String.format( "https://%s.blob.core.windows.net/%%24root", account))); } @@ -166,7 +166,7 @@ public class AzureBlobClientLiveTest { @Test public void testListContainersWithOptions() throws Exception { - BoundedSet response = connection.listContainers(ListOptions.Builder + BoundedSet response = client.listContainers(ListOptions.Builder .prefix(privateContainer).maxResults(1).includeMetadata()); assert null != response; long initialContainerCount = response.size(); @@ -177,7 +177,7 @@ public class AzureBlobClientLiveTest { @Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testCreatePublicRootContainer" }) public void testDeleteRootContainer() throws Exception { - connection.deleteRootContainer(); + client.deleteRootContainer(); // TODO loop for up to 30 seconds checking if they are really gone } @@ -186,19 +186,19 @@ public class AzureBlobClientLiveTest { public void testListOwnedContainers() throws Exception { // Test default listing - Set response = connection.listContainers(); + Set response = client.listContainers(); // assertEquals(response.size(), initialContainerCount + 2);// if the containers already // exist, this will fail // Test listing with options - response = connection.listContainers(ListOptions.Builder.prefix( + response = client.listContainers(ListOptions.Builder.prefix( privateContainer.substring(0, privateContainer.length() - 1)).maxResults(1) .includeMetadata()); assertEquals(response.size(), 1); assertEquals(Iterables.getOnlyElement(response).getName(), privateContainer); assertEquals(Iterables.getOnlyElement(response).getMetadata(), ImmutableMap.of("foo", "bar")); - response = connection.listContainers(ListOptions.Builder.prefix(publicContainer) + response = client.listContainers(ListOptions.Builder.prefix(publicContainer) .maxResults(1)); assertEquals(response.size(), 1); assertEquals(Iterables.getOnlyElement(response).getName(), publicContainer); @@ -207,14 +207,14 @@ public class AzureBlobClientLiveTest { @Test public void testDeleteOneContainer() throws Exception { - connection.deleteContainer("does-not-exist"); + client.deleteContainer("does-not-exist"); } @Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testListOwnedContainers", "testObjectOperations" }) public void testDeleteContainer() throws Exception { - connection.deleteContainer(privateContainer); - connection.deleteContainer(publicContainer); + client.deleteContainer(privateContainer); + client.deleteContainer(publicContainer); // TODO loop for up to 30 seconds checking if they are really gone } @@ -224,7 +224,7 @@ public class AzureBlobClientLiveTest { String data = "Here is my data"; // Test PUT with string data, ETag hash, and a piece of metadata - AzureBlob object = connection.newBlob(); + AzureBlob object = client.newBlob(); object.getProperties().setName("object"); object.setPayload(data); object.setContentLength(data.length()); @@ -232,15 +232,15 @@ public class AzureBlobClientLiveTest { object.getProperties().setContentType("text/plain"); object.getProperties().getMetadata().put("mykey", "metadata-value"); byte[] md5 = object.getProperties().getContentMD5(); - String newEtag = connection.putBlob(privateContainer, object); + String newEtag = client.putBlob(privateContainer, object); assertEquals(encryptionService.toHexString(md5), encryptionService.toHexString(object .getProperties().getContentMD5())); // Test HEAD of missing object - assert connection.getBlobProperties(privateContainer, "non-existent-object") == null; + assert client.getBlobProperties(privateContainer, "non-existent-object") == null; // Test HEAD of object - BlobProperties metadata = connection.getBlobProperties(privateContainer, object + BlobProperties metadata = client.getBlobProperties(privateContainer, object .getProperties().getName()); // TODO assertEquals(metadata.getName(), object.getProperties().getName()); // we can't check this while hacking around lack of content-md5, as GET of the first byte will @@ -261,14 +261,14 @@ public class AzureBlobClientLiveTest { // Multimap userMetadata = LinkedHashMultimap.create(); // userMetadata.put("New-Metadata-1", "value-1"); // userMetadata.put("New-Metadata-2", "value-2"); - // assertTrue(connection.setBlobProperties(privateContainer, object.getProperties().getName(), + // assertTrue(client.setBlobProperties(privateContainer, object.getProperties().getName(), // userMetadata)); // Test GET of missing object - assert connection.getBlob(privateContainer, "non-existent-object") == null; + assert client.getBlob(privateContainer, "non-existent-object") == null; // Test GET of object (including updated metadata) - AzureBlob getBlob = connection.getBlob(privateContainer, object.getProperties().getName()); + AzureBlob getBlob = client.getBlob(privateContainer, object.getProperties().getName()); assertEquals(Utils.toStringAndClose(getBlob.getContent()), data); // TODO assertEquals(getBlob.getName(), object.getProperties().getName()); assertEquals(getBlob.getContentLength(), new Long(data.length())); @@ -288,7 +288,7 @@ public class AzureBlobClientLiveTest { assertEquals(metadata.getMetadata().get("mykey"), "metadata-value"); // test listing - ListBlobsResponse response = connection.listBlobs(privateContainer, ListBlobsOptions.Builder + ListBlobsResponse response = client.listBlobs(privateContainer, ListBlobsOptions.Builder .prefix( object.getProperties().getName().substring(0, object.getProperties().getName().length() - 1)).maxResults(1) @@ -303,25 +303,25 @@ public class AzureBlobClientLiveTest { String incorrectEtag = "0" + correctEtag.substring(1); object.getProperties().setETag(incorrectEtag); try { - connection.putBlob(privateContainer, object); + client.putBlob(privateContainer, object); } catch (Throwable e) { assertEquals(e.getCause().getClass(), HttpResponseException.class); assertEquals(((HttpResponseException) e.getCause()).getResponse().getStatusCode(), 422); } ByteArrayInputStream bais = new ByteArrayInputStream(data.getBytes("UTF-8")); - object = connection.newBlob(); + object = client.newBlob(); object.getProperties().setName("chunked-object"); object.setPayload(bais); object.setContentLength(new Long(data.getBytes().length)); - newEtag = connection.putBlob(privateContainer, object); + newEtag = client.putBlob(privateContainer, object); assertEquals(encryptionService.toHexString(md5), encryptionService.toHexString(getBlob .getProperties().getContentMD5())); // Test GET with options // Non-matching ETag try { - connection.getBlob(privateContainer, object.getProperties().getName(), GetOptions.Builder + client.getBlob(privateContainer, object.getProperties().getName(), GetOptions.Builder .ifETagDoesntMatch(newEtag)); } catch (Exception e) { assertEquals(e.getCause().getClass(), HttpResponseException.class); @@ -330,7 +330,7 @@ public class AzureBlobClientLiveTest { // Matching ETag TODO this shouldn't fail!!! try { - getBlob = connection.getBlob(privateContainer, object.getProperties().getName(), + getBlob = client.getBlob(privateContainer, object.getProperties().getName(), GetOptions.Builder.ifETagMatches(newEtag)); assertEquals(getBlob.getProperties().getETag(), newEtag); } catch (HttpResponseException e) { @@ -340,13 +340,13 @@ public class AzureBlobClientLiveTest { // Range // doesn't work per // http://social.msdn.microsoft.com/Forums/en-US/windowsazure/thread/479fa63f-51df-4b66-96b5-33ae362747b6 - // getBlob = connection + // getBlob = client // .getBlob(privateContainer, object.getProperties().getName(), // GetOptions.Builder.startAt(8)).get(120, // TimeUnit.SECONDS); // assertEquals(Utils.toStringAndClose((InputStream) getBlob.getData()), data.substring(8)); - connection.deleteBlob(privateContainer, "object"); - connection.deleteBlob(privateContainer, "chunked-object"); + client.deleteBlob(privateContainer, "object"); + client.deleteBlob(privateContainer, "chunked-object"); } } diff --git a/azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobInputStreamMapIntegrationTest.java b/azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobInputStreamMapIntegrationTest.java index f498302b39..6700b975ea 100644 --- a/azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobInputStreamMapIntegrationTest.java +++ b/azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobInputStreamMapIntegrationTest.java @@ -27,9 +27,4 @@ import org.testng.annotations.Test; @Test(groups = { "integration", "live" }, testName = "azureblob.AzureBlobInputStreamMapIntegrationTest") public class AzureBlobInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest { - @Override - protected int maxList() { - return 5000; - } - } \ No newline at end of file diff --git a/azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobMapIntegrationTest.java b/azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobMapIntegrationTest.java index 1b180f9090..5a18e1504d 100644 --- a/azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobMapIntegrationTest.java +++ b/azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobMapIntegrationTest.java @@ -27,9 +27,4 @@ import org.testng.annotations.Test; @Test(groups = { "integration", "live" }, testName = "azureblob.AzureBlobMapIntegrationTest") public class AzureBlobMapIntegrationTest extends BaseBlobMapIntegrationTest { - @Override - protected int maxList() { - return 5000; - } - } \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/BlobMap.java b/blobstore/src/main/java/org/jclouds/blobstore/BlobMap.java index 4bfab1e238..40a308c043 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/BlobMap.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/BlobMap.java @@ -18,10 +18,9 @@ */ package org.jclouds.blobstore; -import javax.annotation.Nullable; - import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.internal.BlobMapImpl; +import org.jclouds.blobstore.options.ListContainerOptions; import com.google.inject.ImplementedBy; @@ -38,7 +37,7 @@ public interface BlobMap extends ListableMap { Blob newBlob(String name); public static interface Factory { - BlobMap create(String containerName, @Nullable String dir); + BlobMap create(String containerName, ListContainerOptions options); } } \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/BlobStoreContext.java b/blobstore/src/main/java/org/jclouds/blobstore/BlobStoreContext.java index e911f46005..70f8b425d2 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/BlobStoreContext.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/BlobStoreContext.java @@ -20,6 +20,7 @@ package org.jclouds.blobstore; import org.jclouds.blobstore.attr.ConsistencyModels; import org.jclouds.blobstore.internal.BlobStoreContextImpl; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.rest.RestContext; import com.google.inject.ImplementedBy; @@ -39,6 +40,8 @@ public interface BlobStoreContext { * * @param container */ + InputStreamMap createInputStreamMap(String container, ListContainerOptions options); + InputStreamMap createInputStreamMap(String container); /** @@ -46,6 +49,8 @@ public interface BlobStoreContext { * * @param container */ + BlobMap createBlobMap(String container, ListContainerOptions options); + BlobMap createBlobMap(String container); AsyncBlobStore getAsyncBlobStore(); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/ContainerNotFoundException.java b/blobstore/src/main/java/org/jclouds/blobstore/ContainerNotFoundException.java index 7b3377cec9..55fc2bfaf6 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/ContainerNotFoundException.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/ContainerNotFoundException.java @@ -18,12 +18,14 @@ */ package org.jclouds.blobstore; +import org.jclouds.rest.ResourceNotFoundException; + /** * Thrown when a container cannot be located. * * @author Adrian Cole */ -public class ContainerNotFoundException extends RuntimeException { +public class ContainerNotFoundException extends ResourceNotFoundException { private String container; diff --git a/blobstore/src/main/java/org/jclouds/blobstore/InputStreamMap.java b/blobstore/src/main/java/org/jclouds/blobstore/InputStreamMap.java index 50e0e0a8c9..7dd3a49577 100755 --- a/blobstore/src/main/java/org/jclouds/blobstore/InputStreamMap.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/InputStreamMap.java @@ -22,9 +22,8 @@ import java.io.File; import java.io.InputStream; import java.util.Map; -import javax.annotation.Nullable; - import org.jclouds.blobstore.internal.InputStreamMapImpl; +import org.jclouds.blobstore.options.ListContainerOptions; import com.google.inject.ImplementedBy; @@ -43,7 +42,7 @@ import com.google.inject.ImplementedBy; @ImplementedBy(InputStreamMapImpl.class) public interface InputStreamMap extends ListableMap { public static interface Factory { - InputStreamMap create(String containerName, @Nullable String dir); + InputStreamMap create(String containerName, ListContainerOptions options); } InputStream putString(String key, String value); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/config/BlobStoreMapModule.java b/blobstore/src/main/java/org/jclouds/blobstore/config/BlobStoreMapModule.java index f684e38c95..016ed12c25 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/config/BlobStoreMapModule.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/config/BlobStoreMapModule.java @@ -18,7 +18,6 @@ */ package org.jclouds.blobstore.config; -import javax.annotation.Nullable; import javax.inject.Inject; import org.jclouds.blobstore.BlobMap; @@ -27,10 +26,11 @@ import org.jclouds.blobstore.InputStreamMap; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.internal.BlobMapImpl; import org.jclouds.blobstore.internal.InputStreamMapImpl; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.ContainsValueInListStrategy; import org.jclouds.blobstore.strategy.GetBlobsInListStrategy; import org.jclouds.blobstore.strategy.PutBlobsStrategy; -import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer; +import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders; import com.google.inject.AbstractModule; import com.google.inject.Scopes; @@ -62,11 +62,11 @@ public class BlobStoreMapModule extends AbstractModule { @Inject PutBlobsStrategy putBlobsStrategy; @Inject - ListBlobMetadataInContainer listStrategy; + ListContainerAndRecurseThroughFolders listStrategy; - public BlobMap create(String containerName, @Nullable String dir) { + public BlobMap create(String containerName, ListContainerOptions options) { return new BlobMapImpl(connection, getAllBlobs, containsValueStrategy, putBlobsStrategy, - listStrategy, containerName, dir); + listStrategy, containerName, options); } } @@ -83,11 +83,11 @@ public class BlobStoreMapModule extends AbstractModule { @Inject PutBlobsStrategy putBlobsStrategy; @Inject - ListBlobMetadataInContainer listStrategy; + ListContainerAndRecurseThroughFolders listStrategy; - public InputStreamMap create(String containerName, @Nullable String dir) { + public InputStreamMap create(String containerName, ListContainerOptions options) { return new InputStreamMapImpl(connection, blobFactory, getAllBlobs, listStrategy, - containsValueStrategy, putBlobsStrategy, containerName, dir); + containsValueStrategy, putBlobsStrategy, containerName, options); } } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java index 6285008314..fb0bba9dc4 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java @@ -32,6 +32,7 @@ import javax.inject.Named; import org.jclouds.Constants; import org.jclouds.blobstore.AsyncBlobStore; +import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; @@ -238,8 +239,12 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { try { if (!Utils.enventuallyTrue(new Supplier() { public Boolean get() { - blobUtils.clearContainer(container, recursive()); - return deleteAndVerifyContainerGone(container); + try { + clearContainer(container, recursive()); + return deleteAndVerifyContainerGone(container); + } catch (ContainerNotFoundException e) { + return true; + } } }, 30000)) { diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java index 82ee174cc7..8bda2090ff 100755 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java @@ -20,13 +20,10 @@ package org.jclouds.blobstore.internal; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static org.jclouds.blobstore.options.ListContainerOptions.Builder.inDirectory; import java.util.Map; import java.util.Set; -import javax.annotation.Nullable; - import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; @@ -37,7 +34,7 @@ import org.jclouds.blobstore.options.ListContainerOptions.ImmutableListContainer import org.jclouds.blobstore.strategy.ContainsValueInListStrategy; import org.jclouds.blobstore.strategy.GetBlobsInListStrategy; import org.jclouds.blobstore.strategy.PutBlobsStrategy; -import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer; +import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders; import com.google.common.base.Function; import com.google.common.collect.Iterables; @@ -58,7 +55,7 @@ public abstract class BaseBlobMap implements Map { protected final ListContainerOptions options; protected final GetBlobsInListStrategy getAllBlobs; protected final ContainsValueInListStrategy containsValueStrategy; - protected final ListBlobMetadataInContainer listStrategy; + protected final ListContainerAndRecurseThroughFolders listStrategy; protected final PutBlobsStrategy putBlobsStrategy; static class StripPath implements Function { @@ -98,11 +95,15 @@ public abstract class BaseBlobMap implements Map { @Inject public BaseBlobMap(BlobStore blobstore, GetBlobsInListStrategy getAllBlobs, ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy, - ListBlobMetadataInContainer listStrategy, String containerName, @Nullable String dir) { + ListContainerAndRecurseThroughFolders listStrategy, String containerName, + ListContainerOptions options) { this.blobstore = checkNotNull(blobstore, "blobstore"); this.containerName = checkNotNull(containerName, "container"); - this.options = new ImmutableListContainerOptions(dir != null ? inDirectory(dir) - : ListContainerOptions.NONE); + checkArgument(containerName.indexOf('/') == -1, + "please specify directory path using the option: inDirectory, not encoded in the container name"); + this.options = checkNotNull(options, "options") instanceof ImmutableListContainerOptions ? options + : new ImmutableListContainerOptions(options); + String dir = options.getDir(); if (dir == null) { prefixer = new PassThrough(); pathStripper = prefixer; diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java index a3edbe2e31..9a3d952e66 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java @@ -24,6 +24,7 @@ import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursi import javax.inject.Inject; import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; @@ -172,15 +173,19 @@ public abstract class BaseBlobStore implements BlobStore { */ @Override public void deleteContainer(final String container) { - deleteAndEnsurePathGone(container); + clearAndDeleteContainer(container); } - protected void deleteAndEnsurePathGone(final String container) { + protected void clearAndDeleteContainer(final String container) { try { if (!Utils.enventuallyTrue(new Supplier() { public Boolean get() { - clearContainer(container, recursive()); - return deleteAndVerifyContainerGone(container); + try { + clearContainer(container, recursive()); + return deleteAndVerifyContainerGone(container); + } catch (ContainerNotFoundException e) { + return true; + } } }, 30000)) { diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java index 117d329a23..dd2bd03bba 100755 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java @@ -21,17 +21,17 @@ package org.jclouds.blobstore.internal; import java.util.Collection; import java.util.Map; -import javax.annotation.Nullable; import javax.inject.Inject; import org.jclouds.blobstore.BlobMap; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.ContainsValueInListStrategy; import org.jclouds.blobstore.strategy.GetBlobsInListStrategy; import org.jclouds.blobstore.strategy.PutBlobsStrategy; -import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer; +import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders; /** * Map representation of a live connection to a Blob Service. @@ -46,9 +46,10 @@ public class BlobMapImpl extends BaseBlobMap implements BlobMap { @Inject public BlobMapImpl(BlobStore blobstore, GetBlobsInListStrategy getAllBlobs, ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy, - ListBlobMetadataInContainer listStrategy, String containerName, @Nullable String dir) { + ListContainerAndRecurseThroughFolders listStrategy, String containerName, + ListContainerOptions options) { super(blobstore, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy, - containerName, dir); + containerName, options); } @Override diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobStoreContextImpl.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobStoreContextImpl.java index 5b27e4e80a..159b299842 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobStoreContextImpl.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobStoreContextImpl.java @@ -29,7 +29,7 @@ import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.InputStreamMap; import org.jclouds.blobstore.attr.ConsistencyModel; import org.jclouds.blobstore.attr.ConsistencyModels; -import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.rest.RestContext; /** @@ -67,18 +67,20 @@ public class BlobStoreContextImpl implements BlobStoreContext { this.blobStore = checkNotNull(blobStore, "blobStore"); } - public BlobMap createBlobMap(String path) { - checkNotNull(path, "path"); - String container = BlobStoreUtilsImpl.parseContainerFromPath(path); - String dir = BlobStoreUtilsImpl.parsePrefixFromPath(path); - return blobMapFactory.create(container, dir); + public BlobMap createBlobMap(String container, ListContainerOptions options) { + return blobMapFactory.create(container, options); } - public InputStreamMap createInputStreamMap(String path) { - checkNotNull(path, "path"); - String container = BlobStoreUtilsImpl.parseContainerFromPath(path); - String dir = BlobStoreUtilsImpl.parsePrefixFromPath(path); - return inputStreamMapFactory.create(container, dir); + public BlobMap createBlobMap(String container) { + return blobMapFactory.create(container, ListContainerOptions.NONE); + } + + public InputStreamMap createInputStreamMap(String container, ListContainerOptions options) { + return inputStreamMapFactory.create(container, options); + } + + public InputStreamMap createInputStreamMap(String container) { + return inputStreamMapFactory.create(container, ListContainerOptions.NONE); } public BlobStore getBlobStore() { diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/InputStreamMapImpl.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/InputStreamMapImpl.java index bd344ea5f7..a4437deabb 100755 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/InputStreamMapImpl.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/InputStreamMapImpl.java @@ -23,17 +23,17 @@ import java.io.InputStream; import java.util.Collection; import java.util.Map; -import javax.annotation.Nullable; import javax.inject.Inject; import org.jclouds.blobstore.BlobMap; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.InputStreamMap; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.ContainsValueInListStrategy; import org.jclouds.blobstore.strategy.GetBlobsInListStrategy; import org.jclouds.blobstore.strategy.PutBlobsStrategy; -import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer; +import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders; import org.jclouds.http.Payload; import org.jclouds.http.Payloads; import org.jclouds.http.payloads.ByteArrayPayload; @@ -60,11 +60,11 @@ public class InputStreamMapImpl extends BaseBlobMap implements Inpu @Inject public InputStreamMapImpl(BlobStore connection, Blob.Factory blobFactory, - GetBlobsInListStrategy getAllBlobs, ListBlobMetadataInContainer listStrategy, + GetBlobsInListStrategy getAllBlobs, ListContainerAndRecurseThroughFolders listStrategy, ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy, - String containerName, @Nullable String dir) { + String containerName, ListContainerOptions options) { super(connection, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy, - containerName, dir); + containerName, options); } @Override diff --git a/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java b/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java index 0f9c6f95c1..b0333005ed 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java @@ -99,6 +99,11 @@ public class ListContainerOptions extends ListOptions implements Cloneable { return delegate.clone(); } + @Override + public String toString() { + return delegate.toString(); + } + } public static final ImmutableListContainerOptions NONE = new ImmutableListContainerOptions( diff --git a/blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java b/blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java index 41240f4694..8730ad41e6 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java @@ -68,6 +68,11 @@ public class ListOptions implements Cloneable { public ListOptions clone() { return delegate.clone(); } + + @Override + public String toString() { + return delegate.toString(); + } } public static final ImmutableListOptions NONE = new ImmutableListOptions(new ListOptions()); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobMetadataStrategy.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java similarity index 85% rename from blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobMetadataStrategy.java rename to blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java index 2fa84b0745..34fcd3093e 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobMetadataStrategy.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java @@ -20,7 +20,7 @@ package org.jclouds.blobstore.strategy; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer; +import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders; import com.google.inject.ImplementedBy; @@ -29,8 +29,8 @@ import com.google.inject.ImplementedBy; * * @author Adrian Cole */ -@ImplementedBy(ListBlobMetadataInContainer.class) -public interface ListBlobMetadataStrategy { +@ImplementedBy(ListContainerAndRecurseThroughFolders.class) +public interface ListBlobsInContainer { Iterable execute(String containerName, ListContainerOptions options); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListMetadataStrategy.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListContainerStrategy.java similarity index 87% rename from blobstore/src/main/java/org/jclouds/blobstore/strategy/ListMetadataStrategy.java rename to blobstore/src/main/java/org/jclouds/blobstore/strategy/ListContainerStrategy.java index 8a24baafe4..df82d02399 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListMetadataStrategy.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListContainerStrategy.java @@ -20,7 +20,7 @@ package org.jclouds.blobstore.strategy; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.blobstore.strategy.internal.ListAllMetadataInContainer; +import org.jclouds.blobstore.strategy.internal.ConcatenateContainerLists; import com.google.inject.ImplementedBy; @@ -29,8 +29,8 @@ import com.google.inject.ImplementedBy; * * @author Adrian Cole */ -@ImplementedBy(ListAllMetadataInContainer.class) -public interface ListMetadataStrategy { +@ImplementedBy(ConcatenateContainerLists.class) +public interface ListContainerStrategy { Iterable execute(String containerName, ListContainerOptions options); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListAllMetadataInContainer.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ConcatenateContainerLists.java similarity index 93% rename from blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListAllMetadataInContainer.java rename to blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ConcatenateContainerLists.java index ca0c3900fd..0ecac5dad1 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListAllMetadataInContainer.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ConcatenateContainerLists.java @@ -28,7 +28,7 @@ import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.options.ListContainerOptions.ImmutableListContainerOptions; -import org.jclouds.blobstore.strategy.ListMetadataStrategy; +import org.jclouds.blobstore.strategy.ListContainerStrategy; import com.google.common.base.Throwables; import com.google.common.collect.Iterables; @@ -41,12 +41,12 @@ import com.google.inject.Inject; * @author Adrian Cole */ @Singleton -public class ListAllMetadataInContainer implements ListMetadataStrategy { +public class ConcatenateContainerLists implements ListContainerStrategy { protected final BlobStore connection; @Inject - public ListAllMetadataInContainer(BlobStore connection) { + public ConcatenateContainerLists(BlobStore connection) { this.connection = connection; } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/CountBlobTypeInList.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/CountBlobTypeInList.java index aa5df633be..8a21ebdded 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/CountBlobTypeInList.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/CountBlobTypeInList.java @@ -23,7 +23,7 @@ import javax.inject.Singleton; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.CountListStrategy; -import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy; +import org.jclouds.blobstore.strategy.ListBlobsInContainer; import com.google.common.collect.Iterables; @@ -34,10 +34,10 @@ import com.google.common.collect.Iterables; */ @Singleton public class CountBlobTypeInList implements CountListStrategy { - protected final ListBlobMetadataStrategy getAllBlobMetadata; + protected final ListBlobsInContainer getAllBlobMetadata; @Inject - CountBlobTypeInList(ListBlobMetadataStrategy getAllBlobMetadata) { + CountBlobTypeInList(ListBlobsInContainer getAllBlobMetadata) { this.getAllBlobMetadata = getAllBlobMetadata; } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java index 557ef89f3f..0bcd726c13 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java @@ -36,6 +36,7 @@ import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.blobstore.strategy.ClearContainerStrategy; import org.jclouds.blobstore.strategy.ClearListStrategy; +import org.jclouds.blobstore.strategy.ListContainerStrategy; import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.logging.Logger; @@ -56,7 +57,7 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr @Named(BlobStoreConstants.BLOBSTORE_LOGGER) protected Logger logger = Logger.NULL; - protected final ListAllMetadataInContainer getAllMetadata; + protected final ListContainerStrategy listContainer; protected final BackoffLimitedRetryHandler retryHandler; private final ExecutorService userExecutor; @@ -70,12 +71,12 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr @Inject DeleteAllKeysInList(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, - AsyncBlobStore connection, ListAllMetadataInContainer getAllMetadata, + AsyncBlobStore connection, ListContainerStrategy listContainer, BackoffLimitedRetryHandler retryHandler) { this.userExecutor = userExecutor; this.connection = connection; - this.getAllMetadata = getAllMetadata; + this.listContainer = listContainer; this.retryHandler = retryHandler; } @@ -84,23 +85,34 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr } public void execute(final String containerName, final ListContainerOptions options) { - String message = options.getDir() != null ? String.format("deleting from path: %s/%s", - containerName, options.getDir()) : String.format("deleting from containerName: %s", + String message = options.getDir() != null ? String.format("clearing path %s/%s", + containerName, options.getDir()) : String.format("clearing container %s", containerName); + if (options.isRecursive()) + message = message + " recursively"; Map exceptions = Maps.newHashMap(); Iterable toDelete = getResourcesToDelete(containerName, options); for (int i = 0; i < 3; i++) { // TODO parameterize Map> responses = Maps.newHashMap(); try { - for (StorageMetadata md : toDelete) { + for (final StorageMetadata md : toDelete) { + String fullPath = parentIsFolder(options, md) ? options.getDir() + "/" + + md.getName() : md.getName(); switch (md.getType()) { case BLOB: - responses.put(md, connection.removeBlob(containerName, md.getName())); + responses.put(md, connection.removeBlob(containerName, fullPath)); break; case FOLDER: + if (options.isRecursive() && !fullPath.equals(options.getDir())) { + execute(containerName, options.clone().inDirectory(fullPath)); + } + connection.deleteDirectory(containerName, fullPath); + break; case RELATIVE_PATH: - if (options.isRecursive()) - responses.put(md, connection.deleteDirectory(containerName, md.getName())); + if (options.isRecursive() && !fullPath.equals(options.getDir())) { + execute(containerName, options.clone().inDirectory(fullPath)); + } + connection.deleteDirectory(containerName, md.getName()); break; case CONTAINER: throw new IllegalArgumentException("Container type not supported"); @@ -113,17 +125,24 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr break; } if (exceptions.size() > 0) { + toDelete = Iterables.concat(exceptions.keySet(), toDelete); retryHandler.imposeBackoffExponentialDelay(i + 1, message); } } } if (exceptions.size() > 0) throw new BlobRuntimeException(String.format("error %s: %s", message, exceptions)); + assert Iterables.size(toDelete) == 0 : String.format("items remaining %s: %s", message, + toDelete); + } + + private boolean parentIsFolder(final ListContainerOptions options, final StorageMetadata md) { + return (options.getDir() != null && md.getName().indexOf('/') == -1); } private Iterable getResourcesToDelete(final String containerName, final ListContainerOptions options) { - Iterable toDelete = Iterables.filter(getAllMetadata.execute( + Iterable toDelete = Iterables.filter(listContainer.execute( containerName, options), new Predicate() { @Override diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FindMD5InList.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FindMD5InList.java index 195da4075b..3529a203e4 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FindMD5InList.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FindMD5InList.java @@ -28,7 +28,7 @@ import org.jclouds.blobstore.functions.ObjectMD5; import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.ContainsValueInListStrategy; -import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy; +import org.jclouds.blobstore.strategy.ListBlobsInContainer; import com.google.common.base.Throwables; @@ -41,10 +41,10 @@ import com.google.common.base.Throwables; public class FindMD5InList implements ContainsValueInListStrategy { protected final ObjectMD5 objectMD5; - protected final ListBlobMetadataStrategy getAllBlobMetadata; + protected final ListBlobsInContainer getAllBlobMetadata; @Inject - private FindMD5InList(ObjectMD5 objectMD5, ListBlobMetadataStrategy getAllBlobMetadata) { + private FindMD5InList(ObjectMD5 objectMD5, ListBlobsInContainer getAllBlobMetadata) { this.objectMD5 = objectMD5; this.getAllBlobMetadata = getAllBlobMetadata; } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java index 83fad3c14d..0842f005de 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java @@ -37,7 +37,7 @@ import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.blobstore.strategy.GetBlobsInListStrategy; -import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy; +import org.jclouds.blobstore.strategy.ListBlobsInContainer; import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.logging.Logger; @@ -57,7 +57,7 @@ import com.google.inject.Inject; @Singleton public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrategy { - protected final ListBlobMetadataStrategy getAllBlobMetadata; + protected final ListBlobsInContainer getAllBlobMetadata; protected final BackoffLimitedRetryHandler retryHandler; protected final AsyncBlobStore ablobstore; protected final ExecutorService userExecutor; @@ -74,7 +74,7 @@ public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrateg @Inject GetAllBlobsInListAndRetryOnFailure( @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, - ListBlobMetadataStrategy getAllBlobMetadata, AsyncBlobStore ablobstore, + ListBlobsInContainer getAllBlobMetadata, AsyncBlobStore ablobstore, BackoffLimitedRetryHandler retryHandler) { this.userExecutor = userExecutor; this.ablobstore = ablobstore; diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListBlobMetadataInContainer.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListBlobMetadataInContainer.java deleted file mode 100644 index e996fcdeed..0000000000 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListBlobMetadataInContainer.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * - * Copyright (C) 2009 Cloud Conscious, LLC. - * - * ==================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ==================================================================== - */ -package org.jclouds.blobstore.strategy.internal; - -import javax.inject.Singleton; - -import org.jclouds.blobstore.domain.BlobMetadata; -import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.domain.StorageType; -import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy; -import org.jclouds.blobstore.strategy.ListMetadataStrategy; - -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import com.google.inject.Inject; - -/** - * Retrieves all blobs in the blobstore by the most efficient means possible. - * - * @author Adrian Cole - */ -@Singleton -public class ListBlobMetadataInContainer implements ListBlobMetadataStrategy { - - protected final ListMetadataStrategy lister; - - @Inject - ListBlobMetadataInContainer(ListMetadataStrategy lister) { - this.lister = lister; - } - - @Override - public Iterable execute(String container, ListContainerOptions options) { - return Iterables.transform(Iterables.filter(lister.execute(container, options), - new Predicate() { - @Override - public boolean apply(StorageMetadata input) { - return input.getType() == StorageType.BLOB; - } - - }), new Function() { - @Override - public BlobMetadata apply(StorageMetadata from) { - return (BlobMetadata) from; - } - }); - } -} \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java new file mode 100644 index 0000000000..6d3eef610f --- /dev/null +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java @@ -0,0 +1,83 @@ +/** + * + * Copyright (C) 2009 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ +package org.jclouds.blobstore.strategy.internal; + +import java.util.List; + +import javax.inject.Singleton; + +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.StorageType; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.jclouds.blobstore.strategy.ListBlobsInContainer; +import org.jclouds.blobstore.strategy.ListContainerStrategy; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.inject.Inject; + +/** + * Retrieves all blobs in the blobstore by the most efficient means possible. + * + * @author Adrian Cole + */ +@Singleton +public class ListContainerAndRecurseThroughFolders implements ListBlobsInContainer { + + protected final ListContainerStrategy lister; + + @Inject + ListContainerAndRecurseThroughFolders(ListContainerStrategy lister) { + this.lister = lister; + } + + @Override + public Iterable execute(final String containerName, + final ListContainerOptions options) { + final List> lists = Lists.newArrayList(); + Iterable pwdList = lister.execute(containerName, options); + for (StorageMetadata md : Iterables.filter(pwdList, new Predicate() { + @Override + public boolean apply(StorageMetadata input) { + return (input.getType() == StorageType.FOLDER || input.getType() == StorageType.RELATIVE_PATH) + && options.isRecursive(); + } + })) { + String directory = (options.getDir() != null) ? options.getDir() + "/" + md.getName() : md + .getName(); + lists.add(execute(containerName, options.clone().inDirectory(directory))); + } + lists.add(Iterables.transform(Iterables.filter(pwdList, new Predicate() { + @Override + public boolean apply(StorageMetadata input) { + return input.getType() == StorageType.BLOB; + } + }), new Function() { + @Override + public BlobMetadata apply(StorageMetadata from) { + return (BlobMetadata) from; + } + })); + return Sets.newHashSet(Iterables.concat(lists)); + } +} \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java index 1c0ffd0ed0..d74cba5e03 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java @@ -30,6 +30,7 @@ import javax.inject.Singleton; import org.jclouds.Constants; import org.jclouds.blobstore.AsyncBlobStore; +import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.blobstore.strategy.DeleteDirectoryStrategy; @@ -64,6 +65,7 @@ import com.google.inject.Inject; public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy { private final AsyncBlobStore ablobstore; + private final BlobStore blobstore; private final ExecutorService userExecutor; @Resource @Named(BlobStoreConstants.BLOBSTORE_LOGGER) @@ -78,9 +80,10 @@ public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy { @Inject MarkersDeleteDirectoryStrategy( @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, - AsyncBlobStore ablobstore) { + AsyncBlobStore ablobstore, BlobStore blobstore) { this.userExecutor = userExecutor; this.ablobstore = ablobstore; + this.blobstore = blobstore; } public void execute(String containerName, String directory) { @@ -93,10 +96,13 @@ public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy { for (String name : names) { responses.put(name, ablobstore.removeBlob(containerName, name)); } + String message = String.format("deleting directory %s in containerName: %s", directory, + containerName); Map exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, - String.format("deleting directory in containerName: %s", containerName)); + message); if (exceptions.size() > 0) - throw new BlobRuntimeException(String.format("error deleting from container %s: %s", - containerName, exceptions)); + throw new BlobRuntimeException(String.format("error %s: %s", message, exceptions)); + assert !blobstore.directoryExists(containerName, directory) : String.format( + "still exists %s: %s", message, exceptions); } } \ No newline at end of file diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/StubBlobMapIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/StubBlobMapIntegrationTest.java index 0439ad7984..679f5c13bb 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/StubBlobMapIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/StubBlobMapIntegrationTest.java @@ -28,9 +28,4 @@ import org.testng.annotations.Test; @Test(groups = { "integration", "live" }, testName = "blobstore.StubBlobMapIntegrationTest") public class StubBlobMapIntegrationTest extends BaseBlobMapIntegrationTest { - @Override - protected int maxList() { - return 1; - } - } \ No newline at end of file diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/StubInputStreamMapIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/StubInputStreamMapIntegrationTest.java index c29e7f7e7d..417a804ae3 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/StubInputStreamMapIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/StubInputStreamMapIntegrationTest.java @@ -27,9 +27,4 @@ import org.testng.annotations.Test; @Test(groups = { "integration", "live" }, testName = "blobstore.StubInputStreamMapIntegrationTest") public class StubInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest { - @Override - protected int maxList() { - return 1; - } - } \ No newline at end of file diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobMapIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobMapIntegrationTest.java index fc3f9f0908..e15f5146ff 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobMapIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobMapIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.jclouds.blobstore.integration.internal; +import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults; import static org.testng.Assert.assertEquals; import java.io.IOException; @@ -30,8 +31,10 @@ import java.util.Map.Entry; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import org.jclouds.blobstore.BlobMap; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl; import org.jclouds.util.Utils; import org.testng.annotations.Test; @@ -204,18 +207,16 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest< } } - protected abstract int maxList(); - - @Test(enabled = false, groups = { "integration", "live" }) + @Test(groups = { "integration", "live" }) public void testPutMoreThanSingleListing() throws InterruptedException, ExecutionException, TimeoutException { - if (maxList() == 0) + if (maxResultsForTestListings() == 0) return; String bucketName = getContainerName(); try { Map map = createMap(context, bucketName); Set keySet = Sets.newHashSet(); - for (int i = 0; i < maxList() + 1; i++) { + for (int i = 0; i < maxResultsForTestListings() + 1; i++) { keySet.add(i + ""); } @@ -228,7 +229,7 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest< map.putAll(newMap); newMap.clear(); - assertConsistencyAwareMapSize(map, maxList() + 1); + assertConsistencyAwareMapSize(map, maxResultsForTestListings() + 1); assertConsistencyAwareKeySetEquals(map, keySet); map.clear(); assertConsistencyAwareMapSize(map, 0); @@ -265,8 +266,15 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest< map.putAll(newMap); } - protected Map createMap(BlobStoreContext context, String bucket) { - return context.createBlobMap(bucket); + protected int maxResultsForTestListings() { + return 100; } + protected BlobMap createMap(BlobStoreContext context, String bucket) { + return createMap(context, bucket, maxResults(maxResultsForTestListings())); + } + + protected BlobMap createMap(BlobStoreContext context, String bucket, ListContainerOptions options) { + return context.createBlobMap(bucket, options); + } } diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java index 28313a7ebb..9280114138 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java @@ -207,8 +207,8 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest { addAlphabetUnderRoot(containerName); PageSet container = context.getBlobStore().list(containerName, maxResults(5)); - assert container.getNextMarker() != null; assertEquals(container.size(), 5); + assert container.getNextMarker() != null; } finally { returnContainer(containerName); } diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseInputStreamMapIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseInputStreamMapIntegrationTest.java index f5794de41c..ffde91f01f 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseInputStreamMapIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseInputStreamMapIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.jclouds.blobstore.integration.internal; +import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults; import static org.testng.Assert.assertEquals; import java.io.IOException; @@ -34,6 +35,7 @@ import java.util.concurrent.TimeoutException; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.InputStreamMap; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.util.Utils; import org.testng.annotations.Test; @@ -49,9 +51,9 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati @Override @Test(groups = { "integration", "live" }) public void testValues() throws InterruptedException, IOException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); map.putAll(this.fiveInputs); // this will cause us to block until the bucket updates. assertConsistencyAwareMapSize(map, 5); @@ -65,20 +67,18 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati assert valuesAsString.size() == 0 : valuesAsString.size() + ": " + values + ": " + valuesAsString; } finally { - returnContainer(bucketName); + returnContainer(containerName); } } - protected abstract int maxList(); - - @Test(enabled = false, groups = { "integration", "live" }) + @Test(groups = { "integration", "live" }) public void testPutMoreThanSingleListing() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - InputStreamMap map = createMap(context, bucketName); + InputStreamMap map = createMap(context, containerName); Set keySet = Sets.newHashSet(); - for (int i = 0; i < maxList() + 1; i++) { + for (int i = 0; i < maxResultsForTestListings() + 1; i++) { keySet.add(i + ""); } @@ -89,20 +89,20 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati map.putAllStrings(newMap); newMap.clear(); - assertConsistencyAwareMapSize(map, maxList() + 1); + assertConsistencyAwareMapSize(map, maxResultsForTestListings() + 1); assertConsistencyAwareKeySetEquals(map, keySet); map.clear(); assertConsistencyAwareMapSize(map, 0); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testRemove() throws InterruptedException, IOException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); putStringWithMD5(map, "one", "two"); InputStream old = map.remove("one"); assertEquals(Utils.toStringAndClose(old), "two"); @@ -113,16 +113,16 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati assert old == null; assertConsistencyAwareKeySize(map, 0); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Override @Test(groups = { "integration", "live" }) public void testEntrySet() throws InterruptedException, IOException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); ((InputStreamMap) map).putAllStrings(this.fiveStrings); // this will cause us to block until the bucket updates. assertConsistencyAwareKeySize(map, 5); @@ -137,134 +137,134 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati assertEquals(Utils.toStringAndClose(value), ""); } } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testContainsStringValue() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); ((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple")); assertConsistencyAwareContainsValue(map, fiveStrings.get("one")); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testContainsFileValue() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); ((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple")); assertConsistencyAwareContainsValue(map, fiveFiles.get("one")); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testContainsInputStreamValue() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); ((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple")); assertConsistencyAwareContainsValue(map, this.fiveInputs.get("one")); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testContainsBytesValue() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); ((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple")); assertConsistencyAwareContainsValue(map, this.fiveBytes.get("one")); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Override @Test(groups = { "integration", "live" }) public void testPutAll() throws InterruptedException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); map.putAll(this.fiveInputs); assertConsistencyAwareMapSize(map, 5); assertConsistencyAwareKeySetEquals(map, new TreeSet(fiveInputs.keySet())); fourLeftRemovingOne(map); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testPutAllBytes() throws InterruptedException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); ((InputStreamMap) map).putAllBytes(this.fiveBytes); assertConsistencyAwareMapSize(map, 5); assertConsistencyAwareKeySetEquals(map, new TreeSet(fiveBytes.keySet())); fourLeftRemovingOne(map); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testPutAllFiles() throws InterruptedException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); ((InputStreamMap) map).putAllFiles(this.fiveFiles); assertConsistencyAwareMapSize(map, 5); assertConsistencyAwareKeySetEquals(map, new TreeSet(fiveFiles.keySet())); fourLeftRemovingOne(map); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testPutAllStrings() throws InterruptedException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); ((InputStreamMap) map).putAllStrings(this.fiveStrings); assertConsistencyAwareMapSize(map, 5); assertConsistencyAwareKeySetEquals(map, new TreeSet(fiveStrings.keySet())); fourLeftRemovingOne(map); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testPutString() throws InterruptedException, IOException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); InputStream old = ((InputStreamMap) map).putString("one", fiveStrings.get("one")); getOneReturnsAppleAndOldValueIsNull(map, old); InputStream apple = ((InputStreamMap) map).putString("one", fiveStrings.get("two")); getOneReturnsBearAndOldValueIsApple(map, apple); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @@ -285,46 +285,46 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati @Test(groups = { "integration", "live" }) public void testPutFile() throws IOException, InterruptedException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); InputStream old = ((InputStreamMap) map).putFile("one", fiveFiles.get("one")); getOneReturnsAppleAndOldValueIsNull(map, old); InputStream apple = ((InputStreamMap) map).putFile("one", fiveFiles.get("two")); getOneReturnsBearAndOldValueIsApple(map, apple); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testPutBytes() throws InterruptedException, IOException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); InputStream old = ((InputStreamMap) map).putBytes("one", fiveBytes.get("one")); getOneReturnsAppleAndOldValueIsNull(map, old); InputStream apple = ((InputStreamMap) map).putBytes("one", fiveBytes.get("two")); getOneReturnsBearAndOldValueIsApple(map, apple); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @Test(groups = { "integration", "live" }) public void testPut() throws InterruptedException, IOException { - String bucketName = getContainerName(); + String containerName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerName); InputStream old = map.put("one", fiveInputs.get("one")); getOneReturnsAppleAndOldValueIsNull(map, old); InputStream apple = map.put("one", fiveInputs.get("two")); getOneReturnsBearAndOldValueIsApple(map, apple); } finally { - returnContainer(bucketName); + returnContainer(containerName); } } @@ -334,8 +334,16 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati ((InputStreamMap) map).putString(key, value); } + protected int maxResultsForTestListings() { + return 100; + } + protected InputStreamMap createMap(BlobStoreContext context, String bucket) { - InputStreamMap map = context.createInputStreamMap(bucket); - return map; + return createMap(context, bucket, maxResults(maxResultsForTestListings())); + } + + protected InputStreamMap createMap(BlobStoreContext context, String bucket, + ListContainerOptions options) { + return context.createInputStreamMap(bucket, options); } } diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java index 843fc5d7b1..6a0d5342b3 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java @@ -18,6 +18,8 @@ */ package org.jclouds.blobstore.integration.internal; +import static org.jclouds.blobstore.options.ListContainerOptions.Builder.inDirectory; +import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -25,6 +27,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -34,6 +37,8 @@ import java.util.concurrent.TimeoutException; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.ListableMap; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.util.Utils; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -97,20 +102,23 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration } } - protected abstract Map createMap(BlobStoreContext context, String bucket); + protected abstract Map createMap(BlobStoreContext context, String containerName); + + protected abstract Map createMap(BlobStoreContext context, String containerName, + ListContainerOptions options); @Test(groups = { "integration", "live" }) public void testClear() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerNameName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerNameName); assertConsistencyAwareMapSize(map, 0); putStringWithMD5(map, "one", "apple"); assertConsistencyAwareMapSize(map, 1); map.clear(); assertConsistencyAwareMapSize(map, 0); } finally { - returnContainer(bucketName); + returnContainer(containerNameName); } } @@ -120,18 +128,84 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration @Test(groups = { "integration", "live" }) public void testKeySet() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerNameName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerNameName); assertConsistencyAwareKeySize(map, 0); putStringWithMD5(map, "one", "two"); assertConsistencyAwareKeySize(map, 1); assertConsistencyAwareKeySetEquals(map, ImmutableSet.of("one")); } finally { - returnContainer(bucketName); + returnContainer(containerNameName); } } + protected void addTenObjectsUnderPrefix(String containerName, String prefix) + throws InterruptedException { + for (int i = 0; i < 10; i++) { + Blob blob = context.getBlobStore().newBlob(prefix + "/" + i); + blob.setPayload(i + "content"); + context.getBlobStore().putBlob(containerName, blob); + } + } + + protected void addTenObjectsUnderRoot(String containerName) throws InterruptedException { + for (int i = 0; i < 10; i++) { + Blob blob = context.getBlobStore().newBlob(i + ""); + blob.setPayload(i + "content"); + context.getBlobStore().putBlob(containerName, blob); + } + } + + @Test(groups = { "integration", "live" }) + public void testDirectory() throws InterruptedException, UnsupportedEncodingException { + String containerName = getContainerName(); + try { + String directory = "apps"; + + Map rootMap = createMap(context, containerName); + Map rootRecursiveMap = createMap(context, containerName, recursive()); + Map inDirectoryMap = createMap(context, containerName, inDirectory(directory)); + Map inDirectoryRecursiveMap = createMap(context, containerName, inDirectory( + directory).recursive()); + + context.getBlobStore().createDirectory(containerName, directory); + addTenObjectsUnderRoot(containerName); + assertEquals(rootMap.size(), 10); + assertEquals(rootRecursiveMap.size(), 10); + assertEquals(inDirectoryMap.size(), 0); + assertEquals(inDirectoryRecursiveMap.size(), 0); + + addTenObjectsUnderPrefix(containerName, directory); + assertEquals(rootMap.size(), 10); + assertEquals(rootRecursiveMap.size(), 20); + assertEquals(inDirectoryMap.size(), 10); + assertEquals(inDirectoryRecursiveMap.size(), 10); + + context.getBlobStore().createDirectory(containerName, directory + "/" + directory); + assertEquals(rootMap.size(), 10); + assertEquals(rootRecursiveMap.size(), 20); + assertEquals(inDirectoryMap.size(), 10); + assertEquals(inDirectoryRecursiveMap.size(), 10); + + rootMap.clear(); + assertEquals(rootMap.size(), 0); + assertEquals(rootRecursiveMap.size(), 10); + assertEquals(inDirectoryMap.size(), 10); + assertEquals(inDirectoryRecursiveMap.size(), 10); + + inDirectoryMap.clear(); + assertEquals(rootMap.size(), 0); + assertEquals(rootRecursiveMap.size(), 0); + assertEquals(inDirectoryMap.size(), 0); + assertEquals(inDirectoryRecursiveMap.size(), 0); + + } finally { + returnContainer(containerName); + } + + } + protected void assertConsistencyAwareKeySetEquals(final Map map, final Set expected) throws InterruptedException { assertConsistencyAware(new Runnable() { @@ -177,20 +251,20 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration @Test(groups = { "integration", "live" }) public void testContainsKey() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerNameName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerNameName); assertConsistencyAwareDoesntContainKey(map); putStringWithMD5(map, "one", "apple"); assertConsistencyAwareContainsKey(map); } finally { - returnContainer(bucketName); + returnContainer(containerNameName); } } /** - * containsValue() uses eTag comparison to bucket contents, so this can be subject to eventual - * consistency problems. + * containsValue() uses eTag comparison to containerName contents, so this can be subject to + * eventual consistency problems. */ protected void assertConsistencyAwareContainsValue(final Map map, final Object value) throws InterruptedException { @@ -221,14 +295,14 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration @Test(groups = { "integration", "live" }) public void testIsEmpty() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerNameName = getContainerName(); try { - Map map = createMap(context, bucketName); + Map map = createMap(context, containerNameName); assertConsistencyAwareEmpty(map); putStringWithMD5(map, "one", "apple"); assertConsistencyAwareNotEmpty(map); } finally { - returnContainer(bucketName); + returnContainer(containerNameName); } } @@ -275,17 +349,17 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration @Test(groups = { "integration", "live" }) public void testListContainer() throws InterruptedException, ExecutionException, TimeoutException { - String bucketName = getContainerName(); + String containerNameName = getContainerName(); try { - ListableMap map = (ListableMap) createMap(context, bucketName); - assertConsistencyAwareListContainer(map, bucketName); + ListableMap map = (ListableMap) createMap(context, containerNameName); + assertConsistencyAwareListContainer(map, containerNameName); } finally { - returnContainer(bucketName); + returnContainer(containerNameName); } } protected void assertConsistencyAwareListContainer(final ListableMap map, - final String bucketName) throws InterruptedException { + final String containerNameName) throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assertTrue(Iterables.size(map.list()) >= 0); diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/StubAsyncBlobStore.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/StubAsyncBlobStore.java index 229f6ce3b5..76f37af715 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/StubAsyncBlobStore.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/StubAsyncBlobStore.java @@ -19,6 +19,7 @@ package org.jclouds.blobstore.integration.internal; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; @@ -157,7 +158,12 @@ public class StubAsyncBlobStore extends BaseAsyncBlobStore { SortedSet contents = Sets.newTreeSet(Iterables.transform(realContents .keySet(), new Function() { public StorageMetadata apply(String key) { - MutableBlobMetadata md = copy(realContents.get(key).getMetadata()); + Blob oldBlob = realContents.get(key); + checkState(oldBlob != null, "blob " + key + + " is not present although it was in the list of " + name); + checkState(oldBlob.getMetadata() != null, "blob " + name + "/" + key + + " has no metadata"); + MutableBlobMetadata md = copy(oldBlob.getMetadata()); String directoryName = ifDirectoryReturnName.execute(md); if (directoryName != null) { md.setName(directoryName); @@ -242,7 +248,9 @@ public class StubAsyncBlobStore extends BaseAsyncBlobStore { convertUserMetadataKeysToLowercase(metadata); return metadata; } catch (Exception e) { - throw new RuntimeException(e); + Throwables.propagate(e); + assert false : "exception should have propagated: " + e; + return null; } } diff --git a/core/src/main/java/org/jclouds/http/HttpRequest.java b/core/src/main/java/org/jclouds/http/HttpRequest.java index 8b4be5badd..b0f557888d 100644 --- a/core/src/main/java/org/jclouds/http/HttpRequest.java +++ b/core/src/main/java/org/jclouds/http/HttpRequest.java @@ -103,9 +103,6 @@ public class HttpRequest extends HttpMessage { return payload; } - /** - * {@inheritDoc} - */ public void setPayload(Payload data) { closeContentIfPresent(); this.payload = checkNotNull(data, "data"); diff --git a/core/src/main/java/org/jclouds/rest/functions/ReturnFalseOnResourceNotFound.java b/core/src/main/java/org/jclouds/rest/functions/ReturnFalseOnResourceNotFound.java new file mode 100644 index 0000000000..08d209cec0 --- /dev/null +++ b/core/src/main/java/org/jclouds/rest/functions/ReturnFalseOnResourceNotFound.java @@ -0,0 +1,47 @@ +/** + * + * Copyright (C) 2009 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ +package org.jclouds.rest.functions; + +import static org.jclouds.util.Utils.propagateOrNull; + +import java.util.List; + +import javax.inject.Singleton; + +import org.jclouds.rest.ResourceNotFoundException; + +import com.google.common.base.Function; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class ReturnFalseOnResourceNotFound implements Function { + + public Boolean apply(Exception from) { + List throwables = Throwables.getCausalChain(from); + if (Iterables.size(Iterables.filter(throwables, ResourceNotFoundException.class)) >= 1) { + return false; + } + return Boolean.class.cast(propagateOrNull(from)); + } +} \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/rest/functions/ReturnNullOnResourceNotFound.java b/core/src/main/java/org/jclouds/rest/functions/ReturnNullOnResourceNotFound.java new file mode 100644 index 0000000000..5300f82179 --- /dev/null +++ b/core/src/main/java/org/jclouds/rest/functions/ReturnNullOnResourceNotFound.java @@ -0,0 +1,42 @@ +/** + * + * Copyright (C) 2009 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ +package org.jclouds.rest.functions; + +import static org.jclouds.util.Utils.propagateOrNull; + +import javax.inject.Singleton; + +import org.jclouds.rest.ResourceNotFoundException; + +import com.google.common.base.Function; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class ReturnNullOnResourceNotFound implements Function { + + public Object apply(Exception from) { + if (from instanceof ResourceNotFoundException) { + return null; + } + return propagateOrNull(from); + } +} \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/util/Jsr330.java b/core/src/main/java/org/jclouds/util/Jsr330.java index b2cb8e18b0..ea744f0765 100644 --- a/core/src/main/java/org/jclouds/util/Jsr330.java +++ b/core/src/main/java/org/jclouds/util/Jsr330.java @@ -31,10 +31,9 @@ import com.google.inject.Key; * Utility methods for use with {@code @}{@link Named}. * * @author crazybob@google.com (Bob Lee) - original code taken from - * {@link com.google.inject.name.Names} + * {@code com.google.inject.name.Names} * * @see com.google.inject.util.Jsr330#named - * @see com.google.inject.name.Names * @author Adrian Cole */ public class Jsr330 { diff --git a/core/src/main/java/org/jclouds/util/Utils.java b/core/src/main/java/org/jclouds/util/Utils.java index 5c5cef7648..26c514d938 100644 --- a/core/src/main/java/org/jclouds/util/Utils.java +++ b/core/src/main/java/org/jclouds/util/Utils.java @@ -85,12 +85,8 @@ public class Utils { } /** - * Returns a factory that will supply instances of {@link OutputStream} that read from the given - * outputStream. + * converts an {@link OutputStream} to an {@link OutputSupplier} * - * @param url - * the URL to read from - * @return the factory */ public static OutputSupplier newOutputStreamSupplier(final OutputStream output) { checkNotNull(output, "output"); @@ -146,9 +142,6 @@ public class Utils { * {@link UnsupportedEncodingException}, log a warning and fall back to the system's default * encoding. * - * @see {@link String#getBytes(String)} - * @see {@link String#getBytes()} - used as fall-back. - * * @param str * what to encode * @param charsetName diff --git a/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/ContainerToResourceList.java b/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/ContainerToResourceList.java index 3d7ebe79be..fcf2b77b3e 100644 --- a/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/ContainerToResourceList.java +++ b/rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/ContainerToResourceList.java @@ -24,7 +24,9 @@ import javax.inject.Singleton; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.blobstore.domain.internal.StorageMetadataImpl; import org.jclouds.rackspace.cloudfiles.domain.ObjectInfo; import com.google.common.base.Function; @@ -46,8 +48,13 @@ public class ContainerToResourceList implements public PageSet apply(PageSet from) { return new PageSetImpl(Iterables.transform(Iterables.transform(from, object2blobMd), new Function() { - public StorageMetadata apply(BlobMetadata arg0) { - return arg0; + public StorageMetadata apply(BlobMetadata input) { + if (input.getContentType().equals("application/directory")) { + return new StorageMetadataImpl(StorageType.RELATIVE_PATH, input.getId(), input + .getName(), input.getLocationId(), input.getUri(), input.getETag(), input + .getSize(), input.getLastModified(), input.getUserMetadata()); + } + return input; } }), from.getNextMarker()); diff --git a/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationTest.java b/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationTest.java index 1c5f851a6c..eb3336e8fb 100755 --- a/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationTest.java +++ b/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationTest.java @@ -18,10 +18,7 @@ */ package org.jclouds.rackspace.cloudfiles.blobstore.integration; -import java.io.IOException; - import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; /** @@ -38,17 +35,4 @@ public class CloudFilesBlobIntegrationTest extends BaseBlobIntegrationTest { // not supported in cloud files } - // TODO this should work, not sure why my encoding is set to Mac Roman and not UTF-8 - @DataProvider(name = "delete") - public Object[][] createData() { - return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, - { "path/foo" }, { "colon:" }, { "asteri*k" }, { "quote\"" }, { "p|pe" } }; - } - - @Override - @Test(enabled = false) - public void testPutObject(String key, String type, Object content, Object realObject) - throws InterruptedException, IOException { - // TODO relative path keeps showing up - } } \ No newline at end of file diff --git a/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobMapIntegrationTest.java b/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobMapIntegrationTest.java index 1907ed954c..ee9ae3bea4 100644 --- a/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobMapIntegrationTest.java +++ b/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobMapIntegrationTest.java @@ -26,8 +26,5 @@ import org.testng.annotations.Test; */ @Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesBlobMapIntegrationTest") public class CloudFilesBlobMapIntegrationTest extends BaseBlobMapIntegrationTest { - @Override - protected int maxList() { - return 10000; - } + } \ No newline at end of file diff --git a/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesContainerIntegrationTest.java b/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesContainerIntegrationTest.java index 0fb3302642..ccf3855159 100755 --- a/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesContainerIntegrationTest.java +++ b/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesContainerIntegrationTest.java @@ -18,8 +18,6 @@ */ package org.jclouds.rackspace.cloudfiles.blobstore.integration; -import java.io.UnsupportedEncodingException; - import org.jclouds.blobstore.integration.internal.BaseContainerIntegrationTest; import org.testng.annotations.Test; @@ -30,18 +28,4 @@ import org.testng.annotations.Test; @Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesContainerIntegrationTest") public class CloudFilesContainerIntegrationTest extends BaseContainerIntegrationTest { - @Override - @Test(enabled = false) - public void testListRootUsesDelimiter() throws InterruptedException, - UnsupportedEncodingException { - // TODO occasionally fails due to virtual directories not deleting from the prior run - } - - - @Override - @Test(enabled = false) - public void testListContainerMarker() throws InterruptedException, - UnsupportedEncodingException { - // TODO occasionally fails due to virtual directories not deleting from the prior run - } } \ No newline at end of file diff --git a/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesInputStreamMapIntegrationTest.java b/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesInputStreamMapIntegrationTest.java index 7655618ef0..59d5d19951 100644 --- a/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesInputStreamMapIntegrationTest.java +++ b/rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesInputStreamMapIntegrationTest.java @@ -26,8 +26,5 @@ import org.testng.annotations.Test; */ @Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesInputStreamMapIntegrationTest") public class CloudFilesInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest { - @Override - protected int maxList() { - return 10000; - } + } \ No newline at end of file diff --git a/tools/vfs/src/main/java/org/jclouds/vfs/provider/blobstore/BlobStoreFileObject.java b/tools/vfs/src/main/java/org/jclouds/vfs/provider/blobstore/BlobStoreFileObject.java index 5af489e591..a787ebb97e 100644 --- a/tools/vfs/src/main/java/org/jclouds/vfs/provider/blobstore/BlobStoreFileObject.java +++ b/tools/vfs/src/main/java/org/jclouds/vfs/provider/blobstore/BlobStoreFileObject.java @@ -56,7 +56,7 @@ import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.blobstore.strategy.internal.ListAllMetadataInContainer; +import org.jclouds.blobstore.strategy.internal.ConcatenateContainerLists; import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl; import org.jclouds.util.Utils; @@ -70,7 +70,7 @@ import com.google.common.collect.Sets; */ public class BlobStoreFileObject extends AbstractFileObject { private final BlobStoreContext context; - private final ListAllMetadataInContainer lister; + private final ConcatenateContainerLists lister; private final String container; private StorageMetadata metadata; private static final Logger logger = Logger.getLogger(BlobStoreFileObject.class); @@ -81,7 +81,7 @@ public class BlobStoreFileObject extends AbstractFileObject { super(fileName, fileSystem); this.context = checkNotNull(context, "context"); this.container = checkNotNull(container, "container"); - this.lister = checkNotNull(new ListAllMetadataInContainer(context.getBlobStore()), "lister"); + this.lister = checkNotNull(new ConcatenateContainerLists(context.getBlobStore()), "lister"); } diff --git a/vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/TerremarkVCloudClient.java b/vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/TerremarkVCloudClient.java index 8a9cfdaca9..eafd858e42 100644 --- a/vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/TerremarkVCloudClient.java +++ b/vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/TerremarkVCloudClient.java @@ -129,6 +129,7 @@ public interface TerremarkVCloudClient extends VCloudClient { void deleteNode(int nodeId); + @Timeout(duration = 180, timeUnit = TimeUnit.SECONDS) SortedSet getNodes(int internetServiceId); SortedSet getIpAddressesForNetwork(String networkId);