corrected CNFE, as it should have been a subclass of ResourceNotFoundException; stabalized the directory support for rackspace and atmos

git-svn-id: http://jclouds.googlecode.com/svn/trunk@2753 3d8758e0-26b5-11de-8745-db77d3ebf521

atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java

@@ -45,8 +45,6 @@ import org.jclouds.atmosonline.saas.functions.ReturnEndpointIfAlreadyExists;
 import org.jclouds.atmosonline.saas.options.ListOptions;
 import org.jclouds.blobstore.attr.ConsistencyModel;
 import org.jclouds.blobstore.attr.ConsistencyModels;
-import org.jclouds.blobstore.functions.ReturnFalseOnKeyNotFound;
-import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
 import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
 import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404;
 import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
@@ -59,6 +57,8 @@ import org.jclouds.rest.annotations.QueryParams;
 import org.jclouds.rest.annotations.RequestFilters;
 import org.jclouds.rest.annotations.ResponseParser;
 import org.jclouds.rest.annotations.SkipEncoding;
+import org.jclouds.rest.functions.ReturnFalseOnResourceNotFound;
+import org.jclouds.rest.functions.ReturnNullOnResourceNotFound;
 
 import com.google.common.util.concurrent.ListenableFuture;
 
@@ -75,7 +75,9 @@ import com.google.common.util.concurrent.ListenableFuture;
 @SkipEncoding( { '/' })
 @ConsistencyModel(ConsistencyModels.EVENTUAL)
 public interface AtmosStorageAsyncClient {
-
+   /**
+    * Creates a default implementation of AtmosObject
+    */
    AtmosObject newObject();
 
    /**
@@ -134,7 +136,7 @@ public interface AtmosStorageAsyncClient {
     */
    @GET
    @ResponseParser(ParseObjectFromHeadersAndHttpContent.class)
-   @ExceptionParser(ReturnNullOnKeyNotFound.class)
+   @ExceptionParser(ReturnNullOnResourceNotFound.class)
    @Path("/rest/namespace/{path}")
    @Consumes(MediaType.WILDCARD)
    ListenableFuture<AtmosObject> readFile(@PathParam("path") String path, GetOptions... options);
@@ -144,7 +146,7 @@ public interface AtmosStorageAsyncClient {
     */
    @HEAD
    @ResponseParser(ParseObjectFromHeadersAndHttpContent.class)
-   @ExceptionParser(ReturnNullOnKeyNotFound.class)
+   @ExceptionParser(ReturnNullOnResourceNotFound.class)
    @Path("/rest/namespace/{path}")
    @Consumes(MediaType.WILDCARD)
    ListenableFuture<AtmosObject> headFile(@PathParam("path") String path);
@@ -154,7 +156,7 @@ public interface AtmosStorageAsyncClient {
     */
    @HEAD
    @ResponseParser(ParseSystemMetadataFromHeaders.class)
-   @ExceptionParser(ReturnNullOnKeyNotFound.class)
+   @ExceptionParser(ReturnNullOnResourceNotFound.class)
    // currently throws 403 errors @QueryParams(keys = "metadata/system")
    @Path("/rest/namespace/{path}")
    @Consumes(MediaType.WILDCARD)
@@ -165,7 +167,7 @@ public interface AtmosStorageAsyncClient {
     */
    @HEAD
    @ResponseParser(ParseSystemMetadataFromHeaders.class)
-   @ExceptionParser(ReturnNullOnKeyNotFound.class)
+   @ExceptionParser(ReturnNullOnResourceNotFound.class)
    @Path("/rest/namespace/{path}")
    @QueryParams(keys = "metadata/user")
    @Consumes(MediaType.WILDCARD)
@@ -184,7 +186,7 @@ public interface AtmosStorageAsyncClient {
     * @see AtmosStorageClient#pathExists
     */
    @HEAD
-   @ExceptionParser(ReturnFalseOnKeyNotFound.class)
+   @ExceptionParser(ReturnFalseOnResourceNotFound.class)
    @Path("/rest/namespace/{path}")
    @Consumes(MediaType.WILDCARD)
    ListenableFuture<Boolean> pathExists(@PathParam("path") String path);

atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageClient.java

@@ -40,13 +40,14 @@ import org.jclouds.http.options.GetOptions;
  */
 @Timeout(duration = 120, timeUnit = TimeUnit.SECONDS)
 public interface AtmosStorageClient {
-
+   /**
+    * Creates a default implementation of AtmosObject
+    */
    AtmosObject newObject();
 
    BoundedSet<? extends DirectoryEntry> listDirectories(ListOptions... options);
 
-   BoundedSet<? extends DirectoryEntry> listDirectory(String directoryName,
+   BoundedSet<? extends DirectoryEntry> listDirectory(String directoryName, ListOptions... options);
-            ListOptions... options);
 
    URI createDirectory(String directoryName);
 
@@ -65,23 +66,8 @@ public interface AtmosStorageClient {
 
    UserMetadata getUserMetadata(String path);
 
-   /**
-    * 
-    * @param path
-    * @return
-    * @throws AtmosStorageResponseException
-    *            , if the path is a directory and not empty
-    */
    void deletePath(String path);
 
    boolean pathExists(String path);
 
-   // signature currently doesn't work
-   // @POST
-   // @QueryParams(keys = "acl")
-   // @Headers(keys = { "x-emc-useracl", "x-emc-groupacl" }, values = { "root=FULL_CONTROL",
-   // "other=READ" })
-   // @Consumes(MediaType.WILDCARD)
-   // void makePublic(@Endpoint URI url);
-
 }

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java

@@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.util.concurrent.Futures.compose;
 
 import java.net.URI;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
@@ -46,13 +47,11 @@ import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
 import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
 import org.jclouds.blobstore.util.BlobStoreUtils;
+import org.jclouds.concurrent.ConcurrentUtils;
 import org.jclouds.encryption.EncryptionService;
 import org.jclouds.http.options.GetOptions;
-import org.jclouds.util.Utils;
 
 import com.google.common.base.Function;
-import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
 import com.google.common.util.concurrent.ListenableFuture;
 
 /**
@@ -140,8 +139,8 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
     * {@link AtmosStorageAsyncClient#pathExists} until it is true.
     */
    protected boolean deleteAndVerifyContainerGone(final String container) {
-      sync.deletePath(container);
+      sync.deletePath(container + "/");
-      return !sync.pathExists(container);
+      return !sync.pathExists(container + "/");
    }
 
    /**
@@ -149,7 +148,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
     */
    @Override
    public ListenableFuture<Boolean> containerExists(String container) {
-      return async.pathExists(container);
+      return async.pathExists(container + "/");
    }
 
    /**
@@ -160,6 +159,14 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
       return async.pathExists(container + "/" + directory + "/");
    }
 
+   /**
+    * This implementation invokes {@link #removeBlob}
+    */
+   @Override
+   public ListenableFuture<Void> deleteDirectory(String containerName, String directory) {
+      return removeBlob(containerName, directory + "/");
+   }
+
    /**
     * This implementation invokes {@link AtmosStorageAsyncClient#pathExists}
     * 
@@ -210,31 +217,15 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
     */
    @Override
    public ListenableFuture<String> putBlob(final String container, final Blob blob) {
-      final String path = container + "/" + blob.getMetadata().getName();
+      return ConcurrentUtils.makeListenable(service.submit(new Callable<String>() {
-      return compose(async.deletePath(path), new Function<Void, String>() {
+
+         @Override
+         public String call() throws Exception {
+            return AtmosStorageUtils.putBlob(sync, encryptionService, blob2Object, container, blob);
 
-         public String apply(Void from) {
-            try {
-               if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
-                  public Boolean get() {
-                     return !sync.pathExists(path);
-                  }
-               }, 300)) {
-                  throw new IllegalStateException(path + " still exists after deleting!");
-               }
-               if (blob.getMetadata().getContentMD5() != null)
-                  blob.getMetadata().getUserMetadata().put("content-md5",
-                           encryptionService.toHexString(blob.getMetadata().getContentMD5()));
-               sync.createFile(container, blob2Object.apply(blob));
-               return path;
-            } catch (InterruptedException e) {
-               Throwables.propagate(e);
-            }
-            assert false : " should have propagated error";
-            return null;
          }
 
-      }, service);
+      }), service);
 
    }
 

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java

@@ -87,8 +87,8 @@ public class AtmosBlobStore extends BaseBlobStore {
     * {@link AtmosStorageAsyncClient#pathExists} until it is true.
     */
    protected boolean deleteAndVerifyContainerGone(final String container) {
-      sync.deletePath(container);
+      sync.deletePath(container + "/");
-      return !sync.pathExists(container);
+      return !sync.pathExists(container + "/");
    }
 
    /**
@@ -116,12 +116,20 @@ public class AtmosBlobStore extends BaseBlobStore {
       sync.createDirectory(container + "/" + directory);
    }
 
+   /**
+    * This implementation invokes {@link #removeBlob}
+    */
+   @Override
+   public void deleteDirectory(String containerName, String directory) {
+      removeBlob(containerName, directory + "/");
+   }
+
    /**
     * This implementation invokes {@link AtmosStorageClient#pathExists}
     */
    @Override
    public boolean containerExists(String container) {
-      return sync.pathExists(container);
+      return sync.pathExists(container + "/");
    }
 
    /**
@@ -181,13 +189,7 @@ public class AtmosBlobStore extends BaseBlobStore {
     */
    @Override
    public String putBlob(final String container, final Blob blob) {
-      final String path = container + "/" + blob.getMetadata().getName();
+      return AtmosStorageUtils.putBlob(sync, encryptionService, blob2Object, container, blob);
-      deleteAndEnsurePathGone(path);
-      if (blob.getMetadata().getContentMD5() != null)
-         blob.getMetadata().getUserMetadata().put("content-md5",
-                  encryptionService.toHexString(blob.getMetadata().getContentMD5()));
-      sync.createFile(container, blob2Object.apply(blob));
-      return path;
    }
 
    /**

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/config/AtmosBlobStoreContextModule.java

@@ -25,7 +25,6 @@ import org.jclouds.atmosonline.saas.AtmosStorageClient;
 import org.jclouds.atmosonline.saas.blobstore.AtmosAsyncBlobStore;
 import org.jclouds.atmosonline.saas.blobstore.AtmosBlobStore;
 import org.jclouds.atmosonline.saas.blobstore.strategy.FindMD5InUserMetadata;
-import org.jclouds.atmosonline.saas.blobstore.strategy.RecursiveRemove;
 import org.jclouds.atmosonline.saas.config.AtmosStorageContextModule;
 import org.jclouds.blobstore.AsyncBlobStore;
 import org.jclouds.blobstore.BlobMap;
@@ -34,8 +33,6 @@ import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.InputStreamMap;
 import org.jclouds.blobstore.config.BlobStoreMapModule;
 import org.jclouds.blobstore.internal.BlobStoreContextImpl;
-import org.jclouds.blobstore.strategy.ClearContainerStrategy;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
 import org.jclouds.lifecycle.Closer;
 import org.jclouds.rest.RestContext;
@@ -56,8 +53,6 @@ public class AtmosBlobStoreContextModule extends AtmosStorageContextModule {
       bind(AsyncBlobStore.class).to(AtmosAsyncBlobStore.class).asEagerSingleton();
       bind(BlobStore.class).to(AtmosBlobStore.class).asEagerSingleton();
       bind(ContainsValueInListStrategy.class).to(FindMD5InUserMetadata.class);
-      bind(ClearListStrategy.class).to(RecursiveRemove.class);
-      bind(ClearContainerStrategy.class).to(RecursiveRemove.class);
    }
 
    @Provides

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobToSystemMetadata.java

@@ -32,9 +32,9 @@ import com.google.common.base.Function;
 @Singleton
 public class BlobToSystemMetadata implements Function<BlobMetadata, SystemMetadata> {
    public SystemMetadata apply(BlobMetadata base) {
-      return new SystemMetadata(null, base.getLastModified(), null, null, null, 1, null, base
+      return new SystemMetadata(base.getContentMD5(), null, base.getLastModified(), null, null,
-               .getName(), null, (base.getSize() != null) ? base.getSize() : 0, FileType.REGULAR,
+               null, 1, null, base.getName(), null, (base.getSize() != null) ? base.getSize() : 0,
-               "root");
+               FileType.REGULAR, "root");
    }
 
 }

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlob.java

@@ -49,6 +49,7 @@ public class ObjectToBlob implements Function<AtmosObject, Blob> {
       Blob blob = blobFactory.create(object2BlobMd.apply(from));
       if (from.getContentMetadata().getContentLength() != null)
          blob.setContentLength(from.getContentMetadata().getContentLength());
+      blob.getMetadata().setContentMD5(from.getSystemMetadata().getContentMD5());
       blob.setPayload(checkNotNull(from.getPayload(), "payload: " + from));
       blob.setAllHeaders(from.getAllHeaders());
       return blob;

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java

@@ -31,7 +31,6 @@ import org.jclouds.atmosonline.saas.functions.AtmosObjectName;
 import org.jclouds.blobstore.domain.MutableBlobMetadata;
 import org.jclouds.blobstore.domain.StorageType;
 import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
-import org.jclouds.encryption.EncryptionService;
 
 import com.google.common.base.Function;
 import com.google.common.collect.ImmutableSet;
@@ -46,12 +45,10 @@ public class ObjectToBlobMetadata implements Function<AtmosObject, MutableBlobMe
    private static final Set<String> systemMetadata = ImmutableSet.of("atime", "mtime", "ctime",
             "itime", "type", "uid", "gid", "objectid", "objname", "size", "nlink", "policyname",
             "content-md5");
-   private final EncryptionService encryptionService;
 
    @Inject
-   protected ObjectToBlobMetadata(AtmosObjectName objectName, EncryptionService encryptionService) {
+   protected ObjectToBlobMetadata(AtmosObjectName objectName) {
       this.objectName = objectName;
-      this.encryptionService = encryptionService;
    }
 
    public MutableBlobMetadata apply(AtmosObject from) {
@@ -60,9 +57,7 @@ public class ObjectToBlobMetadata implements Function<AtmosObject, MutableBlobMe
       MutableBlobMetadata to = new MutableBlobMetadataImpl();
       to.setId(from.getSystemMetadata().getObjectID());
       to.setLastModified(from.getSystemMetadata().getLastUserDataModification());
-      String md5hex = from.getUserMetadata().getMetadata().get("content-md5");
+      to.setContentMD5(from.getSystemMetadata().getContentMD5());
-      if (md5hex != null)
-         to.setContentMD5(encryptionService.fromHexString(md5hex));
       if (from.getContentMetadata().getContentType() != null)
          to.setContentType(from.getContentMetadata().getContentType());
       to.setName(objectName.apply(from));

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java

@@ -18,6 +18,7 @@
  */
 package org.jclouds.atmosonline.saas.blobstore.strategy;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
 
 import java.util.Arrays;
@@ -40,7 +41,7 @@ import org.jclouds.blobstore.functions.ObjectMD5;
 import org.jclouds.blobstore.internal.BlobRuntimeException;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
-import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
+import org.jclouds.blobstore.strategy.ListBlobsInContainer;
 import org.jclouds.logging.Logger;
 
 import com.google.common.base.Throwables;
@@ -58,7 +59,7 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
    @Resource
    protected Logger logger = Logger.NULL;
    protected final ObjectMD5 objectMD5;
-   protected final ListBlobMetadataStrategy getAllBlobMetadata;
+   protected final ListBlobsInContainer getAllBlobMetadata;
    private final AtmosStorageAsyncClient client;
    private final ExecutorService userExecutor;
    /**
@@ -71,7 +72,7 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
    @Inject
    private FindMD5InUserMetadata(
             @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
-            ObjectMD5 objectMD5, ListBlobMetadataStrategy getAllBlobMetadata,
+            ObjectMD5 objectMD5, ListBlobsInContainer getAllBlobMetadata,
             AtmosStorageAsyncClient client) {
       this.objectMD5 = objectMD5;
       this.getAllBlobMetadata = getAllBlobMetadata;
@@ -89,9 +90,16 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
          future.addListener(new Runnable() {
             public void run() {
                try {
-                  if (Arrays.equals(toSearch, future.get().getContentMetadata().getContentMD5())) {
+                  AtmosObject object = future.get();
+                  checkNotNull(object.getSystemMetadata(), object + " has no content metadata");
+                  if (object.getSystemMetadata().getContentMD5() != null) {
+                     if (Arrays.equals(toSearch, object.getSystemMetadata().getContentMD5())) {
                         queue.put(true);
                      }
+                  } else {
+                     logger.debug("object %s has no content md5", object.getSystemMetadata()
+                              .getObjectID());
+                  }
                } catch (InterruptedException e) {
                   Throwables.propagate(e);
                } catch (ExecutionException e) {
@@ -107,7 +115,7 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
          throw new BlobRuntimeException(String.format("searching for md5 in container %s: %s",
                   containerName, exceptions));
       try {
-         return queue.poll(1, TimeUnit.MICROSECONDS);
+         return queue.poll(1, TimeUnit.MICROSECONDS) != null;
       } catch (InterruptedException e) {
          Throwables.propagate(e);
          return false;

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/RecursiveRemove.java

@@ -1,132 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
- *
- * ====================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ====================================================================
- */
-package org.jclouds.atmosonline.saas.blobstore.strategy;
-
-import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
-
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-
-import javax.annotation.Resource;
-import javax.inject.Named;
-import javax.inject.Singleton;
-
-import org.jclouds.Constants;
-import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient;
-import org.jclouds.atmosonline.saas.AtmosStorageClient;
-import org.jclouds.atmosonline.saas.domain.DirectoryEntry;
-import org.jclouds.atmosonline.saas.domain.FileType;
-import org.jclouds.blobstore.internal.BlobRuntimeException;
-import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.ClearContainerStrategy;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
-import org.jclouds.logging.Logger;
-import org.jclouds.util.Utils;
-
-import com.google.common.base.Function;
-import com.google.common.base.Supplier;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.inject.Inject;
-
-/**
- * Recursively remove a path.
- * 
- * @author Adrian Cole
- */
-@Singleton
-public class RecursiveRemove implements ClearListStrategy, ClearContainerStrategy {
-   /**
-    * maximum duration of an blob Request
-    */
-   @Inject(optional = true)
-   @Named(Constants.PROPERTY_HTTP_REQUEST_TIMEOUT)
-   protected Long maxTime;
-   protected final AtmosStorageAsyncClient async;
-   protected final AtmosStorageClient sync;
-   private final ExecutorService userExecutor;
-
-   @Resource
-   protected Logger logger = Logger.NULL;
-
-   @Inject
-   public RecursiveRemove(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
-            AtmosStorageAsyncClient connection, AtmosStorageClient sync) {
-      this.async = connection;
-      this.sync = sync;
-      this.userExecutor = userExecutor;
-   }
-
-   public void execute(String containerName) {
-      logger.debug("clearing container ", containerName);
-      execute(containerName, new ListContainerOptions().recursive());
-      logger.trace("cleared container " + containerName);
-   }
-
-   private ListenableFuture<Void> rm(final String fullPath, FileType type, boolean recursive) {
-      Map<String, ListenableFuture<?>> responses = Maps.newHashMap();
-      if ((type == FileType.DIRECTORY) && recursive) {
-         for (DirectoryEntry child : sync.listDirectory(fullPath)) {
-            responses.put(fullPath + "/" + child.getObjectName(), rm(fullPath + "/"
-                     + child.getObjectName(), child.getType(), true));
-         }
-      }
-      Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
-               String.format("deleting from path: %s", fullPath));
-      if (exceptions.size() > 0)
-         throw new BlobRuntimeException(String.format("deleting from path %s: %s", fullPath,
-                  exceptions));
-
-      return Futures.compose(async.deletePath(fullPath), new Function<Void, Void>() {
-
-         public Void apply(Void from) {
-            try {
-               if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
-                  public Boolean get() {
-                     return !sync.pathExists(fullPath);
-                  }
-               }, maxTime != null ? maxTime : 1000)) {
-                  throw new IllegalStateException(fullPath + " still exists after deleting!");
-               }
-               return null;
-            } catch (InterruptedException e) {
-               throw new IllegalStateException(fullPath + " still exists after deleting!", e);
-            }
-         }
-
-      });
-   }
-
-   public void execute(String path, ListContainerOptions options) {
-      if (options.getDir() != null)
-         path += "/" + options.getDir();
-      Map<String, ListenableFuture<?>> responses = Maps.newHashMap();
-      for (DirectoryEntry md : sync.listDirectory(path)) {
-         responses.put(path + "/" + md.getObjectName(), rm(path + "/" + md.getObjectName(), md
-                  .getType(), options.isRecursive()));
-      }
-      Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
-               String.format("deleting from path: %s", path));
-      if (exceptions.size() > 0)
-         throw new BlobRuntimeException(String
-                  .format("deleting from path %s: %s", path, exceptions));
-   }
-
-}

atmos/src/main/java/org/jclouds/atmosonline/saas/domain/DirectoryEntry.java

@@ -88,4 +88,10 @@ public class DirectoryEntry implements Comparable<DirectoryEntry> {
          return false;
       return true;
    }
+
+   @Override
+   public String toString() {
+      return "DirectoryEntry [type=" + type + ", objectid=" + objectid + ", objname=" + objname
+               + "]";
+   }
 }

atmos/src/main/java/org/jclouds/atmosonline/saas/domain/SystemMetadata.java

@@ -20,6 +20,8 @@ package org.jclouds.atmosonline.saas.domain;
 
 import java.util.Date;
 
+import javax.annotation.Nullable;
+
 /**
  * Metadata of a Atmos Online object
  * 
@@ -36,10 +38,12 @@ public class SystemMetadata extends DirectoryEntry {
    private final String policyname;
    private final long size;
    private final String uid;
+   private final byte[] contentmd5;
 
-   public SystemMetadata(Date atime, Date ctime, String gid, Date itime, Date mtime, int nlink,
+   public SystemMetadata(@Nullable byte [] contentmd5, Date atime, Date ctime, String gid, Date itime, Date mtime, int nlink,
             String objectid, String objname, String policyname, long size, FileType type, String uid) {
       super(objectid, type, objname);
+      this.contentmd5 = contentmd5;
       this.atime = atime;
       this.ctime = ctime;
       this.gid = gid;
@@ -87,6 +91,10 @@ public class SystemMetadata extends DirectoryEntry {
       return uid;
    }
 
+   public byte[] getContentMD5() {
+      return contentmd5;
+   }
+
    @Override
    public int hashCode() {
       final int prime = 31;
@@ -154,4 +162,9 @@ public class SystemMetadata extends DirectoryEntry {
       return true;
    }
 
+   @Override
+   public String toString() {
+      return "[type=" + getType() + ", id=" + getObjectID() + ", name=" + getObjectName() + "]";
+   }
+
 }

atmos/src/main/java/org/jclouds/atmosonline/saas/domain/internal/AtmosObjectImpl.java

@@ -78,6 +78,27 @@ public class AtmosObjectImpl extends BasePayloadEnclosingImpl implements AtmosOb
       this.allHeaders = checkNotNull(allHeaders, "allHeaders");
    }
 
+   /**
+    * {@inheritDoc}
+    */
+   @Override
+   public int hashCode() {
+      return systemMetadata != null ? systemMetadata.hashCode() : super.hashCode();
+   }
+
+   /**
+    * {@inheritDoc}
+    */
+   @Override
+   public boolean equals(Object obj) {
+      return systemMetadata != null ? systemMetadata.equals(obj) : super.equals(obj);
+   }
+
+   @Override
+   public String toString() {
+      return "[metadata=" + systemMetadata + "]";
+   }
+
    /**
     * {@inheritDoc}
     */

atmos/src/main/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeaders.java

@@ -29,6 +29,7 @@ import org.jclouds.atmosonline.saas.domain.FileType;
 import org.jclouds.atmosonline.saas.domain.SystemMetadata;
 import org.jclouds.atmosonline.saas.reference.AtmosStorageHeaders;
 import org.jclouds.date.DateService;
+import org.jclouds.encryption.EncryptionService;
 import org.jclouds.http.HttpResponse;
 
 import com.google.common.base.Function;
@@ -40,10 +41,13 @@ import com.google.common.collect.Maps;
 @Singleton
 public class ParseSystemMetadataFromHeaders implements Function<HttpResponse, SystemMetadata> {
    private final DateService dateService;
+   private final EncryptionService encryptionService;
 
    @Inject
-   public ParseSystemMetadataFromHeaders(DateService dateService) {
+   public ParseSystemMetadataFromHeaders(DateService dateService,
+            EncryptionService encryptionService) {
       this.dateService = dateService;
+      this.encryptionService = encryptionService;
    }
 
    public SystemMetadata apply(HttpResponse from) {
@@ -56,8 +60,9 @@ public class ParseSystemMetadataFromHeaders implements Function<HttpResponse, Sy
          metaMap.put(entrySplit[0], entrySplit[1]);
       }
       assert metaMap.size() >= 12 : String.format("Should be 12 entries in %s", metaMap);
-
+      byte[] md5 = metaMap.containsKey("content-md5") ? encryptionService.fromHexString(metaMap
-      return new SystemMetadata(dateService.iso8601SecondsDateParse(checkNotNull(metaMap
+               .get("content-md5")) : null;
+      return new SystemMetadata(md5, dateService.iso8601SecondsDateParse(checkNotNull(metaMap
                .get("atime"), "atime")), dateService.iso8601SecondsDateParse(checkNotNull(metaMap
                .get("ctime"), "ctime")), checkNotNull(metaMap.get("gid"), "gid"), dateService
                .iso8601SecondsDateParse(checkNotNull(metaMap.get("itime"), "itime")), dateService

atmos/src/main/java/org/jclouds/atmosonline/saas/handlers/ParseAtmosStorageErrorFromXmlContent.java

@@ -60,8 +60,8 @@ public class ParseAtmosStorageErrorFromXmlContent implements HttpErrorHandler {
       this.utils = utils;
    }
 
-   public static final Pattern CONTAINER_PATH = Pattern.compile("^/rest/namespace/?([^/]+)[/]?$");
+   public static final Pattern DIRECTORY_PATH = Pattern.compile("^/rest/namespace/?([^/]+)/$");
-   public static final Pattern CONTAINER_KEY_PATH = Pattern
+   public static final Pattern DIRECTORY_KEY_PATH = Pattern
             .compile("^/rest/namespace/?([^/]+)/(.*)");
 
    public void handleError(HttpCommand command, HttpResponse response) {
@@ -84,11 +84,11 @@ public class ParseAtmosStorageErrorFromXmlContent implements HttpErrorHandler {
                               "%s -> %s", command.getRequest().getRequestLine(), response
                                        .getStatusLine());
                      String path = command.getRequest().getEndpoint().getPath();
-                     Matcher matcher = CONTAINER_PATH.matcher(path);
+                     Matcher matcher = DIRECTORY_PATH.matcher(path);
                      if (matcher.find()) {
                         exception = new ContainerNotFoundException(matcher.group(1), message);
                      } else {
-                        matcher = CONTAINER_KEY_PATH.matcher(path);
+                        matcher = DIRECTORY_KEY_PATH.matcher(path);
                         if (matcher.find()) {
                            exception = new KeyNotFoundException(matcher.group(1), matcher.group(2),
                                     message);

atmos/src/main/java/org/jclouds/atmosonline/saas/options/ListOptions.java

@@ -51,7 +51,7 @@ public class ListOptions extends BaseHttpRequestOptions {
     */
    public ListOptions limit(int maxresults) {
       checkState(maxresults >= 0, "maxresults must be >= 0");
-      checkState(maxresults <= 10000, "maxresults must be <= 5000");
+      checkState(maxresults <= 10000, "maxresults must be <= 10000");
       headers.put("x-emc-limit", Integer.toString(maxresults));
       return this;
    }

atmos/src/main/java/org/jclouds/atmosonline/saas/util/AtmosStorageUtils.java

@@ -24,13 +24,20 @@ import java.io.InputStream;
 import javax.inject.Inject;
 import javax.inject.Provider;
 
+import org.jclouds.atmosonline.saas.AtmosStorageClient;
+import org.jclouds.atmosonline.saas.blobstore.functions.BlobToObject;
 import org.jclouds.atmosonline.saas.domain.AtmosStorageError;
 import org.jclouds.atmosonline.saas.filters.SignRequest;
 import org.jclouds.atmosonline.saas.xml.ErrorHandler;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.encryption.EncryptionService;
 import org.jclouds.http.HttpCommand;
 import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpResponse;
 import org.jclouds.http.functions.ParseSax;
+import org.jclouds.util.Utils;
+
+import com.google.common.base.Supplier;
 
 /**
  * Encryption, Hashing, and IO Utilities needed to sign and verify Atmos Storage requests and
@@ -60,6 +67,32 @@ public class AtmosStorageUtils {
 
    }
 
+   public static String putBlob(final AtmosStorageClient sync, EncryptionService encryptionService,
+            BlobToObject blob2Object, String container, Blob blob) {
+      final String path = container + "/" + blob.getMetadata().getName();
+      deleteAndEnsureGone(sync, path);
+      if (blob.getMetadata().getContentMD5() != null)
+         blob.getMetadata().getUserMetadata().put("content-md5",
+                  encryptionService.toHexString(blob.getMetadata().getContentMD5()));
+      sync.createFile(container, blob2Object.apply(blob));
+      return path;
+   }
+
+   public static void deleteAndEnsureGone(final AtmosStorageClient sync, final String path) {
+      try {
+         if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
+            public Boolean get() {
+               sync.deletePath(path);
+               return !sync.pathExists(path);
+            }
+         }, 3000)) {
+            throw new IllegalStateException(path + " still exists after deleting!");
+         }
+      } catch (InterruptedException e) {
+         new IllegalStateException(path + " interrupted during deletion!", e);
+      }
+   }
+
    public AtmosStorageError parseAtmosStorageErrorFromContent(HttpCommand command,
             HttpResponse response, String content) throws HttpException {
       return parseAtmosStorageErrorFromContent(command, response, new ByteArrayInputStream(content
@@ -69,9 +102,9 @@ public class AtmosStorageUtils {
    public static String adjustContainerIfDirOptionPresent(String container,
             org.jclouds.blobstore.options.ListContainerOptions options) {
       if (options != org.jclouds.blobstore.options.ListContainerOptions.NONE) {
-         if (options.isRecursive()) {
+         // if (options.isRecursive()) {
-            throw new UnsupportedOperationException("recursive not currently supported in emcsaas");
+         // throw new UnsupportedOperationException("recursive not currently supported in emcsaas");
-         }
+         // }
          if (options.getDir() != null) {
             container = container + "/" + options.getDir();
          }

atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java

@@ -26,22 +26,18 @@ import java.lang.reflect.UndeclaredThrowableException;
 import java.net.URI;
 import java.security.SecureRandom;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.TimeoutException;
 
-import org.jclouds.atmosonline.saas.blobstore.strategy.RecursiveRemove;
 import org.jclouds.atmosonline.saas.domain.AtmosObject;
 import org.jclouds.atmosonline.saas.domain.BoundedSet;
 import org.jclouds.atmosonline.saas.domain.DirectoryEntry;
 import org.jclouds.atmosonline.saas.domain.FileType;
 import org.jclouds.atmosonline.saas.domain.SystemMetadata;
 import org.jclouds.atmosonline.saas.options.ListOptions;
-import org.jclouds.blobstore.ContainerNotFoundException;
+import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.KeyAlreadyExistsException;
 import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
-import org.jclouds.blobstore.strategy.ClearContainerStrategy;
 import org.jclouds.http.HttpResponseException;
 import org.jclouds.http.Payloads;
 import org.jclouds.http.payloads.InputStreamPayload;
@@ -107,7 +103,7 @@ public class AtmosStorageClientLiveTest {
 
    private static final int INCONSISTENCY_WINDOW = 5000;
    protected AtmosStorageClient connection;
-   private String containerPrefix = BaseBlobStoreIntegrationTest.CONTAINER_PREFIX+"live";
+   private String containerPrefix = BaseBlobStoreIntegrationTest.CONTAINER_PREFIX + "live";
 
    URI container1;
    URI container2;
@@ -116,23 +112,15 @@ public class AtmosStorageClientLiveTest {
    public void setupClient() throws InterruptedException, ExecutionException, TimeoutException {
       String uid = checkNotNull(System.getProperty("jclouds.test.user"), "jclouds.test.user");
       String key = checkNotNull(System.getProperty("jclouds.test.key"), "jclouds.test.key");
-
+      BlobStoreContext blobStoreContext = new AtmosStorageContextBuilder(
-      RestContext<AtmosStorageAsyncClient, AtmosStorageClient> context = new AtmosStorageContextBuilder(
                new AtmosStoragePropertiesBuilder(uid, key).build()).withModules(
-               new Log4JLoggingModule()).buildContext();
+               new Log4JLoggingModule()).buildBlobStoreContext();
-      ExecutorService service = Executors.newCachedThreadPool();
+      RestContext<AtmosStorageAsyncClient, AtmosStorageClient> context = blobStoreContext
+               .getProviderSpecificContext();
       connection = context.getApi();
-      ClearContainerStrategy clearer = new RecursiveRemove(service, context.getAsyncApi(),
-               connection);
       for (DirectoryEntry entry : connection.listDirectories()) {
-         try {
          if (entry.getObjectName().startsWith(containerPrefix)) {
-               clearer.execute(entry.getObjectName());
+            blobStoreContext.getBlobStore().deleteContainer(entry.getObjectName());
-               deleteConfirmed(entry.getObjectName());
-            }
-         } catch (ContainerNotFoundException e) {
-            if (entry.getType() != FileType.DIRECTORY)
-               throw e;
          }
       }
    }

atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientTest.java

@@ -39,7 +39,6 @@ import org.jclouds.atmosonline.saas.options.ListOptions;
 import org.jclouds.atmosonline.saas.reference.AtmosStorageConstants;
 import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
 import org.jclouds.blobstore.config.BlobStoreObjectModule;
-import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
 import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
 import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404;
 import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
@@ -51,6 +50,7 @@ import org.jclouds.http.options.GetOptions;
 import org.jclouds.logging.Logger;
 import org.jclouds.logging.Logger.LoggerFactory;
 import org.jclouds.rest.RestClientTest;
+import org.jclouds.rest.functions.ReturnNullOnResourceNotFound;
 import org.jclouds.rest.internal.GeneratedHttpRequest;
 import org.jclouds.rest.internal.RestAnnotationProcessor;
 import org.jclouds.util.Jsr330;
@@ -221,7 +221,7 @@ public class AtmosStorageClientTest extends RestClientTest<AtmosStorageAsyncClie
       assertResponseParserClassEquals(method, httpMethod,
                ParseObjectFromHeadersAndHttpContent.class);
       assertSaxResponseParserClassEquals(method, null);
-      assertExceptionParserClassEquals(method, ReturnNullOnKeyNotFound.class);
+      assertExceptionParserClassEquals(method, ReturnNullOnResourceNotFound.class);
 
       checkFilters(httpMethod);
    }
@@ -238,7 +238,7 @@ public class AtmosStorageClientTest extends RestClientTest<AtmosStorageAsyncClie
 
       assertResponseParserClassEquals(method, httpMethod, ParseSystemMetadataFromHeaders.class);
       assertSaxResponseParserClassEquals(method, null);
-      assertExceptionParserClassEquals(method, ReturnNullOnKeyNotFound.class);
+      assertExceptionParserClassEquals(method, ReturnNullOnResourceNotFound.class);
 
       checkFilters(httpMethod);
    }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageContainerIntegrationTest.java

@@ -30,16 +30,9 @@ import org.testng.annotations.Test;
 public class AtmosStorageContainerIntegrationTest extends BaseContainerIntegrationTest {
 
    @Override
-   @Test(enabled = false)
+   public void testListContainerMaxResults() throws InterruptedException,
-   // some reason this fails on the stub.
+            UnsupportedEncodingException {
-   public void testClearWhenContentsUnderPath() throws InterruptedException {
+      // Not currently working
-      super.testClearWhenContentsUnderPath();
-   }
-
-   @Override
-   @Test(enabled = false)
-   public void testDirectory() throws InterruptedException, UnsupportedEncodingException {
-      // TODO
    }
 
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageInputStreamMapIntegrationTest.java

@@ -82,9 +82,4 @@ public class AtmosStorageInputStreamMapIntegrationTest extends BaseInputStreamMa
       // TODO not reliable NPE
    }
 
-   @Override
-   protected int maxList() {
-      return 0;
-   }
-
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageMapIntegrationTest.java

@@ -18,9 +18,6 @@
  */
 package org.jclouds.atmosonline.saas.blobstore.integration;
 
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeoutException;
-
 import org.jclouds.blobstore.integration.internal.BaseBlobMapIntegrationTest;
 import org.testng.annotations.Test;
 
@@ -30,21 +27,4 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "emcsaas.AtmosStorageMapIntegrationTest")
 public class AtmosStorageMapIntegrationTest extends BaseBlobMapIntegrationTest {
 
-   @Override
-   @Test(enabled = false)
-   public void testContains() throws InterruptedException, ExecutionException, TimeoutException {
-      // TODO not reliable
-   }
-
-   @Override
-   @Test(enabled = false)
-   public void testValues() {
-      // TODO not reliable KeyAlreadyExistsException@AtmosBlobStore.java:213
-   }
-
-   @Override
-   protected int maxList() {
-      return 0;
-   }
-
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeadersTest.java

@@ -23,6 +23,7 @@ import static org.testng.Assert.assertEquals;
 import org.jclouds.atmosonline.saas.domain.FileType;
 import org.jclouds.atmosonline.saas.domain.SystemMetadata;
 import org.jclouds.date.DateService;
+import org.jclouds.encryption.EncryptionService;
 import org.jclouds.http.HttpResponse;
 import org.testng.annotations.Test;
 
@@ -42,17 +43,19 @@ public class ParseSystemMetadataFromHeadersTest {
       ParseSystemMetadataFromHeaders parser = injector
                .getInstance(ParseSystemMetadataFromHeaders.class);
       DateService dateService = injector.getInstance(DateService.class);
+      EncryptionService encryptionService = injector.getInstance(EncryptionService.class);
 
       HttpResponse response = new HttpResponse();
       response
                .getHeaders()
                .put(
                         "x-emc-meta",
-                        "atime=2009-10-12T16:09:42Z, mtime=2009-10-19T04:37:00Z,"
+                        "content-md5=1f3870be274f6c49b3e31a0c6728957f, atime=2009-10-12T16:09:42Z, mtime=2009-10-19T04:37:00Z,"
                                  + " ctime=2009-10-19T04:37:00Z, itime=2009-10-12T16:09:42Z, type=directory, uid=root, "
                                  + "gid=rootr, objectid=4980cdb2b010109b04a44f7bb83f5f04ad354c638ae5, "
                                  + "objname=e913e09366364e9ba384b8fead643d43, size=4096, nlink=1, policyname=default");
-      SystemMetadata expected = new SystemMetadata(
+      SystemMetadata expected = new SystemMetadata(encryptionService
+               .fromHexString("1f3870be274f6c49b3e31a0c6728957f"),
 
       dateService.iso8601SecondsDateParse("2009-10-12T16:09:42Z"), dateService
                .iso8601SecondsDateParse("2009-10-19T04:37:00Z"), "rootr", dateService

atmos/src/test/java/org/jclouds/atmosonline/saas/internal/StubAtmosStorageAsyncClient.java

@@ -135,15 +135,17 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
    }
 
    public ListenableFuture<Void> deletePath(String path) {
-      if (path.indexOf('/') == -1)
+      if (path.indexOf('/') == path.length() - 1) {
-         return compose(blobStore.deleteContainerImpl(path), new Function<Boolean, Void>() {
+         // chop off the trailing slash
+         return compose(blobStore.deleteContainerImpl(path.substring(0, path.length() - 1)),
+                  new Function<Boolean, Void>() {
 
                      public Void apply(Boolean from) {
                         return null;
                      }
 
                   });
-      else {
+      } else {
          String container = path.substring(0, path.indexOf('/'));
          path = path.substring(path.indexOf('/') + 1);
          return blobStore.removeBlob(container, path);
@@ -205,9 +207,10 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
    }
 
    public ListenableFuture<Boolean> pathExists(final String path) {
-      if (path.indexOf('/') == -1 )
+      if (path.indexOf('/') == path.length() - 1) {
-         return blobStore.containerExists(path);
+         // chop off the trailing slash
-      else {
+         return blobStore.containerExists(path.substring(0, path.length() - 1));
+      } else {
          String container = path.substring(0, path.indexOf('/'));
          String blobName = path.substring(path.indexOf('/') + 1);
          try {

atmos/src/test/resources/log4j.xml

@@ -1,4 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+    <!--
+
+
+        Copyright (C) 2009 Cloud Conscious, LLC.
+        <info@cloudconscious.com>
+
+        ====================================================================
+        Licensed under the Apache License, Version 2.0 (the "License");
+        you may not use this file except in compliance with the License.
+        You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+        applicable law or agreed to in writing, software distributed
+        under the License is distributed on an "AS IS" BASIS, WITHOUT
+        WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        See the License for the specific language governing permissions
+        and limitations under the License.
+        ====================================================================
+    -->
 <!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+    <!--
+        For more configuration infromation and examples see the Apache
+        Log4j website: http://logging.apache.org/log4j/
+    -->
 <log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
     debug="false">
 
@@ -46,9 +71,17 @@
         </layout>
     </appender>
 
+    <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+        <appender-ref ref="FILE" />
+    </appender>
+
+    <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
+        <appender-ref ref="WIREFILE" />
+    </appender>
+
     <!-- A time/date based rolling appender -->
-    <appender name="COMPUTEFILE" class="org.apache.log4j.DailyRollingFileAppender">
+    <appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
-        <param name="File" value="target/test-data/jclouds-compute.log" />
+        <param name="File" value="target/test-data/jclouds-blobstore.log" />
         <param name="Append" value="true" />
 
         <!-- Rollover at midnight each day -->
@@ -61,28 +94,23 @@
             <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
 
             <!--
-                The full pattern: Date MS Priority [Category] (Thread:NDC) Message\n
+                The full pattern: Date MS Priority [Category]
-                <param name="ConversionPattern" value="%d %-5r %-5p [%c] (%t:%x)
+                (Thread:NDC) Message\n <param name="ConversionPattern"
-                %m%n"/>
+                value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
             -->
         </layout>
     </appender>
 
-    <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
+    <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
-        <appender-ref ref="COMPUTEFILE" />
+        <appender-ref ref="BLOBSTOREFILE" />
     </appender>
-
-    <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
-        <appender-ref ref="FILE" />
-    </appender>
-
-    <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
-        <appender-ref ref="WIREFILE" />
-    </appender>
-
     <!-- ================ -->
     <!-- Limit categories -->
     <!-- ================ -->
+    <category name="jclouds.blobstore">
+        <priority value="TRACE" />
+        <appender-ref ref="ASYNCBLOBSTORE" />
+    </category>
 
     <category name="org.jclouds">
         <priority value="DEBUG" />
@@ -92,20 +120,15 @@
     <category name="jclouds.headers">
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCWIRE" />
-    </category>
+    </category><!--
 
     <category name="jclouds.wire">
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCWIRE" />
-    </category><!--
- 
-    <category name="jclouds.signature">
-        <priority value="DEBUG" />
-        <appender-ref ref="ASYNCWIRE" />
     </category>
    
    
-     --><!-- ================ -->
+    --><!-- ======================= -->
     <!-- Setup the Root category -->
     <!-- ======================= -->
 

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java

@@ -143,14 +143,14 @@ public class S3BlobStore extends BaseBlobStore {
     */
    @Override
    public void deleteContainer(String container) {
-      deleteAndEnsurePathGone(container);
+      clearAndDeleteContainer(container);
    }
 
    /**
     * This implementation invokes {@link #clearContainer} then {@link S3Client#deleteBucketIfEmpty}
     * until it is true.
     */
-   public void deleteAndEnsurePathGone(final String container) {
+   public void clearAndDeleteContainer(final String container) {
       try {
          if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
             public Boolean get() {

aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3BlobMapIntegrationTest.java

@@ -27,9 +27,4 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "s3.S3BlobMapIntegrationTest")
 public class S3BlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
 
-   @Override
-   protected int maxList() {
-      return 1000;
-   }
-
 }

aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3InputStreamMapIntegrationTest.java

@@ -26,8 +26,5 @@ import org.testng.annotations.Test;
  */
 @Test(groups = { "integration", "live" }, testName = "s3.S3InputStreamMapIntegrationTest")
 public class S3InputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
-   @Override
+
-   protected int maxList() {
-      return 1000;
-   }
 }

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/strategy/FindMD5InBlobProperties.java

@@ -30,7 +30,7 @@ import org.jclouds.blobstore.functions.ObjectMD5;
 import org.jclouds.blobstore.internal.BlobRuntimeException;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
-import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
+import org.jclouds.blobstore.strategy.ListBlobsInContainer;
 
 import com.google.common.base.Throwables;
 
@@ -43,12 +43,12 @@ import com.google.common.base.Throwables;
 public class FindMD5InBlobProperties implements ContainsValueInListStrategy {
 
    protected final ObjectMD5 objectMD5;
-   protected final ListBlobMetadataStrategy getAllBlobMetadata;
+   protected final ListBlobsInContainer getAllBlobMetadata;
    private final AzureBlobClient client;
 
    @Inject
    private FindMD5InBlobProperties(ObjectMD5 objectMD5,
-            ListBlobMetadataStrategy getAllBlobMetadata, AzureBlobClient client) {
+            ListBlobsInContainer getAllBlobMetadata, AzureBlobClient client) {
       this.objectMD5 = objectMD5;
       this.getAllBlobMetadata = getAllBlobMetadata;
       this.client = client;

azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobClientLiveTest.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.azure.storage.blob;
 
+import static org.jclouds.azure.storage.blob.options.CreateContainerOptions.Builder.withMetadata;
+import static org.jclouds.azure.storage.blob.options.CreateContainerOptions.Builder.withPublicAcl;
 import static org.jclouds.azure.storage.options.ListOptions.Builder.includeMetadata;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
@@ -33,7 +35,6 @@ import org.jclouds.azure.storage.blob.domain.AzureBlob;
 import org.jclouds.azure.storage.blob.domain.BlobProperties;
 import org.jclouds.azure.storage.blob.domain.ContainerProperties;
 import org.jclouds.azure.storage.blob.domain.ListBlobsResponse;
-import org.jclouds.azure.storage.blob.options.CreateContainerOptions;
 import org.jclouds.azure.storage.blob.options.ListBlobsOptions;
 import org.jclouds.azure.storage.domain.BoundedSet;
 import org.jclouds.azure.storage.options.ListOptions;
@@ -59,7 +60,7 @@ import com.google.inject.internal.Iterables;
 @Test(groups = "live", sequential = true, testName = "azureblob.AzureBlobClientLiveTest")
 public class AzureBlobClientLiveTest {
 
-   protected AzureBlobClient connection;
+   protected AzureBlobClient client;
 
    private String containerPrefix = System.getProperty("user.name") + "-azureblob";
    private EncryptionService encryptionService = new JCEEncryptionService();
@@ -68,13 +69,13 @@ public class AzureBlobClientLiveTest {
    public void setupClient() {
       account = System.getProperty("jclouds.test.user");
       String key = System.getProperty("jclouds.test.key");
-      connection = (AzureBlobClient) AzureBlobContextFactory.createContext(account, key,
+      client = (AzureBlobClient) AzureBlobContextFactory.createContext(account, key,
                new Log4JLoggingModule()).getProviderSpecificContext().getApi();
    }
 
    @Test
    public void testListContainers() throws Exception {
-      Set<ContainerProperties> response = connection.listContainers();
+      Set<ContainerProperties> response = client.listContainers();
       assert null != response;
       long initialContainerCount = response.size();
       assertTrue(initialContainerCount >= 0);
@@ -91,8 +92,8 @@ public class AzureBlobClientLiveTest {
       while (!created) {
          privateContainer = containerPrefix + new SecureRandom().nextInt();
          try {
-            created = connection.createContainer(privateContainer, CreateContainerOptions.Builder
+            created = client.createContainer(privateContainer, withMetadata(ImmutableMultimap
-                     .withMetadata(ImmutableMultimap.of("foo", "bar")));
+                     .of("foo", "bar")));
          } catch (UndeclaredThrowableException e) {
             HttpResponseException htpe = (HttpResponseException) e.getCause().getCause();
             if (htpe.getResponse().getStatusCode() == 409)
@@ -100,11 +101,11 @@ public class AzureBlobClientLiveTest {
             throw e;
          }
       }
-      Set<ContainerProperties> response = connection.listContainers(includeMetadata());
+      Set<ContainerProperties> response = client.listContainers(includeMetadata());
       assert null != response;
       long containerCount = response.size();
       assertTrue(containerCount >= 1);
-      ListBlobsResponse list = connection.listBlobs(privateContainer);
+      ListBlobsResponse list = client.listBlobs(privateContainer);
       assertEquals(list.getUrl(), URI.create(String.format("https://%s.blob.core.windows.net/%s",
                account, privateContainer)));
       // TODO ... check to see the container actually exists
@@ -116,8 +117,7 @@ public class AzureBlobClientLiveTest {
       while (!created) {
          publicContainer = containerPrefix + new SecureRandom().nextInt();
          try {
-            created = connection.createContainer(publicContainer, CreateContainerOptions.Builder
+            created = client.createContainer(publicContainer, withPublicAcl());
-                     .withPublicAcl());
          } catch (UndeclaredThrowableException e) {
             HttpResponseException htpe = (HttpResponseException) e.getCause().getCause();
             if (htpe.getResponse().getStatusCode() == 409)
@@ -134,7 +134,7 @@ public class AzureBlobClientLiveTest {
    @Test(timeOut = 5 * 60 * 1000)
    public void testCreatePublicRootContainer() throws Exception {
       try {
-         connection.deleteRootContainer();
+         client.deleteRootContainer();
       } catch (ContainerNotFoundException e) {
          Thread.sleep(5000);
       } catch (AzureStorageResponseException htpe) {
@@ -148,7 +148,7 @@ public class AzureBlobClientLiveTest {
       boolean created = false;
       while (!created) {
          try {
-            created = connection.createRootContainer();
+            created = client.createRootContainer();
          } catch (AzureStorageResponseException htpe) {
             if (htpe.getResponse().getStatusCode() == 409) {// TODO look for specific message
                Thread.sleep(5000);
@@ -158,7 +158,7 @@ public class AzureBlobClientLiveTest {
             }
          }
       }
-      ListBlobsResponse list = connection.listBlobs();
+      ListBlobsResponse list = client.listBlobs();
       assertEquals(list.getUrl(), URI.create(String.format(
                "https://%s.blob.core.windows.net/%%24root", account)));
    }
@@ -166,7 +166,7 @@ public class AzureBlobClientLiveTest {
    @Test
    public void testListContainersWithOptions() throws Exception {
 
-      BoundedSet<ContainerProperties> response = connection.listContainers(ListOptions.Builder
+      BoundedSet<ContainerProperties> response = client.listContainers(ListOptions.Builder
                .prefix(privateContainer).maxResults(1).includeMetadata());
       assert null != response;
       long initialContainerCount = response.size();
@@ -177,7 +177,7 @@ public class AzureBlobClientLiveTest {
 
    @Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testCreatePublicRootContainer" })
    public void testDeleteRootContainer() throws Exception {
-      connection.deleteRootContainer();
+      client.deleteRootContainer();
       // TODO loop for up to 30 seconds checking if they are really gone
    }
 
@@ -186,19 +186,19 @@ public class AzureBlobClientLiveTest {
    public void testListOwnedContainers() throws Exception {
 
       // Test default listing
-      Set<ContainerProperties> response = connection.listContainers();
+      Set<ContainerProperties> response = client.listContainers();
       // assertEquals(response.size(), initialContainerCount + 2);// if the containers already
       // exist, this will fail
 
       // Test listing with options
-      response = connection.listContainers(ListOptions.Builder.prefix(
+      response = client.listContainers(ListOptions.Builder.prefix(
                privateContainer.substring(0, privateContainer.length() - 1)).maxResults(1)
                .includeMetadata());
       assertEquals(response.size(), 1);
       assertEquals(Iterables.getOnlyElement(response).getName(), privateContainer);
       assertEquals(Iterables.getOnlyElement(response).getMetadata(), ImmutableMap.of("foo", "bar"));
 
-      response = connection.listContainers(ListOptions.Builder.prefix(publicContainer)
+      response = client.listContainers(ListOptions.Builder.prefix(publicContainer)
                .maxResults(1));
       assertEquals(response.size(), 1);
       assertEquals(Iterables.getOnlyElement(response).getName(), publicContainer);
@@ -207,14 +207,14 @@ public class AzureBlobClientLiveTest {
 
    @Test
    public void testDeleteOneContainer() throws Exception {
-      connection.deleteContainer("does-not-exist");
+      client.deleteContainer("does-not-exist");
    }
 
    @Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testListOwnedContainers",
             "testObjectOperations" })
    public void testDeleteContainer() throws Exception {
-      connection.deleteContainer(privateContainer);
+      client.deleteContainer(privateContainer);
-      connection.deleteContainer(publicContainer);
+      client.deleteContainer(publicContainer);
       // TODO loop for up to 30 seconds checking if they are really gone
    }
 
@@ -224,7 +224,7 @@ public class AzureBlobClientLiveTest {
       String data = "Here is my data";
 
       // Test PUT with string data, ETag hash, and a piece of metadata
-      AzureBlob object = connection.newBlob();
+      AzureBlob object = client.newBlob();
       object.getProperties().setName("object");
       object.setPayload(data);
       object.setContentLength(data.length());
@@ -232,15 +232,15 @@ public class AzureBlobClientLiveTest {
       object.getProperties().setContentType("text/plain");
       object.getProperties().getMetadata().put("mykey", "metadata-value");
       byte[] md5 = object.getProperties().getContentMD5();
-      String newEtag = connection.putBlob(privateContainer, object);
+      String newEtag = client.putBlob(privateContainer, object);
       assertEquals(encryptionService.toHexString(md5), encryptionService.toHexString(object
                .getProperties().getContentMD5()));
 
       // Test HEAD of missing object
-      assert connection.getBlobProperties(privateContainer, "non-existent-object") == null;
+      assert client.getBlobProperties(privateContainer, "non-existent-object") == null;
 
       // Test HEAD of object
-      BlobProperties metadata = connection.getBlobProperties(privateContainer, object
+      BlobProperties metadata = client.getBlobProperties(privateContainer, object
                .getProperties().getName());
       // TODO assertEquals(metadata.getName(), object.getProperties().getName());
       // we can't check this while hacking around lack of content-md5, as GET of the first byte will
@@ -261,14 +261,14 @@ public class AzureBlobClientLiveTest {
       // Multimap<String, String> userMetadata = LinkedHashMultimap.create();
       // userMetadata.put("New-Metadata-1", "value-1");
       // userMetadata.put("New-Metadata-2", "value-2");
-      // assertTrue(connection.setBlobProperties(privateContainer, object.getProperties().getName(),
+      // assertTrue(client.setBlobProperties(privateContainer, object.getProperties().getName(),
       // userMetadata));
 
       // Test GET of missing object
-      assert connection.getBlob(privateContainer, "non-existent-object") == null;
+      assert client.getBlob(privateContainer, "non-existent-object") == null;
 
       // Test GET of object (including updated metadata)
-      AzureBlob getBlob = connection.getBlob(privateContainer, object.getProperties().getName());
+      AzureBlob getBlob = client.getBlob(privateContainer, object.getProperties().getName());
       assertEquals(Utils.toStringAndClose(getBlob.getContent()), data);
       // TODO assertEquals(getBlob.getName(), object.getProperties().getName());
       assertEquals(getBlob.getContentLength(), new Long(data.length()));
@@ -288,7 +288,7 @@ public class AzureBlobClientLiveTest {
       assertEquals(metadata.getMetadata().get("mykey"), "metadata-value");
 
       // test listing
-      ListBlobsResponse response = connection.listBlobs(privateContainer, ListBlobsOptions.Builder
+      ListBlobsResponse response = client.listBlobs(privateContainer, ListBlobsOptions.Builder
                .prefix(
                         object.getProperties().getName().substring(0,
                                  object.getProperties().getName().length() - 1)).maxResults(1)
@@ -303,25 +303,25 @@ public class AzureBlobClientLiveTest {
       String incorrectEtag = "0" + correctEtag.substring(1);
       object.getProperties().setETag(incorrectEtag);
       try {
-         connection.putBlob(privateContainer, object);
+         client.putBlob(privateContainer, object);
       } catch (Throwable e) {
          assertEquals(e.getCause().getClass(), HttpResponseException.class);
          assertEquals(((HttpResponseException) e.getCause()).getResponse().getStatusCode(), 422);
       }
 
       ByteArrayInputStream bais = new ByteArrayInputStream(data.getBytes("UTF-8"));
-      object = connection.newBlob();
+      object = client.newBlob();
       object.getProperties().setName("chunked-object");
       object.setPayload(bais);
       object.setContentLength(new Long(data.getBytes().length));
-      newEtag = connection.putBlob(privateContainer, object);
+      newEtag = client.putBlob(privateContainer, object);
       assertEquals(encryptionService.toHexString(md5), encryptionService.toHexString(getBlob
                .getProperties().getContentMD5()));
 
       // Test GET with options
       // Non-matching ETag
       try {
-         connection.getBlob(privateContainer, object.getProperties().getName(), GetOptions.Builder
+         client.getBlob(privateContainer, object.getProperties().getName(), GetOptions.Builder
                   .ifETagDoesntMatch(newEtag));
       } catch (Exception e) {
          assertEquals(e.getCause().getClass(), HttpResponseException.class);
@@ -330,7 +330,7 @@ public class AzureBlobClientLiveTest {
 
       // Matching ETag TODO this shouldn't fail!!!
       try {
-         getBlob = connection.getBlob(privateContainer, object.getProperties().getName(),
+         getBlob = client.getBlob(privateContainer, object.getProperties().getName(),
                   GetOptions.Builder.ifETagMatches(newEtag));
          assertEquals(getBlob.getProperties().getETag(), newEtag);
       } catch (HttpResponseException e) {
@@ -340,13 +340,13 @@ public class AzureBlobClientLiveTest {
       // Range
       // doesn't work per
       // http://social.msdn.microsoft.com/Forums/en-US/windowsazure/thread/479fa63f-51df-4b66-96b5-33ae362747b6
-      // getBlob = connection
+      // getBlob = client
       // .getBlob(privateContainer, object.getProperties().getName(),
       // GetOptions.Builder.startAt(8)).get(120,
       // TimeUnit.SECONDS);
       // assertEquals(Utils.toStringAndClose((InputStream) getBlob.getData()), data.substring(8));
 
-      connection.deleteBlob(privateContainer, "object");
+      client.deleteBlob(privateContainer, "object");
-      connection.deleteBlob(privateContainer, "chunked-object");
+      client.deleteBlob(privateContainer, "chunked-object");
    }
 }

azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobInputStreamMapIntegrationTest.java

@@ -27,9 +27,4 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "azureblob.AzureBlobInputStreamMapIntegrationTest")
 public class AzureBlobInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
 
-   @Override
-   protected int maxList() {
-      return 5000;
-   }
-
 }

azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobMapIntegrationTest.java

@@ -27,9 +27,4 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "azureblob.AzureBlobMapIntegrationTest")
 public class AzureBlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
 
-   @Override
-   protected int maxList() {
-      return 5000;
-   }
-
 }

blobstore/src/main/java/org/jclouds/blobstore/BlobMap.java

@@ -18,10 +18,9 @@
  */
 package org.jclouds.blobstore;
 
-import javax.annotation.Nullable;
-
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.internal.BlobMapImpl;
+import org.jclouds.blobstore.options.ListContainerOptions;
 
 import com.google.inject.ImplementedBy;
 
@@ -38,7 +37,7 @@ public interface BlobMap extends ListableMap<String, Blob> {
    Blob newBlob(String name);
 
    public static interface Factory {
-      BlobMap create(String containerName, @Nullable String dir);
+      BlobMap create(String containerName, ListContainerOptions options);
    }
 
 }

blobstore/src/main/java/org/jclouds/blobstore/BlobStoreContext.java

@@ -20,6 +20,7 @@ package org.jclouds.blobstore;
 
 import org.jclouds.blobstore.attr.ConsistencyModels;
 import org.jclouds.blobstore.internal.BlobStoreContextImpl;
+import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.rest.RestContext;
 
 import com.google.inject.ImplementedBy;
@@ -39,6 +40,8 @@ public interface BlobStoreContext {
     * 
     * @param container
     */
+   InputStreamMap createInputStreamMap(String container, ListContainerOptions options);
+
    InputStreamMap createInputStreamMap(String container);
 
    /**
@@ -46,6 +49,8 @@ public interface BlobStoreContext {
     * 
     * @param container
     */
+   BlobMap createBlobMap(String container, ListContainerOptions options);
+
    BlobMap createBlobMap(String container);
 
    AsyncBlobStore getAsyncBlobStore();

blobstore/src/main/java/org/jclouds/blobstore/ContainerNotFoundException.java

@@ -18,12 +18,14 @@
  */
 package org.jclouds.blobstore;
 
+import org.jclouds.rest.ResourceNotFoundException;
+
 /**
  * Thrown when a container cannot be located.
  * 
  * @author Adrian Cole
  */
-public class ContainerNotFoundException extends RuntimeException {
+public class ContainerNotFoundException extends ResourceNotFoundException {
 
    private String container;
 

blobstore/src/main/java/org/jclouds/blobstore/InputStreamMap.java

@@ -22,9 +22,8 @@ import java.io.File;
 import java.io.InputStream;
 import java.util.Map;
 
-import javax.annotation.Nullable;
-
 import org.jclouds.blobstore.internal.InputStreamMapImpl;
+import org.jclouds.blobstore.options.ListContainerOptions;
 
 import com.google.inject.ImplementedBy;
 
@@ -43,7 +42,7 @@ import com.google.inject.ImplementedBy;
 @ImplementedBy(InputStreamMapImpl.class)
 public interface InputStreamMap extends ListableMap<String, InputStream> {
    public static interface Factory {
-      InputStreamMap create(String containerName, @Nullable String dir);
+      InputStreamMap create(String containerName, ListContainerOptions options);
    }
 
    InputStream putString(String key, String value);

blobstore/src/main/java/org/jclouds/blobstore/config/BlobStoreMapModule.java

@@ -18,7 +18,6 @@
  */
 package org.jclouds.blobstore.config;
 
-import javax.annotation.Nullable;
 import javax.inject.Inject;
 
 import org.jclouds.blobstore.BlobMap;
@@ -27,10 +26,11 @@ import org.jclouds.blobstore.InputStreamMap;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.internal.BlobMapImpl;
 import org.jclouds.blobstore.internal.InputStreamMapImpl;
+import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
 import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
 import org.jclouds.blobstore.strategy.PutBlobsStrategy;
-import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
+import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Scopes;
@@ -62,11 +62,11 @@ public class BlobStoreMapModule extends AbstractModule {
       @Inject
       PutBlobsStrategy putBlobsStrategy;
       @Inject
-      ListBlobMetadataInContainer listStrategy;
+      ListContainerAndRecurseThroughFolders listStrategy;
 
-      public BlobMap create(String containerName, @Nullable String dir) {
+      public BlobMap create(String containerName, ListContainerOptions options) {
          return new BlobMapImpl(connection, getAllBlobs, containsValueStrategy, putBlobsStrategy,
-                  listStrategy, containerName, dir);
+                  listStrategy, containerName, options);
       }
 
    }
@@ -83,11 +83,11 @@ public class BlobStoreMapModule extends AbstractModule {
       @Inject
       PutBlobsStrategy putBlobsStrategy;
       @Inject
-      ListBlobMetadataInContainer listStrategy;
+      ListContainerAndRecurseThroughFolders listStrategy;
 
-      public InputStreamMap create(String containerName, @Nullable String dir) {
+      public InputStreamMap create(String containerName, ListContainerOptions options) {
          return new InputStreamMapImpl(connection, blobFactory, getAllBlobs, listStrategy,
-                  containsValueStrategy, putBlobsStrategy, containerName, dir);
+                  containsValueStrategy, putBlobsStrategy, containerName, options);
       }
 
    }

blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java

@@ -32,6 +32,7 @@ import javax.inject.Named;
 
 import org.jclouds.Constants;
 import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.blobstore.ContainerNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
@@ -238,8 +239,12 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore {
       try {
          if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
             public Boolean get() {
-               blobUtils.clearContainer(container, recursive());
+               try {
+                  clearContainer(container, recursive());
                   return deleteAndVerifyContainerGone(container);
+               } catch (ContainerNotFoundException e) {
+                  return true;
+               }
             }
 
          }, 30000)) {

blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java

@@ -20,13 +20,10 @@ package org.jclouds.blobstore.internal;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static org.jclouds.blobstore.options.ListContainerOptions.Builder.inDirectory;
 
 import java.util.Map;
 import java.util.Set;
 
-import javax.annotation.Nullable;
-
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
@@ -37,7 +34,7 @@ import org.jclouds.blobstore.options.ListContainerOptions.ImmutableListContainer
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
 import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
 import org.jclouds.blobstore.strategy.PutBlobsStrategy;
-import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
+import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
@@ -58,7 +55,7 @@ public abstract class BaseBlobMap<V> implements Map<String, V> {
    protected final ListContainerOptions options;
    protected final GetBlobsInListStrategy getAllBlobs;
    protected final ContainsValueInListStrategy containsValueStrategy;
-   protected final ListBlobMetadataInContainer listStrategy;
+   protected final ListContainerAndRecurseThroughFolders listStrategy;
    protected final PutBlobsStrategy putBlobsStrategy;
 
    static class StripPath implements Function<String, String> {
@@ -98,11 +95,15 @@ public abstract class BaseBlobMap<V> implements Map<String, V> {
    @Inject
    public BaseBlobMap(BlobStore blobstore, GetBlobsInListStrategy getAllBlobs,
             ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy,
-            ListBlobMetadataInContainer listStrategy, String containerName, @Nullable String dir) {
+            ListContainerAndRecurseThroughFolders listStrategy, String containerName,
+            ListContainerOptions options) {
       this.blobstore = checkNotNull(blobstore, "blobstore");
       this.containerName = checkNotNull(containerName, "container");
-      this.options = new ImmutableListContainerOptions(dir != null ? inDirectory(dir)
+      checkArgument(containerName.indexOf('/') == -1,
-               : ListContainerOptions.NONE);
+               "please specify directory path using the option: inDirectory, not encoded in the container name");
+      this.options = checkNotNull(options, "options") instanceof ImmutableListContainerOptions ? options
+               : new ImmutableListContainerOptions(options);
+      String dir = options.getDir();
       if (dir == null) {
          prefixer = new PassThrough<String>();
          pathStripper = prefixer;

blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java

@@ -24,6 +24,7 @@ import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursi
 import javax.inject.Inject;
 
 import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.ContainerNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
@@ -172,15 +173,19 @@ public abstract class BaseBlobStore implements BlobStore {
     */
    @Override
    public void deleteContainer(final String container) {
-      deleteAndEnsurePathGone(container);
+      clearAndDeleteContainer(container);
    }
 
-   protected void deleteAndEnsurePathGone(final String container) {
+   protected void clearAndDeleteContainer(final String container) {
       try {
          if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
             public Boolean get() {
+               try {
                   clearContainer(container, recursive());
                   return deleteAndVerifyContainerGone(container);
+               } catch (ContainerNotFoundException e) {
+                  return true;
+               }
             }
 
          }, 30000)) {

blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java

@@ -21,17 +21,17 @@ package org.jclouds.blobstore.internal;
 import java.util.Collection;
 import java.util.Map;
 
-import javax.annotation.Nullable;
 import javax.inject.Inject;
 
 import org.jclouds.blobstore.BlobMap;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
 import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
 import org.jclouds.blobstore.strategy.PutBlobsStrategy;
-import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
+import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
 
 /**
  * Map representation of a live connection to a Blob Service.
@@ -46,9 +46,10 @@ public class BlobMapImpl extends BaseBlobMap<Blob> implements BlobMap {
    @Inject
    public BlobMapImpl(BlobStore blobstore, GetBlobsInListStrategy getAllBlobs,
             ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy,
-            ListBlobMetadataInContainer listStrategy, String containerName, @Nullable String dir) {
+            ListContainerAndRecurseThroughFolders listStrategy, String containerName,
+            ListContainerOptions options) {
       super(blobstore, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy,
-               containerName, dir);
+               containerName, options);
    }
 
    @Override

blobstore/src/main/java/org/jclouds/blobstore/internal/BlobStoreContextImpl.java

@@ -29,7 +29,7 @@ import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.InputStreamMap;
 import org.jclouds.blobstore.attr.ConsistencyModel;
 import org.jclouds.blobstore.attr.ConsistencyModels;
-import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl;
+import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.rest.RestContext;
 
 /**
@@ -67,18 +67,20 @@ public class BlobStoreContextImpl<X, Y> implements BlobStoreContext {
       this.blobStore = checkNotNull(blobStore, "blobStore");
    }
 
-   public BlobMap createBlobMap(String path) {
+   public BlobMap createBlobMap(String container, ListContainerOptions options) {
-      checkNotNull(path, "path");
+      return blobMapFactory.create(container, options);
-      String container = BlobStoreUtilsImpl.parseContainerFromPath(path);
-      String dir = BlobStoreUtilsImpl.parsePrefixFromPath(path);
-      return blobMapFactory.create(container, dir);
    }
 
-   public InputStreamMap createInputStreamMap(String path) {
+   public BlobMap createBlobMap(String container) {
-      checkNotNull(path, "path");
+      return blobMapFactory.create(container, ListContainerOptions.NONE);
-      String container = BlobStoreUtilsImpl.parseContainerFromPath(path);
+   }
-      String dir = BlobStoreUtilsImpl.parsePrefixFromPath(path);
+
-      return inputStreamMapFactory.create(container, dir);
+   public InputStreamMap createInputStreamMap(String container, ListContainerOptions options) {
+      return inputStreamMapFactory.create(container, options);
+   }
+
+   public InputStreamMap createInputStreamMap(String container) {
+      return inputStreamMapFactory.create(container, ListContainerOptions.NONE);
    }
 
    public BlobStore getBlobStore() {

blobstore/src/main/java/org/jclouds/blobstore/internal/InputStreamMapImpl.java

@@ -23,17 +23,17 @@ import java.io.InputStream;
 import java.util.Collection;
 import java.util.Map;
 
-import javax.annotation.Nullable;
 import javax.inject.Inject;
 
 import org.jclouds.blobstore.BlobMap;
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.InputStreamMap;
 import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
 import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
 import org.jclouds.blobstore.strategy.PutBlobsStrategy;
-import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
+import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
 import org.jclouds.http.Payload;
 import org.jclouds.http.Payloads;
 import org.jclouds.http.payloads.ByteArrayPayload;
@@ -60,11 +60,11 @@ public class InputStreamMapImpl extends BaseBlobMap<InputStream> implements Inpu
 
    @Inject
    public InputStreamMapImpl(BlobStore connection, Blob.Factory blobFactory,
-            GetBlobsInListStrategy getAllBlobs, ListBlobMetadataInContainer listStrategy,
+            GetBlobsInListStrategy getAllBlobs, ListContainerAndRecurseThroughFolders listStrategy,
             ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy,
-            String containerName, @Nullable String dir) {
+            String containerName, ListContainerOptions options) {
       super(connection, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy,
-               containerName, dir);
+               containerName, options);
    }
 
    @Override

blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java

@@ -99,6 +99,11 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
          return delegate.clone();
       }
 
+      @Override
+      public String toString() {
+         return delegate.toString();
+      }
+
    }
 
    public static final ImmutableListContainerOptions NONE = new ImmutableListContainerOptions(

blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java

@@ -68,6 +68,11 @@ public class ListOptions implements Cloneable {
       public ListOptions clone() {
          return delegate.clone();
       }
+
+      @Override
+      public String toString() {
+         return delegate.toString();
+      }
    }
 
    public static final ImmutableListOptions NONE = new ImmutableListOptions(new ListOptions());

blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java

@@ -20,7 +20,7 @@ package org.jclouds.blobstore.strategy;
 
 import org.jclouds.blobstore.domain.BlobMetadata;
 import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
+import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
 
 import com.google.inject.ImplementedBy;
 
@@ -29,8 +29,8 @@ import com.google.inject.ImplementedBy;
  * 
  * @author Adrian Cole
  */
-@ImplementedBy(ListBlobMetadataInContainer.class)
+@ImplementedBy(ListContainerAndRecurseThroughFolders.class)
-public interface ListBlobMetadataStrategy {
+public interface ListBlobsInContainer {
 
    Iterable<? extends BlobMetadata> execute(String containerName, ListContainerOptions options);
 

blobstore/src/main/java/org/jclouds/blobstore/strategy/ListContainerStrategy.java

@@ -20,7 +20,7 @@ package org.jclouds.blobstore.strategy;
 
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.internal.ListAllMetadataInContainer;
+import org.jclouds.blobstore.strategy.internal.ConcatenateContainerLists;
 
 import com.google.inject.ImplementedBy;
 
@@ -29,8 +29,8 @@ import com.google.inject.ImplementedBy;
  * 
  * @author Adrian Cole
  */
-@ImplementedBy(ListAllMetadataInContainer.class)
+@ImplementedBy(ConcatenateContainerLists.class)
-public interface ListMetadataStrategy {
+public interface ListContainerStrategy {
 
    Iterable<? extends StorageMetadata> execute(String containerName, ListContainerOptions options);
 

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ConcatenateContainerLists.java

@@ -28,7 +28,7 @@ import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.internal.BlobRuntimeException;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.options.ListContainerOptions.ImmutableListContainerOptions;
-import org.jclouds.blobstore.strategy.ListMetadataStrategy;
+import org.jclouds.blobstore.strategy.ListContainerStrategy;
 
 import com.google.common.base.Throwables;
 import com.google.common.collect.Iterables;
@@ -41,12 +41,12 @@ import com.google.inject.Inject;
  * @author Adrian Cole
  */
 @Singleton
-public class ListAllMetadataInContainer implements ListMetadataStrategy {
+public class ConcatenateContainerLists implements ListContainerStrategy {
 
    protected final BlobStore connection;
 
    @Inject
-   public ListAllMetadataInContainer(BlobStore connection) {
+   public ConcatenateContainerLists(BlobStore connection) {
       this.connection = connection;
    }
 

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/CountBlobTypeInList.java

@@ -23,7 +23,7 @@ import javax.inject.Singleton;
 
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.strategy.CountListStrategy;
-import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
+import org.jclouds.blobstore.strategy.ListBlobsInContainer;
 
 import com.google.common.collect.Iterables;
 
@@ -34,10 +34,10 @@ import com.google.common.collect.Iterables;
  */
 @Singleton
 public class CountBlobTypeInList implements CountListStrategy {
-   protected final ListBlobMetadataStrategy getAllBlobMetadata;
+   protected final ListBlobsInContainer getAllBlobMetadata;
 
    @Inject
-   CountBlobTypeInList(ListBlobMetadataStrategy getAllBlobMetadata) {
+   CountBlobTypeInList(ListBlobsInContainer getAllBlobMetadata) {
       this.getAllBlobMetadata = getAllBlobMetadata;
    }
 

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java

@@ -36,6 +36,7 @@ import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.reference.BlobStoreConstants;
 import org.jclouds.blobstore.strategy.ClearContainerStrategy;
 import org.jclouds.blobstore.strategy.ClearListStrategy;
+import org.jclouds.blobstore.strategy.ListContainerStrategy;
 import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
 import org.jclouds.logging.Logger;
 
@@ -56,7 +57,7 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
    @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
    protected Logger logger = Logger.NULL;
 
-   protected final ListAllMetadataInContainer getAllMetadata;
+   protected final ListContainerStrategy listContainer;
    protected final BackoffLimitedRetryHandler retryHandler;
    private final ExecutorService userExecutor;
 
@@ -70,12 +71,12 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
 
    @Inject
    DeleteAllKeysInList(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
-            AsyncBlobStore connection, ListAllMetadataInContainer getAllMetadata,
+            AsyncBlobStore connection, ListContainerStrategy listContainer,
             BackoffLimitedRetryHandler retryHandler) {
 
       this.userExecutor = userExecutor;
       this.connection = connection;
-      this.getAllMetadata = getAllMetadata;
+      this.listContainer = listContainer;
       this.retryHandler = retryHandler;
    }
 
@@ -84,23 +85,34 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
    }
 
    public void execute(final String containerName, final ListContainerOptions options) {
-      String message = options.getDir() != null ? String.format("deleting from path: %s/%s",
+      String message = options.getDir() != null ? String.format("clearing path %s/%s",
-               containerName, options.getDir()) : String.format("deleting from containerName: %s",
+               containerName, options.getDir()) : String.format("clearing container %s",
                containerName);
+      if (options.isRecursive())
+         message = message + " recursively";
       Map<StorageMetadata, Exception> exceptions = Maps.newHashMap();
       Iterable<? extends StorageMetadata> toDelete = getResourcesToDelete(containerName, options);
       for (int i = 0; i < 3; i++) { // TODO parameterize
          Map<StorageMetadata, ListenableFuture<?>> responses = Maps.newHashMap();
          try {
-            for (StorageMetadata md : toDelete) {
+            for (final StorageMetadata md : toDelete) {
+               String fullPath = parentIsFolder(options, md) ? options.getDir() + "/"
+                        + md.getName() : md.getName();
                switch (md.getType()) {
                   case BLOB:
-                     responses.put(md, connection.removeBlob(containerName, md.getName()));
+                     responses.put(md, connection.removeBlob(containerName, fullPath));
                      break;
                   case FOLDER:
+                     if (options.isRecursive() && !fullPath.equals(options.getDir())) {
+                        execute(containerName, options.clone().inDirectory(fullPath));
+                     }
+                     connection.deleteDirectory(containerName, fullPath);
+                     break;
                   case RELATIVE_PATH:
-                     if (options.isRecursive())
+                     if (options.isRecursive() && !fullPath.equals(options.getDir())) {
-                        responses.put(md, connection.deleteDirectory(containerName, md.getName()));
+                        execute(containerName, options.clone().inDirectory(fullPath));
+                     }
+                     connection.deleteDirectory(containerName, md.getName());
                      break;
                   case CONTAINER:
                      throw new IllegalArgumentException("Container type not supported");
@@ -113,17 +125,24 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
                break;
             }
             if (exceptions.size() > 0) {
+               toDelete = Iterables.concat(exceptions.keySet(), toDelete);
                retryHandler.imposeBackoffExponentialDelay(i + 1, message);
             }
          }
       }
       if (exceptions.size() > 0)
          throw new BlobRuntimeException(String.format("error %s: %s", message, exceptions));
+      assert Iterables.size(toDelete) == 0 : String.format("items remaining %s: %s", message,
+               toDelete);
+   }
+
+   private boolean parentIsFolder(final ListContainerOptions options, final StorageMetadata md) {
+      return (options.getDir() != null && md.getName().indexOf('/') == -1);
    }
 
    private Iterable<? extends StorageMetadata> getResourcesToDelete(final String containerName,
             final ListContainerOptions options) {
-      Iterable<? extends StorageMetadata> toDelete = Iterables.filter(getAllMetadata.execute(
+      Iterable<? extends StorageMetadata> toDelete = Iterables.filter(listContainer.execute(
                containerName, options), new Predicate<StorageMetadata>() {
 
          @Override

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FindMD5InList.java

@@ -28,7 +28,7 @@ import org.jclouds.blobstore.functions.ObjectMD5;
 import org.jclouds.blobstore.internal.BlobRuntimeException;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
-import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
+import org.jclouds.blobstore.strategy.ListBlobsInContainer;
 
 import com.google.common.base.Throwables;
 
@@ -41,10 +41,10 @@ import com.google.common.base.Throwables;
 public class FindMD5InList implements ContainsValueInListStrategy {
 
    protected final ObjectMD5 objectMD5;
-   protected final ListBlobMetadataStrategy getAllBlobMetadata;
+   protected final ListBlobsInContainer getAllBlobMetadata;
 
    @Inject
-   private FindMD5InList(ObjectMD5 objectMD5, ListBlobMetadataStrategy getAllBlobMetadata) {
+   private FindMD5InList(ObjectMD5 objectMD5, ListBlobsInContainer getAllBlobMetadata) {
       this.objectMD5 = objectMD5;
       this.getAllBlobMetadata = getAllBlobMetadata;
    }

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java

@@ -37,7 +37,7 @@ import org.jclouds.blobstore.internal.BlobRuntimeException;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.reference.BlobStoreConstants;
 import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
-import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
+import org.jclouds.blobstore.strategy.ListBlobsInContainer;
 import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
 import org.jclouds.logging.Logger;
 
@@ -57,7 +57,7 @@ import com.google.inject.Inject;
 @Singleton
 public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrategy {
 
-   protected final ListBlobMetadataStrategy getAllBlobMetadata;
+   protected final ListBlobsInContainer getAllBlobMetadata;
    protected final BackoffLimitedRetryHandler retryHandler;
    protected final AsyncBlobStore ablobstore;
    protected final ExecutorService userExecutor;
@@ -74,7 +74,7 @@ public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrateg
    @Inject
    GetAllBlobsInListAndRetryOnFailure(
             @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
-            ListBlobMetadataStrategy getAllBlobMetadata, AsyncBlobStore ablobstore,
+            ListBlobsInContainer getAllBlobMetadata, AsyncBlobStore ablobstore,
             BackoffLimitedRetryHandler retryHandler) {
       this.userExecutor = userExecutor;
       this.ablobstore = ablobstore;

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListBlobMetadataInContainer.java

@@ -1,66 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
- *
- * ====================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ====================================================================
- */
-package org.jclouds.blobstore.strategy.internal;
-
-import javax.inject.Singleton;
-
-import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.StorageType;
-import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
-import org.jclouds.blobstore.strategy.ListMetadataStrategy;
-
-import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-import com.google.inject.Inject;
-
-/**
- * Retrieves all blobs in the blobstore by the most efficient means possible.
- * 
- * @author Adrian Cole
- */
-@Singleton
-public class ListBlobMetadataInContainer implements ListBlobMetadataStrategy {
-
-   protected final ListMetadataStrategy lister;
-
-   @Inject
-   ListBlobMetadataInContainer(ListMetadataStrategy lister) {
-      this.lister = lister;
-   }
-
-   @Override
-   public Iterable<? extends BlobMetadata> execute(String container, ListContainerOptions options) {
-      return Iterables.transform(Iterables.filter(lister.execute(container, options),
-               new Predicate<StorageMetadata>() {
-                  @Override
-                  public boolean apply(StorageMetadata input) {
-                     return input.getType() == StorageType.BLOB;
-                  }
-
-               }), new Function<StorageMetadata, BlobMetadata>() {
-         @Override
-         public BlobMetadata apply(StorageMetadata from) {
-            return (BlobMetadata) from;
-         }
-      });
-   }
-}

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java

@@ -0,0 +1,83 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.blobstore.strategy.internal;
+
+import java.util.List;
+
+import javax.inject.Singleton;
+
+import org.jclouds.blobstore.domain.BlobMetadata;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.domain.StorageType;
+import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.strategy.ListBlobsInContainer;
+import org.jclouds.blobstore.strategy.ListContainerStrategy;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.inject.Inject;
+
+/**
+ * Retrieves all blobs in the blobstore by the most efficient means possible.
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class ListContainerAndRecurseThroughFolders implements ListBlobsInContainer {
+
+   protected final ListContainerStrategy lister;
+
+   @Inject
+   ListContainerAndRecurseThroughFolders(ListContainerStrategy lister) {
+      this.lister = lister;
+   }
+
+   @Override
+   public Iterable<? extends BlobMetadata> execute(final String containerName,
+            final ListContainerOptions options) {
+      final List<Iterable<? extends BlobMetadata>> lists = Lists.newArrayList();
+      Iterable<? extends StorageMetadata> pwdList = lister.execute(containerName, options);
+      for (StorageMetadata md : Iterables.filter(pwdList, new Predicate<StorageMetadata>() {
+         @Override
+         public boolean apply(StorageMetadata input) {
+            return (input.getType() == StorageType.FOLDER || input.getType() == StorageType.RELATIVE_PATH)
+                     && options.isRecursive();
+         }
+      })) {
+         String directory = (options.getDir() != null) ? options.getDir() + "/" + md.getName() : md
+                  .getName();
+         lists.add(execute(containerName, options.clone().inDirectory(directory)));
+      }
+      lists.add(Iterables.transform(Iterables.filter(pwdList, new Predicate<StorageMetadata>() {
+         @Override
+         public boolean apply(StorageMetadata input) {
+            return input.getType() == StorageType.BLOB;
+         }
+      }), new Function<StorageMetadata, BlobMetadata>() {
+         @Override
+         public BlobMetadata apply(StorageMetadata from) {
+            return (BlobMetadata) from;
+         }
+      }));
+      return Sets.newHashSet(Iterables.concat(lists));
+   }
+}

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java

@@ -30,6 +30,7 @@ import javax.inject.Singleton;
 
 import org.jclouds.Constants;
 import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.internal.BlobRuntimeException;
 import org.jclouds.blobstore.reference.BlobStoreConstants;
 import org.jclouds.blobstore.strategy.DeleteDirectoryStrategy;
@@ -64,6 +65,7 @@ import com.google.inject.Inject;
 public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy {
 
    private final AsyncBlobStore ablobstore;
+   private final BlobStore blobstore;
    private final ExecutorService userExecutor;
    @Resource
    @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
@@ -78,9 +80,10 @@ public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy {
    @Inject
    MarkersDeleteDirectoryStrategy(
             @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
-            AsyncBlobStore ablobstore) {
+            AsyncBlobStore ablobstore, BlobStore blobstore) {
       this.userExecutor = userExecutor;
       this.ablobstore = ablobstore;
+      this.blobstore = blobstore;
    }
 
    public void execute(String containerName, String directory) {
@@ -93,10 +96,13 @@ public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy {
       for (String name : names) {
          responses.put(name, ablobstore.removeBlob(containerName, name));
       }
+      String message = String.format("deleting directory %s in containerName: %s", directory,
+               containerName);
       Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
-               String.format("deleting directory in containerName: %s", containerName));
+               message);
       if (exceptions.size() > 0)
-         throw new BlobRuntimeException(String.format("error deleting from container %s: %s",
+         throw new BlobRuntimeException(String.format("error %s: %s", message, exceptions));
-                  containerName, exceptions));
+      assert !blobstore.directoryExists(containerName, directory) : String.format(
+               "still exists %s: %s", message, exceptions);
    }
 }

blobstore/src/test/java/org/jclouds/blobstore/integration/StubBlobMapIntegrationTest.java

@@ -28,9 +28,4 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "blobstore.StubBlobMapIntegrationTest")
 public class StubBlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
 
-   @Override
-   protected int maxList() {
-      return 1;
-   }
-
 }

blobstore/src/test/java/org/jclouds/blobstore/integration/StubInputStreamMapIntegrationTest.java

@@ -27,9 +27,4 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "blobstore.StubInputStreamMapIntegrationTest")
 public class StubInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
 
-   @Override
-   protected int maxList() {
-      return 1;
-   }
-
 }

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobMapIntegrationTest.java

@@ -18,6 +18,7 @@
  */
 package org.jclouds.blobstore.integration.internal;
 
+import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults;
 import static org.testng.Assert.assertEquals;
 
 import java.io.IOException;
@@ -30,8 +31,10 @@ import java.util.Map.Entry;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
 
+import org.jclouds.blobstore.BlobMap;
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl;
 import org.jclouds.util.Utils;
 import org.testng.annotations.Test;
@@ -204,18 +207,16 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
       }
    }
 
-   protected abstract int maxList();
+   @Test(groups = { "integration", "live" })
-
-   @Test(enabled = false, groups = { "integration", "live" })
    public void testPutMoreThanSingleListing() throws InterruptedException, ExecutionException,
             TimeoutException {
-      if (maxList() == 0)
+      if (maxResultsForTestListings() == 0)
          return;
       String bucketName = getContainerName();
       try {
          Map<String, Blob> map = createMap(context, bucketName);
          Set<String> keySet = Sets.newHashSet();
-         for (int i = 0; i < maxList() + 1; i++) {
+         for (int i = 0; i < maxResultsForTestListings() + 1; i++) {
             keySet.add(i + "");
          }
 
@@ -228,7 +229,7 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
          map.putAll(newMap);
          newMap.clear();
 
-         assertConsistencyAwareMapSize(map, maxList() + 1);
+         assertConsistencyAwareMapSize(map, maxResultsForTestListings() + 1);
          assertConsistencyAwareKeySetEquals(map, keySet);
          map.clear();
          assertConsistencyAwareMapSize(map, 0);
@@ -265,8 +266,15 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
       map.putAll(newMap);
    }
 
-   protected Map<String, Blob> createMap(BlobStoreContext context, String bucket) {
+   protected int maxResultsForTestListings() {
-      return context.createBlobMap(bucket);
+      return 100;
    }
 
+   protected BlobMap createMap(BlobStoreContext context, String bucket) {
+      return createMap(context, bucket, maxResults(maxResultsForTestListings()));
+   }
+
+   protected BlobMap createMap(BlobStoreContext context, String bucket, ListContainerOptions options) {
+      return context.createBlobMap(bucket, options);
+   }
 }

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java

@@ -207,8 +207,8 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
          addAlphabetUnderRoot(containerName);
          PageSet<? extends StorageMetadata> container = context.getBlobStore().list(containerName,
                   maxResults(5));
-         assert container.getNextMarker() != null;
          assertEquals(container.size(), 5);
+         assert container.getNextMarker() != null;
       } finally {
          returnContainer(containerName);
       }

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseInputStreamMapIntegrationTest.java

@@ -18,6 +18,7 @@
  */
 package org.jclouds.blobstore.integration.internal;
 
+import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults;
 import static org.testng.Assert.assertEquals;
 
 import java.io.IOException;
@@ -34,6 +35,7 @@ import java.util.concurrent.TimeoutException;
 
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.InputStreamMap;
+import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.util.Utils;
 import org.testng.annotations.Test;
 
@@ -49,9 +51,9 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
    @Override
    @Test(groups = { "integration", "live" })
    public void testValues() throws InterruptedException, IOException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
          map.putAll(this.fiveInputs);
          // this will cause us to block until the bucket updates.
          assertConsistencyAwareMapSize(map, 5);
@@ -65,20 +67,18 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
          assert valuesAsString.size() == 0 : valuesAsString.size() + ": " + values + ": "
                   + valuesAsString;
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
-   protected abstract int maxList();
+   @Test(groups = { "integration", "live" })
-
-   @Test(enabled = false, groups = { "integration", "live" })
    public void testPutMoreThanSingleListing() throws InterruptedException, ExecutionException,
             TimeoutException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         InputStreamMap map = createMap(context, bucketName);
+         InputStreamMap map = createMap(context, containerName);
          Set<String> keySet = Sets.newHashSet();
-         for (int i = 0; i < maxList() + 1; i++) {
+         for (int i = 0; i < maxResultsForTestListings() + 1; i++) {
             keySet.add(i + "");
          }
 
@@ -89,20 +89,20 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
          map.putAllStrings(newMap);
          newMap.clear();
 
-         assertConsistencyAwareMapSize(map, maxList() + 1);
+         assertConsistencyAwareMapSize(map, maxResultsForTestListings() + 1);
          assertConsistencyAwareKeySetEquals(map, keySet);
          map.clear();
          assertConsistencyAwareMapSize(map, 0);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testRemove() throws InterruptedException, IOException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
          putStringWithMD5(map, "one", "two");
          InputStream old = map.remove("one");
          assertEquals(Utils.toStringAndClose(old), "two");
@@ -113,16 +113,16 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
          assert old == null;
          assertConsistencyAwareKeySize(map, 0);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Override
    @Test(groups = { "integration", "live" })
    public void testEntrySet() throws InterruptedException, IOException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
          ((InputStreamMap) map).putAllStrings(this.fiveStrings);
          // this will cause us to block until the bucket updates.
          assertConsistencyAwareKeySize(map, 5);
@@ -137,134 +137,134 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
             assertEquals(Utils.toStringAndClose(value), "");
          }
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testContainsStringValue() throws InterruptedException, ExecutionException,
             TimeoutException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
          ((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple"));
          assertConsistencyAwareContainsValue(map, fiveStrings.get("one"));
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testContainsFileValue() throws InterruptedException, ExecutionException,
             TimeoutException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
          ((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple"));
          assertConsistencyAwareContainsValue(map, fiveFiles.get("one"));
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testContainsInputStreamValue() throws InterruptedException, ExecutionException,
             TimeoutException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
          ((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple"));
          assertConsistencyAwareContainsValue(map, this.fiveInputs.get("one"));
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testContainsBytesValue() throws InterruptedException, ExecutionException,
             TimeoutException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
          ((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple"));
          assertConsistencyAwareContainsValue(map, this.fiveBytes.get("one"));
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Override
    @Test(groups = { "integration", "live" })
    public void testPutAll() throws InterruptedException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
          map.putAll(this.fiveInputs);
          assertConsistencyAwareMapSize(map, 5);
          assertConsistencyAwareKeySetEquals(map, new TreeSet<String>(fiveInputs.keySet()));
          fourLeftRemovingOne(map);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testPutAllBytes() throws InterruptedException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
 
          ((InputStreamMap) map).putAllBytes(this.fiveBytes);
          assertConsistencyAwareMapSize(map, 5);
          assertConsistencyAwareKeySetEquals(map, new TreeSet<String>(fiveBytes.keySet()));
          fourLeftRemovingOne(map);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testPutAllFiles() throws InterruptedException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
 
          ((InputStreamMap) map).putAllFiles(this.fiveFiles);
          assertConsistencyAwareMapSize(map, 5);
          assertConsistencyAwareKeySetEquals(map, new TreeSet<String>(fiveFiles.keySet()));
          fourLeftRemovingOne(map);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testPutAllStrings() throws InterruptedException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
 
          ((InputStreamMap) map).putAllStrings(this.fiveStrings);
          assertConsistencyAwareMapSize(map, 5);
          assertConsistencyAwareKeySetEquals(map, new TreeSet<String>(fiveStrings.keySet()));
          fourLeftRemovingOne(map);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testPutString() throws InterruptedException, IOException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
 
          InputStream old = ((InputStreamMap) map).putString("one", fiveStrings.get("one"));
          getOneReturnsAppleAndOldValueIsNull(map, old);
          InputStream apple = ((InputStreamMap) map).putString("one", fiveStrings.get("two"));
          getOneReturnsBearAndOldValueIsApple(map, apple);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
@@ -285,46 +285,46 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
 
    @Test(groups = { "integration", "live" })
    public void testPutFile() throws IOException, InterruptedException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
 
          InputStream old = ((InputStreamMap) map).putFile("one", fiveFiles.get("one"));
          getOneReturnsAppleAndOldValueIsNull(map, old);
          InputStream apple = ((InputStreamMap) map).putFile("one", fiveFiles.get("two"));
          getOneReturnsBearAndOldValueIsApple(map, apple);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testPutBytes() throws InterruptedException, IOException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
 
          InputStream old = ((InputStreamMap) map).putBytes("one", fiveBytes.get("one"));
          getOneReturnsAppleAndOldValueIsNull(map, old);
          InputStream apple = ((InputStreamMap) map).putBytes("one", fiveBytes.get("two"));
          getOneReturnsBearAndOldValueIsApple(map, apple);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
    @Test(groups = { "integration", "live" })
    public void testPut() throws InterruptedException, IOException {
-      String bucketName = getContainerName();
+      String containerName = getContainerName();
       try {
-         Map<String, InputStream> map = createMap(context, bucketName);
+         Map<String, InputStream> map = createMap(context, containerName);
 
          InputStream old = map.put("one", fiveInputs.get("one"));
          getOneReturnsAppleAndOldValueIsNull(map, old);
          InputStream apple = map.put("one", fiveInputs.get("two"));
          getOneReturnsBearAndOldValueIsApple(map, apple);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerName);
       }
    }
 
@@ -334,8 +334,16 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
       ((InputStreamMap) map).putString(key, value);
    }
 
+   protected int maxResultsForTestListings() {
+      return 100;
+   }
+
    protected InputStreamMap createMap(BlobStoreContext context, String bucket) {
-      InputStreamMap map = context.createInputStreamMap(bucket);
+      return createMap(context, bucket, maxResults(maxResultsForTestListings()));
-      return map;
+   }
+
+   protected InputStreamMap createMap(BlobStoreContext context, String bucket,
+            ListContainerOptions options) {
+      return context.createInputStreamMap(bucket, options);
    }
 }

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.blobstore.integration.internal;
 
+import static org.jclouds.blobstore.options.ListContainerOptions.Builder.inDirectory;
+import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 
@@ -25,6 +27,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
@@ -34,6 +37,8 @@ import java.util.concurrent.TimeoutException;
 
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.ListableMap;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.util.Utils;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
@@ -97,20 +102,23 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
       }
    }
 
-   protected abstract Map<String, V> createMap(BlobStoreContext context, String bucket);
+   protected abstract Map<String, V> createMap(BlobStoreContext context, String containerName);
+
+   protected abstract Map<String, V> createMap(BlobStoreContext context, String containerName,
+            ListContainerOptions options);
 
    @Test(groups = { "integration", "live" })
    public void testClear() throws InterruptedException, ExecutionException, TimeoutException {
-      String bucketName = getContainerName();
+      String containerNameName = getContainerName();
       try {
-         Map<String, V> map = createMap(context, bucketName);
+         Map<String, V> map = createMap(context, containerNameName);
          assertConsistencyAwareMapSize(map, 0);
          putStringWithMD5(map, "one", "apple");
          assertConsistencyAwareMapSize(map, 1);
          map.clear();
          assertConsistencyAwareMapSize(map, 0);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerNameName);
       }
    }
 
@@ -120,18 +128,84 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
 
    @Test(groups = { "integration", "live" })
    public void testKeySet() throws InterruptedException, ExecutionException, TimeoutException {
-      String bucketName = getContainerName();
+      String containerNameName = getContainerName();
       try {
-         Map<String, V> map = createMap(context, bucketName);
+         Map<String, V> map = createMap(context, containerNameName);
          assertConsistencyAwareKeySize(map, 0);
          putStringWithMD5(map, "one", "two");
          assertConsistencyAwareKeySize(map, 1);
          assertConsistencyAwareKeySetEquals(map, ImmutableSet.of("one"));
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerNameName);
       }
    }
 
+   protected void addTenObjectsUnderPrefix(String containerName, String prefix)
+            throws InterruptedException {
+      for (int i = 0; i < 10; i++) {
+         Blob blob = context.getBlobStore().newBlob(prefix + "/" + i);
+         blob.setPayload(i + "content");
+         context.getBlobStore().putBlob(containerName, blob);
+      }
+   }
+
+   protected void addTenObjectsUnderRoot(String containerName) throws InterruptedException {
+      for (int i = 0; i < 10; i++) {
+         Blob blob = context.getBlobStore().newBlob(i + "");
+         blob.setPayload(i + "content");
+         context.getBlobStore().putBlob(containerName, blob);
+      }
+   }
+
+   @Test(groups = { "integration", "live" })
+   public void testDirectory() throws InterruptedException, UnsupportedEncodingException {
+      String containerName = getContainerName();
+      try {
+         String directory = "apps";
+
+         Map<String, V> rootMap = createMap(context, containerName);
+         Map<String, V> rootRecursiveMap = createMap(context, containerName, recursive());
+         Map<String, V> inDirectoryMap = createMap(context, containerName, inDirectory(directory));
+         Map<String, V> inDirectoryRecursiveMap = createMap(context, containerName, inDirectory(
+                  directory).recursive());
+
+         context.getBlobStore().createDirectory(containerName, directory);
+         addTenObjectsUnderRoot(containerName);
+         assertEquals(rootMap.size(), 10);
+         assertEquals(rootRecursiveMap.size(), 10);
+         assertEquals(inDirectoryMap.size(), 0);
+         assertEquals(inDirectoryRecursiveMap.size(), 0);
+
+         addTenObjectsUnderPrefix(containerName, directory);
+         assertEquals(rootMap.size(), 10);
+         assertEquals(rootRecursiveMap.size(), 20);
+         assertEquals(inDirectoryMap.size(), 10);
+         assertEquals(inDirectoryRecursiveMap.size(), 10);
+
+         context.getBlobStore().createDirectory(containerName, directory + "/" + directory);
+         assertEquals(rootMap.size(), 10);
+         assertEquals(rootRecursiveMap.size(), 20);
+         assertEquals(inDirectoryMap.size(), 10);
+         assertEquals(inDirectoryRecursiveMap.size(), 10);
+
+         rootMap.clear();
+         assertEquals(rootMap.size(), 0);
+         assertEquals(rootRecursiveMap.size(), 10);
+         assertEquals(inDirectoryMap.size(), 10);
+         assertEquals(inDirectoryRecursiveMap.size(), 10);
+
+         inDirectoryMap.clear();
+         assertEquals(rootMap.size(), 0);
+         assertEquals(rootRecursiveMap.size(), 0);
+         assertEquals(inDirectoryMap.size(), 0);
+         assertEquals(inDirectoryRecursiveMap.size(), 0);
+
+      } finally {
+         returnContainer(containerName);
+      }
+
+   }
+
    protected void assertConsistencyAwareKeySetEquals(final Map<String, V> map,
             final Set<String> expected) throws InterruptedException {
       assertConsistencyAware(new Runnable() {
@@ -177,20 +251,20 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
 
    @Test(groups = { "integration", "live" })
    public void testContainsKey() throws InterruptedException, ExecutionException, TimeoutException {
-      String bucketName = getContainerName();
+      String containerNameName = getContainerName();
       try {
-         Map<String, V> map = createMap(context, bucketName);
+         Map<String, V> map = createMap(context, containerNameName);
          assertConsistencyAwareDoesntContainKey(map);
          putStringWithMD5(map, "one", "apple");
          assertConsistencyAwareContainsKey(map);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerNameName);
       }
    }
 
    /**
-    * containsValue() uses eTag comparison to bucket contents, so this can be subject to eventual
+    * containsValue() uses eTag comparison to containerName contents, so this can be subject to
-    * consistency problems.
+    * eventual consistency problems.
     */
    protected void assertConsistencyAwareContainsValue(final Map<String, V> map, final Object value)
             throws InterruptedException {
@@ -221,14 +295,14 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
 
    @Test(groups = { "integration", "live" })
    public void testIsEmpty() throws InterruptedException, ExecutionException, TimeoutException {
-      String bucketName = getContainerName();
+      String containerNameName = getContainerName();
       try {
-         Map<String, V> map = createMap(context, bucketName);
+         Map<String, V> map = createMap(context, containerNameName);
          assertConsistencyAwareEmpty(map);
          putStringWithMD5(map, "one", "apple");
          assertConsistencyAwareNotEmpty(map);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerNameName);
       }
    }
 
@@ -275,17 +349,17 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
    @Test(groups = { "integration", "live" })
    public void testListContainer() throws InterruptedException, ExecutionException,
             TimeoutException {
-      String bucketName = getContainerName();
+      String containerNameName = getContainerName();
       try {
-         ListableMap<?, ?> map = (ListableMap<?, ?>) createMap(context, bucketName);
+         ListableMap<?, ?> map = (ListableMap<?, ?>) createMap(context, containerNameName);
-         assertConsistencyAwareListContainer(map, bucketName);
+         assertConsistencyAwareListContainer(map, containerNameName);
       } finally {
-         returnContainer(bucketName);
+         returnContainer(containerNameName);
       }
    }
 
    protected void assertConsistencyAwareListContainer(final ListableMap<?, ?> map,
-            final String bucketName) throws InterruptedException {
+            final String containerNameName) throws InterruptedException {
       assertConsistencyAware(new Runnable() {
          public void run() {
             assertTrue(Iterables.size(map.list()) >= 0);

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/StubAsyncBlobStore.java

@@ -19,6 +19,7 @@
 package org.jclouds.blobstore.integration.internal;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
 import static com.google.common.util.concurrent.Futures.immediateFuture;
 
@@ -157,7 +158,12 @@ public class StubAsyncBlobStore extends BaseAsyncBlobStore {
       SortedSet<StorageMetadata> contents = Sets.newTreeSet(Iterables.transform(realContents
                .keySet(), new Function<String, StorageMetadata>() {
          public StorageMetadata apply(String key) {
-            MutableBlobMetadata md = copy(realContents.get(key).getMetadata());
+            Blob oldBlob = realContents.get(key);
+            checkState(oldBlob != null, "blob " + key
+                     + " is not present although it was in the list of " + name);
+            checkState(oldBlob.getMetadata() != null, "blob " + name + "/" + key
+                     + " has no metadata");
+            MutableBlobMetadata md = copy(oldBlob.getMetadata());
             String directoryName = ifDirectoryReturnName.execute(md);
             if (directoryName != null) {
                md.setName(directoryName);
@@ -242,7 +248,9 @@ public class StubAsyncBlobStore extends BaseAsyncBlobStore {
          convertUserMetadataKeysToLowercase(metadata);
          return metadata;
       } catch (Exception e) {
-         throw new RuntimeException(e);
+         Throwables.propagate(e);
+         assert false : "exception should have propagated: " + e;
+         return null;
       }
    }
 

core/src/main/java/org/jclouds/http/HttpRequest.java

@@ -103,9 +103,6 @@ public class HttpRequest extends HttpMessage {
       return payload;
    }
 
-   /**
-    * {@inheritDoc}
-    */
    public void setPayload(Payload data) {
       closeContentIfPresent();
       this.payload = checkNotNull(data, "data");

core/src/main/java/org/jclouds/rest/functions/ReturnFalseOnResourceNotFound.java

@@ -0,0 +1,47 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.rest.functions;
+
+import static org.jclouds.util.Utils.propagateOrNull;
+
+import java.util.List;
+
+import javax.inject.Singleton;
+
+import org.jclouds.rest.ResourceNotFoundException;
+
+import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class ReturnFalseOnResourceNotFound implements Function<Exception, Boolean> {
+
+   public Boolean apply(Exception from) {
+      List<Throwable> throwables = Throwables.getCausalChain(from);
+      if (Iterables.size(Iterables.filter(throwables, ResourceNotFoundException.class)) >= 1) {
+         return false;
+      }
+      return Boolean.class.cast(propagateOrNull(from));
+   }
+}

core/src/main/java/org/jclouds/rest/functions/ReturnNullOnResourceNotFound.java

@@ -0,0 +1,42 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.rest.functions;
+
+import static org.jclouds.util.Utils.propagateOrNull;
+
+import javax.inject.Singleton;
+
+import org.jclouds.rest.ResourceNotFoundException;
+
+import com.google.common.base.Function;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class ReturnNullOnResourceNotFound implements Function<Exception, Object> {
+
+   public Object apply(Exception from) {
+      if (from instanceof ResourceNotFoundException) {
+         return null;
+      }
+      return propagateOrNull(from);
+   }
+}

core/src/main/java/org/jclouds/util/Jsr330.java

@@ -31,10 +31,9 @@ import com.google.inject.Key;
  * Utility methods for use with {@code @}{@link Named}.
  * 
  * @author crazybob@google.com (Bob Lee) - original code taken from
- *         {@link com.google.inject.name.Names}
+ *         {@code com.google.inject.name.Names}
  * 
  * @see com.google.inject.util.Jsr330#named
- * @see com.google.inject.name.Names
  * @author Adrian Cole
  */
 public class Jsr330 {

core/src/main/java/org/jclouds/util/Utils.java

@@ -85,12 +85,8 @@ public class Utils {
    }
 
    /**
-    * Returns a factory that will supply instances of {@link OutputStream} that read from the given
+    * converts an {@link OutputStream} to an {@link OutputSupplier}
-    * outputStream.
     * 
-    * @param url
-    *           the URL to read from
-    * @return the factory
     */
    public static OutputSupplier<OutputStream> newOutputStreamSupplier(final OutputStream output) {
       checkNotNull(output, "output");
@@ -146,9 +142,6 @@ public class Utils {
     * {@link UnsupportedEncodingException}, log a warning and fall back to the system's default
     * encoding.
     * 
-    * @see {@link String#getBytes(String)}
-    * @see {@link String#getBytes()} - used as fall-back.
-    * 
     * @param str
     *           what to encode
     * @param charsetName

rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/ContainerToResourceList.java

@@ -24,7 +24,9 @@ import javax.inject.Singleton;
 import org.jclouds.blobstore.domain.BlobMetadata;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.blobstore.domain.StorageType;
 import org.jclouds.blobstore.domain.internal.PageSetImpl;
+import org.jclouds.blobstore.domain.internal.StorageMetadataImpl;
 import org.jclouds.rackspace.cloudfiles.domain.ObjectInfo;
 
 import com.google.common.base.Function;
@@ -46,8 +48,13 @@ public class ContainerToResourceList implements
    public PageSet<? extends StorageMetadata> apply(PageSet<ObjectInfo> from) {
       return new PageSetImpl<StorageMetadata>(Iterables.transform(Iterables.transform(from,
                object2blobMd), new Function<BlobMetadata, StorageMetadata>() {
-         public StorageMetadata apply(BlobMetadata arg0) {
+         public StorageMetadata apply(BlobMetadata input) {
-            return arg0;
+            if (input.getContentType().equals("application/directory")) {
+               return new StorageMetadataImpl(StorageType.RELATIVE_PATH, input.getId(), input
+                        .getName(), input.getLocationId(), input.getUri(), input.getETag(), input
+                        .getSize(), input.getLastModified(), input.getUserMetadata());
+            }
+            return input;
          }
       }), from.getNextMarker());
 

rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationTest.java

@@ -18,10 +18,7 @@
  */
 package org.jclouds.rackspace.cloudfiles.blobstore.integration;
 
-import java.io.IOException;
-
 import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
-import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 /**
@@ -38,17 +35,4 @@ public class CloudFilesBlobIntegrationTest extends BaseBlobIntegrationTest {
       // not supported in cloud files
    }
 
-   // TODO this should work, not sure why my encoding is set to Mac Roman and not UTF-8
-   @DataProvider(name = "delete")
-   public Object[][] createData() {
-      return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" },
-               { "path/foo" }, { "colon:" }, { "asteri*k" }, { "quote\"" }, { "p|pe" } };
-   }
-
-   @Override
-   @Test(enabled = false)
-   public void testPutObject(String key, String type, Object content, Object realObject)
-            throws InterruptedException, IOException {
-      // TODO relative path keeps showing up
-   }
 }

rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobMapIntegrationTest.java

@@ -26,8 +26,5 @@ import org.testng.annotations.Test;
  */
 @Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesBlobMapIntegrationTest")
 public class CloudFilesBlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
-   @Override
+
-   protected int maxList() {
-      return 10000;
-   }
 }

rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesContainerIntegrationTest.java

@@ -18,8 +18,6 @@
  */
 package org.jclouds.rackspace.cloudfiles.blobstore.integration;
 
-import java.io.UnsupportedEncodingException;
-
 import org.jclouds.blobstore.integration.internal.BaseContainerIntegrationTest;
 import org.testng.annotations.Test;
 
@@ -30,18 +28,4 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesContainerIntegrationTest")
 public class CloudFilesContainerIntegrationTest extends BaseContainerIntegrationTest {
 
-   @Override
-   @Test(enabled = false)
-   public void testListRootUsesDelimiter() throws InterruptedException,
-            UnsupportedEncodingException {
-      // TODO occasionally fails due to virtual directories not deleting from the prior run
-   }
-   
-
-   @Override
-   @Test(enabled = false)
-   public void testListContainerMarker() throws InterruptedException,
-            UnsupportedEncodingException {
-      // TODO occasionally fails due to virtual directories not deleting from the prior run
-   }
 }

rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesInputStreamMapIntegrationTest.java

@@ -26,8 +26,5 @@ import org.testng.annotations.Test;
  */
 @Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesInputStreamMapIntegrationTest")
 public class CloudFilesInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
-   @Override
+
-   protected int maxList() {
-      return 10000;
-   }
 }

tools/vfs/src/main/java/org/jclouds/vfs/provider/blobstore/BlobStoreFileObject.java

@@ -56,7 +56,7 @@ import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.domain.StorageType;
 import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.internal.ListAllMetadataInContainer;
+import org.jclouds.blobstore.strategy.internal.ConcatenateContainerLists;
 import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl;
 import org.jclouds.util.Utils;
 
@@ -70,7 +70,7 @@ import com.google.common.collect.Sets;
  */
 public class BlobStoreFileObject extends AbstractFileObject {
    private final BlobStoreContext context;
-   private final ListAllMetadataInContainer lister;
+   private final ConcatenateContainerLists lister;
    private final String container;
    private StorageMetadata metadata;
    private static final Logger logger = Logger.getLogger(BlobStoreFileObject.class);
@@ -81,7 +81,7 @@ public class BlobStoreFileObject extends AbstractFileObject {
       super(fileName, fileSystem);
       this.context = checkNotNull(context, "context");
       this.container = checkNotNull(container, "container");
-      this.lister = checkNotNull(new ListAllMetadataInContainer(context.getBlobStore()), "lister");
+      this.lister = checkNotNull(new ConcatenateContainerLists(context.getBlobStore()), "lister");
 
    }
 

vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/TerremarkVCloudClient.java

@@ -129,6 +129,7 @@ public interface TerremarkVCloudClient extends VCloudClient {
 
    void deleteNode(int nodeId);
 
+   @Timeout(duration = 180, timeUnit = TimeUnit.SECONDS)
    SortedSet<Node> getNodes(int internetServiceId);
 
    SortedSet<IpAddress> getIpAddressesForNetwork(String networkId);