overhauled the blobstore api to work with pseudo directories and continuable lists. fixed connection errors in http executor. changed to return null on resource not found

git-svn-id: http://jclouds.googlecode.com/svn/trunk@2745 3d8758e0-26b5-11de-8745-db77d3ebf521

atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java

@@ -46,6 +46,7 @@ import org.jclouds.atmosonline.saas.options.ListOptions;
 import org.jclouds.blobstore.attr.ConsistencyModel;
 import org.jclouds.blobstore.attr.ConsistencyModels;
 import org.jclouds.blobstore.functions.ReturnFalseOnKeyNotFound;
+import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
 import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
 import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404;
 import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
@@ -133,7 +134,7 @@ public interface AtmosStorageAsyncClient {
     */
    @GET
    @ResponseParser(ParseObjectFromHeadersAndHttpContent.class)
-   @ExceptionParser(ThrowKeyNotFoundOn404.class)
+   @ExceptionParser(ReturnNullOnKeyNotFound.class)
    @Path("/rest/namespace/{path}")
    @Consumes(MediaType.WILDCARD)
    ListenableFuture<AtmosObject> readFile(@PathParam("path") String path, GetOptions... options);
@@ -143,7 +144,7 @@ public interface AtmosStorageAsyncClient {
     */
    @HEAD
    @ResponseParser(ParseObjectFromHeadersAndHttpContent.class)
-   @ExceptionParser(ThrowKeyNotFoundOn404.class)
+   @ExceptionParser(ReturnNullOnKeyNotFound.class)
    @Path("/rest/namespace/{path}")
    @Consumes(MediaType.WILDCARD)
    ListenableFuture<AtmosObject> headFile(@PathParam("path") String path);
@@ -153,7 +154,7 @@ public interface AtmosStorageAsyncClient {
     */
    @HEAD
    @ResponseParser(ParseSystemMetadataFromHeaders.class)
-   @ExceptionParser(ThrowKeyNotFoundOn404.class)
+   @ExceptionParser(ReturnNullOnKeyNotFound.class)
    // currently throws 403 errors @QueryParams(keys = "metadata/system")
    @Path("/rest/namespace/{path}")
    @Consumes(MediaType.WILDCARD)
@@ -164,7 +165,7 @@ public interface AtmosStorageAsyncClient {
     */
    @HEAD
    @ResponseParser(ParseSystemMetadataFromHeaders.class)
-   @ExceptionParser(ThrowKeyNotFoundOn404.class)
+   @ExceptionParser(ReturnNullOnKeyNotFound.class)
    @Path("/rest/namespace/{path}")
    @QueryParams(keys = "metadata/user")
    @Consumes(MediaType.WILDCARD)

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java

@@ -18,17 +18,15 @@
  */
 package org.jclouds.atmosonline.saas.blobstore;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.util.concurrent.Futures.compose;
-import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
-import static org.jclouds.concurrent.ConcurrentUtils.convertExceptionToValue;
-import static org.jclouds.concurrent.ConcurrentUtils.makeListenable;
 
 import java.net.URI;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
 import javax.inject.Named;
+import javax.inject.Singleton;
 
 import org.jclouds.Constants;
 import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient;
@@ -38,50 +36,59 @@ import org.jclouds.atmosonline.saas.blobstore.functions.BlobToObject;
 import org.jclouds.atmosonline.saas.blobstore.functions.DirectoryEntryListToResourceMetadataList;
 import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlob;
 import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlobMetadata;
-import org.jclouds.atmosonline.saas.blobstore.internal.BaseAtmosBlobStore;
 import org.jclouds.atmosonline.saas.domain.AtmosObject;
 import org.jclouds.atmosonline.saas.options.ListOptions;
-import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.atmosonline.saas.util.AtmosStorageUtils;
-import org.jclouds.blobstore.ContainerNotFoundException;
-import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
-import org.jclouds.blobstore.domain.ListResponse;
 import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.Blob.Factory;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
-import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
+import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.encryption.EncryptionService;
 import org.jclouds.http.options.GetOptions;
-import org.jclouds.logging.Logger.LoggerFactory;
 import org.jclouds.util.Utils;
 
 import com.google.common.base.Function;
 import com.google.common.base.Supplier;
+import com.google.common.base.Throwables;
 import com.google.common.util.concurrent.ListenableFuture;
 
 /**
  * @author Adrian Cole
  */
-public class AtmosAsyncBlobStore extends BaseAtmosBlobStore implements AsyncBlobStore {
+@Singleton
+public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
+   private final AtmosStorageAsyncClient async;
+   private final AtmosStorageClient sync;
+   private final ObjectToBlob object2Blob;
+   private final ObjectToBlobMetadata object2BlobMd;
+   private final BlobToObject blob2Object;
+   private final BlobStoreListOptionsToListOptions container2ContainerListOptions;
+   private final DirectoryEntryListToResourceMetadataList container2ResourceList;
    private final EncryptionService encryptionService;
+   private final BlobToHttpGetOptions blob2ObjectGetOptions;
 
    @Inject
-   public AtmosAsyncBlobStore(AtmosStorageAsyncClient async, AtmosStorageClient sync,
+   AtmosAsyncBlobStore(BlobStoreUtils blobUtils,
-            Factory blobFactory, LoggerFactory logFactory,
-            ClearListStrategy clearContainerStrategy, ObjectToBlobMetadata object2BlobMd,
-            ObjectToBlob object2Blob, BlobToObject blob2Object,
-            BlobStoreListOptionsToListOptions container2ContainerListOptions,
-            BlobToHttpGetOptions blob2ObjectGetOptions,
-            DirectoryEntryListToResourceMetadataList container2ResourceList,
             @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service,
-            EncryptionService encryptionService) {
+            AtmosStorageAsyncClient async, AtmosStorageClient sync, ObjectToBlob object2Blob,
-      super(async, sync, blobFactory, logFactory, clearContainerStrategy, object2BlobMd,
+            ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object,
-               object2Blob, blob2Object, container2ContainerListOptions, blob2ObjectGetOptions,
+            BlobStoreListOptionsToListOptions container2ContainerListOptions,
-               container2ResourceList, service);
+            DirectoryEntryListToResourceMetadataList container2ResourceList,
-      this.encryptionService = encryptionService;
+            EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions) {
+      super(blobUtils, service);
+      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
+      this.sync = checkNotNull(sync, "sync");
+      this.async = checkNotNull(async, "async");
+      this.container2ContainerListOptions = checkNotNull(container2ContainerListOptions,
+               "container2ContainerListOptions");
+      this.container2ResourceList = checkNotNull(container2ResourceList, "container2ResourceList");
+      this.object2Blob = checkNotNull(object2Blob, "object2Blob");
+      this.blob2Object = checkNotNull(blob2Object, "blob2Object");
+      this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
+      this.encryptionService = checkNotNull(encryptionService, "encryptionService");
    }
 
    /**
@@ -89,29 +96,13 @@ public class AtmosAsyncBlobStore extends BaseAtmosBlobStore implements AsyncBlob
     */
    @Override
    public ListenableFuture<BlobMetadata> blobMetadata(String container, String key) {
-      return compose(convertExceptionToValue(async.headFile(container + "/" + key),
+      return compose(async.headFile(container + "/" + key),
-               KeyNotFoundException.class, null), new Function<AtmosObject, BlobMetadata>() {
+               new Function<AtmosObject, BlobMetadata>() {
-         @Override
+                  @Override
-         public BlobMetadata apply(AtmosObject from) {
+                  public BlobMetadata apply(AtmosObject from) {
-            return object2BlobMd.apply(from);
+                     return object2BlobMd.apply(from);
-         }
+                  }
-      }, service);
+               }, service);
-   }
-
-   /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
-    * {@link ListContainerOptions#recursive} option.
-    */
-   @Override
-   public ListenableFuture<Void> clearContainer(final String container) {
-      return makeListenable(service.submit(new Callable<Void>() {
-
-         public Void call() throws Exception {
-            clearContainerStrategy.execute(container, recursive());
-            return null;
-         }
-
-      }), service);
    }
 
    /**
@@ -145,28 +136,12 @@ public class AtmosAsyncBlobStore extends BaseAtmosBlobStore implements AsyncBlob
    }
 
    /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
+    * This implementation invokes {@link AtmosStorageAsyncClient#deletePath} followed by
-    * {@link ListContainerOptions#recursive} option. Then, it blocks until
+    * {@link AtmosStorageAsyncClient#pathExists} until it is true.
-    * {@link AtmosStorageAsyncClient#pathExists} fails.
     */
-   @Override
+   protected boolean deleteAndVerifyContainerGone(final String container) {
-   public ListenableFuture<Void> deleteContainer(final String container) {
+      sync.deletePath(container);
-      return makeListenable(service.submit(new Callable<Void>() {
+      return !sync.pathExists(container);
-
-         public Void call() throws Exception {
-            clearContainerStrategy.execute(container, recursive());
-            sync.deletePath(container);
-            if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
-               public Boolean get() {
-                  return !sync.pathExists(container);
-               }
-            }, 300)) {
-               throw new IllegalStateException(container + " still exists after deleting!");
-            }
-            return null;
-         }
-
-      }), service);
    }
 
    /**
@@ -174,8 +149,7 @@ public class AtmosAsyncBlobStore extends BaseAtmosBlobStore implements AsyncBlob
     */
    @Override
    public ListenableFuture<Boolean> containerExists(String container) {
-      return convertExceptionToValue(async.pathExists(container), ContainerNotFoundException.class,
+      return async.pathExists(container);
-               false);
    }
 
    /**
@@ -183,7 +157,7 @@ public class AtmosAsyncBlobStore extends BaseAtmosBlobStore implements AsyncBlob
     */
    @Override
    public ListenableFuture<Boolean> directoryExists(String container, String directory) {
-      return async.pathExists(container + "/" + directory);
+      return async.pathExists(container + "/" + directory + "/");
    }
 
    /**
@@ -199,15 +173,6 @@ public class AtmosAsyncBlobStore extends BaseAtmosBlobStore implements AsyncBlob
       return async.pathExists(container + "/" + key);
    }
 
-   /**
-    * This implementation invokes
-    * {@link #getBlob(String,String,org.jclouds.blobstore.options.GetOptions)}
-    */
-   @Override
-   public ListenableFuture<Blob> getBlob(String container, String key) {
-      return this.getBlob(container, key, org.jclouds.blobstore.options.GetOptions.NONE);
-   }
-
    /**
     * This implementation invokes {@link AtmosStorageAsyncClient#readFile}
     */
@@ -216,35 +181,24 @@ public class AtmosAsyncBlobStore extends BaseAtmosBlobStore implements AsyncBlob
             org.jclouds.blobstore.options.GetOptions options) {
       GetOptions httpOptions = blob2ObjectGetOptions.apply(options);
       ListenableFuture<AtmosObject> returnVal = async.readFile(container + "/" + key, httpOptions);
-      return compose(convertExceptionToValue(returnVal, KeyNotFoundException.class, null),
+      return compose(returnVal, object2Blob, service);
-               object2Blob, service);
    }
 
    /**
     * This implementation invokes {@link AtmosStorageAsyncClient#listDirectories}
     */
    @Override
-   public ListenableFuture<? extends ListResponse<? extends StorageMetadata>> list() {
+   public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list() {
       return compose(async.listDirectories(), container2ResourceList, service);
    }
 
-   /**
-    * This implementation invokes
-    * {@link #list(String,org.jclouds.blobstore.options.ListContainerOptions)}
-    */
-   @Override
-   public ListenableFuture<? extends ListContainerResponse<? extends StorageMetadata>> list(
-            String container) {
-      return this.list(container, org.jclouds.blobstore.options.ListContainerOptions.NONE);
-   }
-
    /**
     * This implementation invokes {@link AtmosStorageAsyncClient#listDirectory}
     */
    @Override
-   public ListenableFuture<? extends ListContainerResponse<? extends StorageMetadata>> list(
+   public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(String container,
-            String container, org.jclouds.blobstore.options.ListContainerOptions options) {
+            org.jclouds.blobstore.options.ListContainerOptions options) {
-      container = adjustContainerIfDirOptionPresent(container, options);
+      container = AtmosStorageUtils.adjustContainerIfDirOptionPresent(container, options);
       ListOptions nativeOptions = container2ContainerListOptions.apply(options);
       return compose(async.listDirectory(container, nativeOptions), container2ResourceList, service);
    }
@@ -274,8 +228,10 @@ public class AtmosAsyncBlobStore extends BaseAtmosBlobStore implements AsyncBlob
                sync.createFile(container, blob2Object.apply(blob));
                return path;
             } catch (InterruptedException e) {
-               throw new RuntimeException(e);
+               Throwables.propagate(e);
             }
+            assert false : " should have propagated error";
+            return null;
          }
 
       }, service);

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java

@@ -18,15 +18,11 @@
  */
 package org.jclouds.atmosonline.saas.blobstore;
 
-import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
+import static com.google.common.base.Preconditions.checkNotNull;
-import static org.jclouds.blobstore.util.BlobStoreUtils.keyNotFoundToNullOrPropagate;
-
-import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
-import javax.inject.Named;
+import javax.inject.Singleton;
 
-import org.jclouds.Constants;
 import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient;
 import org.jclouds.atmosonline.saas.AtmosStorageClient;
 import org.jclouds.atmosonline.saas.blobstore.functions.BlobStoreListOptionsToListOptions;
@@ -34,43 +30,48 @@ import org.jclouds.atmosonline.saas.blobstore.functions.BlobToObject;
 import org.jclouds.atmosonline.saas.blobstore.functions.DirectoryEntryListToResourceMetadataList;
 import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlob;
 import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlobMetadata;
-import org.jclouds.atmosonline.saas.blobstore.internal.BaseAtmosBlobStore;
 import org.jclouds.atmosonline.saas.options.ListOptions;
-import org.jclouds.blobstore.BlobStore;
+import org.jclouds.atmosonline.saas.util.AtmosStorageUtils;
-import org.jclouds.blobstore.ContainerNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
-import org.jclouds.blobstore.domain.ListResponse;
 import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.Blob.Factory;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
-import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.internal.BaseBlobStore;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
+import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.encryption.EncryptionService;
 import org.jclouds.http.options.GetOptions;
-import org.jclouds.logging.Logger.LoggerFactory;
-import org.jclouds.util.Utils;
 
-import com.google.common.base.Supplier;
+/**
-
+ * @author Adrian Cole
-public class AtmosBlobStore extends BaseAtmosBlobStore implements BlobStore {
+ */
+@Singleton
+public class AtmosBlobStore extends BaseBlobStore {
+   private final AtmosStorageClient sync;
+   private final ObjectToBlob object2Blob;
+   private final ObjectToBlobMetadata object2BlobMd;
+   private final BlobToObject blob2Object;
+   private final BlobStoreListOptionsToListOptions container2ContainerListOptions;
+   private final DirectoryEntryListToResourceMetadataList container2ResourceList;
    private final EncryptionService encryptionService;
+   private final BlobToHttpGetOptions blob2ObjectGetOptions;
 
    @Inject
-   public AtmosBlobStore(AtmosStorageAsyncClient async, AtmosStorageClient sync,
+   AtmosBlobStore(BlobStoreUtils blobUtils, AtmosStorageClient sync, ObjectToBlob object2Blob,
-            Factory blobFactory, LoggerFactory logFactory,
+            ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object,
-            ClearListStrategy clearContainerStrategy, ObjectToBlobMetadata object2BlobMd,
-            ObjectToBlob object2Blob, BlobToObject blob2Object,
             BlobStoreListOptionsToListOptions container2ContainerListOptions,
-            BlobToHttpGetOptions blob2ObjectGetOptions,
             DirectoryEntryListToResourceMetadataList container2ResourceList,
-            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service,
+            EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions) {
-            EncryptionService encryptionService) {
+      super(blobUtils);
-      super(async, sync, blobFactory, logFactory, clearContainerStrategy, object2BlobMd,
+      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
-               object2Blob, blob2Object, container2ContainerListOptions, blob2ObjectGetOptions,
+      this.sync = checkNotNull(sync, "sync");
-               container2ResourceList, service);
+      this.container2ContainerListOptions = checkNotNull(container2ContainerListOptions,
-      this.encryptionService = encryptionService;
+               "container2ContainerListOptions");
+      this.container2ResourceList = checkNotNull(container2ResourceList, "container2ResourceList");
+      this.object2Blob = checkNotNull(object2Blob, "object2Blob");
+      this.blob2Object = checkNotNull(blob2Object, "blob2Object");
+      this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
+      this.encryptionService = checkNotNull(encryptionService, "encryptionService");
    }
 
    /**
@@ -78,54 +79,16 @@ public class AtmosBlobStore extends BaseAtmosBlobStore implements BlobStore {
     */
    @Override
    public BlobMetadata blobMetadata(String container, String key) {
-      try {
+      return object2BlobMd.apply(sync.headFile(container + "/" + key));
-         return object2BlobMd.apply(sync.headFile(container + "/" + key));
-      } catch (Exception e) {
-         return keyNotFoundToNullOrPropagate(e);
-      }
    }
 
    /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
+    * This implementation invokes {@link AtmosStorageAsyncClient#deletePath} followed by
-    * {@link ListContainerOptions#recursive} option.
+    * {@link AtmosStorageAsyncClient#pathExists} until it is true.
     */
-   @Override
+   protected boolean deleteAndVerifyContainerGone(final String container) {
-   public void clearContainer(final String container) {
+      sync.deletePath(container);
-      clearContainerStrategy.execute(container, recursive());
+      return !sync.pathExists(container);
-   }
-
-   /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
-    * {@link ListContainerOptions#recursive} option. Then, it invokes
-    * {@link #deleteAndEnsurePathGone}
-    */
-   @Override
-   public void deleteContainer(final String container) {
-      try {
-         clearContainerStrategy.execute(container, recursive());
-         deleteAndEnsurePathGone(container);
-      } catch (ContainerNotFoundException e) {
-
-      }
-   }
-
-   /**
-    * This implementation invokes {@link AtmosStorageClient#deleteAndEnsurePathGone} then blocks
-    * until {@link AtmosStorageClient#pathExists} returns false.
-    */
-   public void deleteAndEnsurePathGone(final String path) {
-      sync.deletePath(path);
-      try {
-         if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
-            public Boolean get() {
-               return !sync.pathExists(path);
-            }
-         }, 30000)) {
-            throw new IllegalStateException(path + " still exists after deleting!");
-         }
-      } catch (InterruptedException e) {
-         new IllegalStateException(path + " interrupted during deletion!", e);
-      }
    }
 
    /**
@@ -158,11 +121,7 @@ public class AtmosBlobStore extends BaseAtmosBlobStore implements BlobStore {
     */
    @Override
    public boolean containerExists(String container) {
-      try {
+      return sync.pathExists(container);
-         return sync.pathExists(container);
-      } catch (ContainerNotFoundException e) {
-         return false;
-      }
    }
 
    /**
@@ -170,12 +129,7 @@ public class AtmosBlobStore extends BaseAtmosBlobStore implements BlobStore {
     */
    @Override
    public boolean directoryExists(String container, String directory) {
-      try {
+      return sync.pathExists(container + "/" + directory + "/");
-         return sync.pathExists(container + "/" + directory);
-      } catch (Exception e) {
-         keyNotFoundToNullOrPropagate(e);
-         return false;
-      }
    }
 
    /**
@@ -191,15 +145,6 @@ public class AtmosBlobStore extends BaseAtmosBlobStore implements BlobStore {
       return sync.pathExists(container + "/" + key);
    }
 
-   /**
-    * This implementation invokes
-    * {@link #getBlob(String,String,org.jclouds.blobstore.options.GetOptions)}
-    */
-   @Override
-   public Blob getBlob(String container, String key) {
-      return this.getBlob(container, key, org.jclouds.blobstore.options.GetOptions.NONE);
-   }
-
    /**
     * This implementation invokes {@link AtmosStorageClient#readFile}
     */
@@ -207,37 +152,24 @@ public class AtmosBlobStore extends BaseAtmosBlobStore implements BlobStore {
    public Blob getBlob(String container, String key,
             org.jclouds.blobstore.options.GetOptions options) {
       GetOptions httpOptions = blob2ObjectGetOptions.apply(options);
-      try {
+      return object2Blob.apply(sync.readFile(container + "/" + key, httpOptions));
-         return object2Blob.apply(sync.readFile(container + "/" + key, httpOptions));
-      } catch (Exception e) {
-         return keyNotFoundToNullOrPropagate(e);
-      }
    }
 
    /**
     * This implementation invokes {@link AtmosStorageClient#listDirectories}
     */
    @Override
-   public ListResponse<? extends StorageMetadata> list() {
+   public PageSet<? extends StorageMetadata> list() {
       return container2ResourceList.apply(sync.listDirectories());
    }
 
-   /**
-    * This implementation invokes
-    * {@link #list(String,org.jclouds.blobstore.options.ListContainerOptions)}
-    */
-   @Override
-   public ListContainerResponse<? extends StorageMetadata> list(String container) {
-      return this.list(container, org.jclouds.blobstore.options.ListContainerOptions.NONE);
-   }
-
    /**
     * This implementation invokes {@link AtmosStorageClient#listDirectory}
     */
    @Override
-   public ListContainerResponse<? extends StorageMetadata> list(String container,
+   public PageSet<? extends StorageMetadata> list(String container,
             org.jclouds.blobstore.options.ListContainerOptions options) {
-      container = adjustContainerIfDirOptionPresent(container, options);
+      container = AtmosStorageUtils.adjustContainerIfDirOptionPresent(container, options);
       ListOptions nativeOptions = container2ContainerListOptions.apply(options);
       return container2ResourceList.apply(sync.listDirectory(container, nativeOptions));
    }

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobStoreListOptionsToListOptions.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.atmosonline.saas.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Singleton;
 
 import org.jclouds.blobstore.options.ListContainerOptions;
@@ -32,14 +34,13 @@ public class BlobStoreListOptionsToListOptions implements
          Function<ListContainerOptions, org.jclouds.atmosonline.saas.options.ListOptions> {
    @Override
    public org.jclouds.atmosonline.saas.options.ListOptions apply(ListContainerOptions from) {
+      checkNotNull(from, "set options to instance NONE instead of passing null");
       org.jclouds.atmosonline.saas.options.ListOptions httpOptions = new org.jclouds.atmosonline.saas.options.ListOptions();
-      if (from != null && from != ListContainerOptions.NONE) {
+      if (from.getMarker() != null) {
-         if (from.getMarker() != null) {
+         httpOptions.token(from.getMarker());
-            httpOptions.token(from.getMarker());
+      }
-         }
+      if (from.getMaxResults() != null) {
-         if (from.getMaxResults() != null) {
+         httpOptions.limit(from.getMaxResults());
-            httpOptions.limit(from.getMaxResults());
-         }
       }
       return httpOptions;
    }

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobToObject.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.atmosonline.saas.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -42,7 +44,9 @@ public class BlobToObject implements Function<Blob, AtmosObject> {
       if (from == null)
          return null;
       AtmosObject object = blobMd2Object.apply(from.getMetadata());
-      object.setPayload(from.getPayload());
+      if (from.getContentLength() != null)
+         object.setContentLength(from.getContentLength());
+      object.setPayload(checkNotNull(from.getPayload(), "payload: " + from));
       object.setAllHeaders(from.getAllHeaders());
       return object;
    }

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/DirectoryEntryListToResourceMetadataList.java

@@ -23,11 +23,11 @@ import javax.inject.Singleton;
 import org.jclouds.atmosonline.saas.domain.BoundedSet;
 import org.jclouds.atmosonline.saas.domain.DirectoryEntry;
 import org.jclouds.atmosonline.saas.domain.FileType;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.domain.StorageType;
 import org.jclouds.blobstore.domain.internal.BlobMetadataImpl;
-import org.jclouds.blobstore.domain.internal.ListContainerResponseImpl;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
 import org.jclouds.blobstore.domain.internal.StorageMetadataImpl;
 
 import com.google.common.base.Function;
@@ -38,14 +38,12 @@ import com.google.common.collect.Maps;
  * @author Adrian Cole
  */
 @Singleton
-public class DirectoryEntryListToResourceMetadataList
+public class DirectoryEntryListToResourceMetadataList implements
-         implements
+         Function<BoundedSet<? extends DirectoryEntry>, PageSet<? extends StorageMetadata>> {
-         Function<BoundedSet<? extends DirectoryEntry>, ListContainerResponse<? extends StorageMetadata>> {
 
-   public ListContainerResponse<? extends StorageMetadata> apply(
+   public PageSet<? extends StorageMetadata> apply(BoundedSet<? extends DirectoryEntry> from) {
-            BoundedSet<? extends DirectoryEntry> from) {
 
-      return new ListContainerResponseImpl<StorageMetadata>(Iterables.transform(from,
+      return new PageSetImpl<StorageMetadata>(Iterables.transform(from,
                new Function<DirectoryEntry, StorageMetadata>() {
 
                   public StorageMetadata apply(DirectoryEntry from) {
@@ -57,12 +55,11 @@ public class DirectoryEntryListToResourceMetadataList
                                  .<String, String> newHashMap());
                      else
                         return new BlobMetadataImpl(from.getObjectID(), from.getObjectName(), null,
-                                 null, null, null, null, Maps.<String, String> newHashMap(), null, null);
+                                 null, null, null, null, Maps.<String, String> newHashMap(), null,
+                                 null);
                   }
 
-               }), null, from.getToken(),
+               }), from.getToken());
-
-      null, from.getToken() != null);
 
    }
 }

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlob.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.atmosonline.saas.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -47,8 +49,7 @@ public class ObjectToBlob implements Function<AtmosObject, Blob> {
       Blob blob = blobFactory.create(object2BlobMd.apply(from));
       if (from.getContentMetadata().getContentLength() != null)
          blob.setContentLength(from.getContentMetadata().getContentLength());
-      if (from.getPayload() != null)
+      blob.setPayload(checkNotNull(from.getPayload(), "payload: " + from));
-         blob.setPayload(from.getPayload());
       blob.setAllHeaders(from.getAllHeaders());
       return blob;
    }

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/internal/BaseAtmosBlobStore.java

@@ -1,95 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
- *
- * ====================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ====================================================================
- */
-package org.jclouds.atmosonline.saas.blobstore.internal;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.concurrent.ExecutorService;
-
-import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient;
-import org.jclouds.atmosonline.saas.AtmosStorageClient;
-import org.jclouds.atmosonline.saas.blobstore.functions.BlobStoreListOptionsToListOptions;
-import org.jclouds.atmosonline.saas.blobstore.functions.BlobToObject;
-import org.jclouds.atmosonline.saas.blobstore.functions.DirectoryEntryListToResourceMetadataList;
-import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlob;
-import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlobMetadata;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
-import org.jclouds.logging.Logger.LoggerFactory;
-
-import com.google.inject.Inject;
-
-public class BaseAtmosBlobStore {
-   protected final AtmosStorageAsyncClient async;
-   protected final AtmosStorageClient sync;
-   protected final Blob.Factory blobFactory;
-   protected final LoggerFactory logFactory;
-   protected final ClearListStrategy clearContainerStrategy;
-   protected final ObjectToBlobMetadata object2BlobMd;
-   protected final ObjectToBlob object2Blob;
-   protected final BlobToObject blob2Object;
-   protected final BlobStoreListOptionsToListOptions container2ContainerListOptions;
-   protected final BlobToHttpGetOptions blob2ObjectGetOptions;
-   protected final DirectoryEntryListToResourceMetadataList container2ResourceList;
-   protected final ExecutorService service;
-
-   @Inject
-   protected BaseAtmosBlobStore(AtmosStorageAsyncClient async, AtmosStorageClient sync,
-            Blob.Factory blobFactory, LoggerFactory logFactory,
-            ClearListStrategy clearContainerStrategy, ObjectToBlobMetadata object2BlobMd,
-            ObjectToBlob object2Blob, BlobToObject blob2Object,
-            BlobStoreListOptionsToListOptions container2ContainerListOptions,
-            BlobToHttpGetOptions blob2ObjectGetOptions,
-            DirectoryEntryListToResourceMetadataList container2ResourceList, ExecutorService service) {
-      this.async = checkNotNull(async, "async");
-      this.sync = checkNotNull(sync, "sync");
-      this.blobFactory = checkNotNull(blobFactory, "blobFactory");
-      this.logFactory = checkNotNull(logFactory, "logFactory");
-      this.clearContainerStrategy = checkNotNull(clearContainerStrategy, "clearContainerStrategy");
-      this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
-      this.object2Blob = checkNotNull(object2Blob, "object2Blob");
-      this.blob2Object = checkNotNull(blob2Object, "blob2Object");
-      this.container2ContainerListOptions = checkNotNull(container2ContainerListOptions,
-               "container2ContainerListOptions");
-      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
-      this.container2ResourceList = checkNotNull(container2ResourceList, "container2ResourceList");
-      this.service = checkNotNull(service, "service");
-   }
-
-   public Blob newBlob(String name) {
-      Blob blob = blobFactory.create(null);
-      blob.getMetadata().setName(name);
-      return blob;
-   }
-
-   protected String adjustContainerIfDirOptionPresent(String container,
-            org.jclouds.blobstore.options.ListContainerOptions options) {
-      if (options != org.jclouds.blobstore.options.ListContainerOptions.NONE) {
-         if (options.isRecursive()) {
-            throw new UnsupportedOperationException("recursive not currently supported in emcsaas");
-         }
-         if (options.getDir() != null) {
-            container = container + "/" + options.getDir();
-         }
-      }
-      return container;
-   }
-
-}

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java

@@ -18,10 +18,12 @@
  */
 package org.jclouds.atmosonline.saas.blobstore.strategy;
 
+import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
+
 import java.util.Arrays;
-import java.util.SortedSet;
+import java.util.Map;
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.TimeUnit;
@@ -41,9 +43,8 @@ import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
 import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
 import org.jclouds.logging.Logger;
 
-import com.google.common.base.Function;
 import com.google.common.base.Throwables;
-import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.inject.Inject;
 
@@ -81,42 +82,34 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
    public boolean execute(final String containerName, Object value, ListContainerOptions options) {
       final byte[] toSearch = objectMD5.apply(value);
       final BlockingQueue<Boolean> queue = new SynchronousQueue<Boolean>();
-
+      Map<String, ListenableFuture<?>> responses = Maps.newHashMap();
-      SortedSet<? extends BlobMetadata> allMd = getAllBlobMetadata.execute(containerName, options);
+      for (BlobMetadata md : getAllBlobMetadata.execute(containerName, options)) {
-
+         final ListenableFuture<AtmosObject> future = client.headFile(containerName + "/"
-      final CountDownLatch doneSignal = new CountDownLatch(allMd.size());
+                  + md.getName());
-
-      for (final ListenableFuture<AtmosObject> future : Iterables.transform(getAllBlobMetadata
-               .execute(containerName, options),
-               new Function<BlobMetadata, ListenableFuture<AtmosObject>>() {
-                  @Override
-                  public ListenableFuture<AtmosObject> apply(BlobMetadata from) {
-                     return client.headFile(containerName + "/" + from.getName());
-                  }
-
-               })) {
          future.addListener(new Runnable() {
             public void run() {
                try {
-                  future.get();
-                  doneSignal.countDown();
                   if (Arrays.equals(toSearch, future.get().getContentMetadata().getContentMD5())) {
                      queue.put(true);
                   }
-               } catch (Exception e) {
+               } catch (InterruptedException e) {
-                  doneSignal.countDown();
+                  Throwables.propagate(e);
+               } catch (ExecutionException e) {
+                  Throwables.propagate(e);
                }
             }
          }, userExecutor);
+         responses.put(md.getName(), future);
       }
+      Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
+               String.format("searching for md5 in container %s", containerName));
+      if (exceptions.size() > 0)
+         throw new BlobRuntimeException(String.format("searching for md5 in container %s: %s",
+                  containerName, exceptions));
       try {
-         if (maxTime != null) {
+         return queue.poll(1, TimeUnit.MICROSECONDS);
-            return queue.poll(maxTime, TimeUnit.MILLISECONDS);
-         } else {
-            doneSignal.await();
-            return queue.poll(1, TimeUnit.MICROSECONDS);
-         }
       } catch (InterruptedException e) {
+         Throwables.propagate(e);
          return false;
       } catch (Exception e) {
          Throwables.propagateIfPossible(e, BlobRuntimeException.class);

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/RecursiveRemove.java

@@ -20,7 +20,7 @@ package org.jclouds.atmosonline.saas.blobstore.strategy;
 
 import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
 
-import java.util.Set;
+import java.util.Map;
 import java.util.concurrent.ExecutorService;
 
 import javax.annotation.Resource;
@@ -32,6 +32,7 @@ import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient;
 import org.jclouds.atmosonline.saas.AtmosStorageClient;
 import org.jclouds.atmosonline.saas.domain.DirectoryEntry;
 import org.jclouds.atmosonline.saas.domain.FileType;
+import org.jclouds.blobstore.internal.BlobRuntimeException;
 import org.jclouds.blobstore.options.ListContainerOptions;
 import org.jclouds.blobstore.strategy.ClearContainerStrategy;
 import org.jclouds.blobstore.strategy.ClearListStrategy;
@@ -40,7 +41,7 @@ import org.jclouds.util.Utils;
 
 import com.google.common.base.Function;
 import com.google.common.base.Supplier;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.inject.Inject;
@@ -80,14 +81,18 @@ public class RecursiveRemove implements ClearListStrategy, ClearContainerStrateg
    }
 
    private ListenableFuture<Void> rm(final String fullPath, FileType type, boolean recursive) {
-      Set<ListenableFuture<Void>> responses = Sets.newHashSet();
+      Map<String, ListenableFuture<?>> responses = Maps.newHashMap();
       if ((type == FileType.DIRECTORY) && recursive) {
          for (DirectoryEntry child : sync.listDirectory(fullPath)) {
-            responses.add(rm(fullPath + "/" + child.getObjectName(), child.getType(), true));
+            responses.put(fullPath + "/" + child.getObjectName(), rm(fullPath + "/"
+                     + child.getObjectName(), child.getType(), true));
          }
       }
-      awaitCompletion(responses, userExecutor, maxTime, logger, String.format(
+      Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
-               "deleting from path: %s", fullPath));
+               String.format("deleting from path: %s", fullPath));
+      if (exceptions.size() > 0)
+         throw new BlobRuntimeException(String.format("deleting from path %s: %s", fullPath,
+                  exceptions));
 
       return Futures.compose(async.deletePath(fullPath), new Function<Void, Void>() {
 
@@ -112,12 +117,16 @@ public class RecursiveRemove implements ClearListStrategy, ClearContainerStrateg
    public void execute(String path, ListContainerOptions options) {
       if (options.getDir() != null)
          path += "/" + options.getDir();
-      Set<ListenableFuture<Void>> responses = Sets.newHashSet();
+      Map<String, ListenableFuture<?>> responses = Maps.newHashMap();
       for (DirectoryEntry md : sync.listDirectory(path)) {
-         responses.add(rm(path + "/" + md.getObjectName(), md.getType(), options.isRecursive()));
+         responses.put(path + "/" + md.getObjectName(), rm(path + "/" + md.getObjectName(), md
+                  .getType(), options.isRecursive()));
       }
-      awaitCompletion(responses, userExecutor, maxTime, logger, String.format(
+      Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
-               "deleting from path: %s", path));
+               String.format("deleting from path: %s", path));
+      if (exceptions.size() > 0)
+         throw new BlobRuntimeException(String
+                  .format("deleting from path %s: %s", path, exceptions));
    }
 
 }

atmos/src/main/java/org/jclouds/atmosonline/saas/functions/ParseObjectFromHeadersAndHttpContent.java

@@ -62,12 +62,20 @@ public class ParseObjectFromHeadersAndHttpContent implements Function<HttpRespon
       AtmosObject object = objectProvider.create(systemMetadataParser.apply(from),
                userMetadataParser.apply(from));
       addAllHeadersTo(from, object);
-      if (from.getContent() != null)
+
-         object.setPayload(from.getContent());
       String contentLength = from.getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH);
       if (contentLength != null) {
          object.getContentMetadata().setContentLength(Long.parseLong(contentLength));
       }
+      
+      if (from.getContent() != null) {
+         object.setPayload(from.getContent());
+      } else if (object.getContentLength() != null && object.getContentLength() == 0) {
+         object.setPayload(new byte[0]);
+      } else {
+         assert false : "no content in " + from;
+      }
+
       String contentType = from.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE);
       if (contentType != null) {
          object.getContentMetadata().setContentType(contentType);

atmos/src/main/java/org/jclouds/atmosonline/saas/handlers/ParseAtmosStorageErrorFromXmlContent.java

@@ -67,48 +67,58 @@ public class ParseAtmosStorageErrorFromXmlContent implements HttpErrorHandler {
    public void handleError(HttpCommand command, HttpResponse response) {
       Exception exception = new HttpResponseException(command, response);
       try {
-         switch (response.getStatusCode()) {
+         AtmosStorageError error = parseErrorFromContentOrNull(command, response);
-            case 401:
+         if (error != null && error.getCode() == 1016) {
-               exception = new AuthorizationException(command.getRequest().getRequestLine());
+            File file = new File(command.getRequest().getEndpoint().getPath());
-               break;
+            exception = new KeyAlreadyExistsException(file.getParentFile().getAbsolutePath(), file
-            case 404:
+                     .getName());
-               if (!command.getRequest().getMethod().equals("DELETE")) {
+         } else {
-                  String path = command.getRequest().getEndpoint().getPath();
+            switch (response.getStatusCode()) {
-                  Matcher matcher = CONTAINER_PATH.matcher(path);
+               case 401:
-                  if (matcher.find()) {
+                  exception = new AuthorizationException(command.getRequest(),
-                     exception = new ContainerNotFoundException(matcher.group(1));
+                           error != null ? error.getMessage() : response.getStatusLine());
-                  } else {
+                  break;
-                     matcher = CONTAINER_KEY_PATH.matcher(path);
+               case 404:
+                  if (!command.getRequest().getMethod().equals("DELETE")) {
+                     String message = error != null ? error.getMessage() : String.format(
+                              "%s -> %s", command.getRequest().getRequestLine(), response
+                                       .getStatusLine());
+                     String path = command.getRequest().getEndpoint().getPath();
+                     Matcher matcher = CONTAINER_PATH.matcher(path);
                      if (matcher.find()) {
-                        exception = new KeyNotFoundException(matcher.group(1), matcher.group(2));
+                        exception = new ContainerNotFoundException(matcher.group(1), message);
-                     }
+                     } else {
-                  }
+                        matcher = CONTAINER_KEY_PATH.matcher(path);
-               }
+                        if (matcher.find()) {
-               break;
+                           exception = new KeyNotFoundException(matcher.group(1), matcher.group(2),
-            default:
+                                    message);
-               if (response.getContent() != null) {
-                  try {
-                     String content = Utils.toStringAndClose(response.getContent());
-                     if (content.indexOf('<') >= 0) {
-                        AtmosStorageError error = utils.parseAtmosStorageErrorFromContent(command,
-                                 response, content);
-                        if (error.getCode() == 1016) {
-                           File file = new File(command.getRequest().getEndpoint().getPath());
-                           exception = new KeyAlreadyExistsException(file.getParentFile()
-                                    .getAbsolutePath(), file.getName());
-                        } else {
-                           exception = new AtmosStorageResponseException(command, response, error);
                         }
                      }
-                  } catch (IOException e) {
-                     logger.warn(e, "exception reading error from response", response);
-                     exception = new HttpResponseException(command, response);
                   }
-               }
+                  break;
+               default:
+                  exception = error != null ? new AtmosStorageResponseException(command, response,
+                           error) : new HttpResponseException(command, response);
+
+            }
          }
       } finally {
          Closeables.closeQuietly(response.getContent());
          command.setException(exception);
       }
    }
+
+   AtmosStorageError parseErrorFromContentOrNull(HttpCommand command, HttpResponse response) {
+      if (response.getContent() != null) {
+         try {
+            String content = Utils.toStringAndClose(response.getContent());
+            if (content != null && content.indexOf('<') >= 0)
+               return utils.parseAtmosStorageErrorFromContent(command, response, Utils
+                        .toInputStream(content));
+         } catch (IOException e) {
+            logger.warn(e, "exception reading error from response", response);
+         }
+      }
+      return null;
+   }
 }

atmos/src/main/java/org/jclouds/atmosonline/saas/util/AtmosStorageUtils.java

@@ -65,5 +65,17 @@ public class AtmosStorageUtils {
       return parseAtmosStorageErrorFromContent(command, response, new ByteArrayInputStream(content
                .getBytes()));
    }
-
+   
+   public static String adjustContainerIfDirOptionPresent(String container,
+            org.jclouds.blobstore.options.ListContainerOptions options) {
+      if (options != org.jclouds.blobstore.options.ListContainerOptions.NONE) {
+         if (options.isRecursive()) {
+            throw new UnsupportedOperationException("recursive not currently supported in emcsaas");
+         }
+         if (options.getDir() != null) {
+            container = container + "/" + options.getDir();
+         }
+      }
+      return container;
+   }
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientTest.java

@@ -39,13 +39,14 @@ import org.jclouds.atmosonline.saas.options.ListOptions;
 import org.jclouds.atmosonline.saas.reference.AtmosStorageConstants;
 import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
 import org.jclouds.blobstore.config.BlobStoreObjectModule;
+import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
 import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
 import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404;
 import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
 import org.jclouds.date.TimeStamp;
 import org.jclouds.encryption.internal.Base64;
-import org.jclouds.http.functions.ParseURIFromListOrLocationHeaderIf20x;
 import org.jclouds.http.functions.CloseContentAndReturn;
+import org.jclouds.http.functions.ParseURIFromListOrLocationHeaderIf20x;
 import org.jclouds.http.options.GetOptions;
 import org.jclouds.logging.Logger;
 import org.jclouds.logging.Logger.LoggerFactory;
@@ -220,7 +221,7 @@ public class AtmosStorageClientTest extends RestClientTest<AtmosStorageAsyncClie
       assertResponseParserClassEquals(method, httpMethod,
                ParseObjectFromHeadersAndHttpContent.class);
       assertSaxResponseParserClassEquals(method, null);
-      assertExceptionParserClassEquals(method, ThrowKeyNotFoundOn404.class);
+      assertExceptionParserClassEquals(method, ReturnNullOnKeyNotFound.class);
 
       checkFilters(httpMethod);
    }
@@ -237,7 +238,7 @@ public class AtmosStorageClientTest extends RestClientTest<AtmosStorageAsyncClie
 
       assertResponseParserClassEquals(method, httpMethod, ParseSystemMetadataFromHeaders.class);
       assertSaxResponseParserClassEquals(method, null);
-      assertExceptionParserClassEquals(method, ThrowKeyNotFoundOn404.class);
+      assertExceptionParserClassEquals(method, ReturnNullOnKeyNotFound.class);
 
       checkFilters(httpMethod);
    }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/functions/ResourceMetadataListToDirectoryEntryList.java

@@ -36,10 +36,10 @@ import com.google.common.collect.Iterables;
 @Singleton
 public class ResourceMetadataListToDirectoryEntryList
          implements
-         Function<org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata>, BoundedSet<? extends DirectoryEntry>> {
+         Function<org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>, BoundedSet<? extends DirectoryEntry>> {
 
    public BoundedSet<DirectoryEntry> apply(
-            org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata> from) {
+            org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> from) {
 
       return new BoundedHashSet<DirectoryEntry>(Iterables.transform(from,
                new Function<StorageMetadata, DirectoryEntry>() {
@@ -49,7 +49,7 @@ public class ResourceMetadataListToDirectoryEntryList
                      return new DirectoryEntry(from.getId(), type, from.getName());
                   }
 
-               }), from.getMarker());
+               }), from.getNextMarker());
 
    }
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageContainerIntegrationTest.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.atmosonline.saas.blobstore.integration;
 
+import java.io.UnsupportedEncodingException;
+
 import org.jclouds.blobstore.integration.internal.BaseContainerIntegrationTest;
 import org.testng.annotations.Test;
 
@@ -34,4 +36,10 @@ public class AtmosStorageContainerIntegrationTest extends BaseContainerIntegrati
       super.testClearWhenContentsUnderPath();
    }
 
+   @Override
+   @Test(enabled = false)
+   public void testDirectory() throws InterruptedException, UnsupportedEncodingException {
+      // TODO
+   }
+
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageInputStreamMapIntegrationTest.java

@@ -82,4 +82,9 @@ public class AtmosStorageInputStreamMapIntegrationTest extends BaseInputStreamMa
       // TODO not reliable NPE
    }
 
+   @Override
+   protected int maxList() {
+      return 0;
+   }
+
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageIntegrationTest.java

@@ -60,6 +60,12 @@ public class AtmosStorageIntegrationTest extends BaseBlobIntegrationTest {
       // not supported
    }
 
+   @Override
+   @Test(enabled = false)
+   public void testGetRange() {
+      // TODO this should work
+   }
+
    @Override
    @Test(enabled = false)
    public void testGetTwoRanges() {

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageMapIntegrationTest.java

@@ -42,4 +42,9 @@ public class AtmosStorageMapIntegrationTest extends BaseBlobMapIntegrationTest {
       // TODO not reliable KeyAlreadyExistsException@AtmosBlobStore.java:213
    }
 
+   @Override
+   protected int maxList() {
+      return 0;
+   }
+
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/internal/StubAtmosStorageAsyncClient.java

@@ -89,14 +89,24 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
 
    public ListenableFuture<URI> createDirectory(String directoryName) {
       final String container;
-      if (directoryName.indexOf('/') != -1)
+      final String path;
+      if (directoryName.indexOf('/') != -1) {
          container = directoryName.substring(0, directoryName.indexOf('/'));
-      else
+         path = directoryName.substring(directoryName.indexOf('/') + 1);
+      } else {
          container = directoryName;
+         path = null;
+      }
       return Futures.compose(blobStore.createContainerInLocation("default", container),
                new Function<Boolean, URI>() {
 
                   public URI apply(Boolean from) {
+                     if (path != null) {
+                        Blob blob = blobStore.newBlob(path + "/");
+                        blob.getMetadata().setContentType("application/directory");
+                        blob.setPayload("");
+                        blobStore.putBlob(container, blob);
+                     }
                      return URI.create("http://stub/containers/" + container);
                   }
 
@@ -195,7 +205,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
    }
 
    public ListenableFuture<Boolean> pathExists(final String path) {
-      if (path.indexOf('/') == -1 || (path.endsWith("/")))
+      if (path.indexOf('/') == -1 )
          return blobStore.containerExists(path);
       else {
          String container = path.substring(0, path.indexOf('/'));

aws/core/src/main/java/org/jclouds/aws/ec2/compute/EC2ComputeService.java

@@ -63,7 +63,7 @@ import org.jclouds.compute.domain.internal.NodeSetImpl;
 import org.jclouds.compute.options.TemplateOptions;
 import org.jclouds.compute.reference.ComputeServiceConstants;
 import org.jclouds.compute.util.ComputeUtils;
-import org.jclouds.concurrent.ConcurrentUtils;
+import static org.jclouds.concurrent.ConcurrentUtils.*;
 import org.jclouds.domain.Location;
 import org.jclouds.domain.LocationScope;
 import org.jclouds.logging.Logger;
@@ -190,38 +190,40 @@ public class EC2ComputeService implements ComputeService {
                .asType(ec2Size.getInstanceType())// instance size
                .withSecurityGroup(tag)// group I created above
                .withAdditionalInfo(tag);
-
-      Reservation reservation = ec2Client.getInstanceServices().runInstancesInRegion(region, zone,
-               template.getImage().getId(), 1, count, instanceOptions);
-      Iterable<String> ids = Iterables.transform(reservation, instanceToId);
-
-      String idsString = Joiner.on(',').join(ids);
-      logger.debug("<< started instances(%s)", idsString);
-      Iterables.all(reservation, instanceStateRunning);
-      logger.debug("<< running instances(%s)", idsString);
       final Set<NodeMetadata> nodes = Sets.newHashSet();
-      Set<ListenableFuture<Void>> responses = Sets.newHashSet();
+      int nodesToStart = count;
-      for (final NodeMetadata node : Iterables.transform(Iterables.concat(ec2Client
+      while (nodesToStart > 0) {
-               .getInstanceServices().describeInstancesInRegion(region,
+         Reservation reservation = ec2Client.getInstanceServices().runInstancesInRegion(region,
-                        Iterables.toArray(ids, String.class))), runningInstanceToNodeMetadata)) {
+                  zone, template.getImage().getId(), 1, nodesToStart, instanceOptions);
-         responses.add(ConcurrentUtils.makeListenable(executor.submit(new Callable<Void>() {
+         Iterable<String> ids = Iterables.transform(reservation, instanceToId);
-            @Override
-            public Void call() throws Exception {
-               try {
-                  utils.runOptionsOnNode(node, template.getOptions());
-                  logger.debug("<< options applied instance(%s)", node.getId());
-                  nodes.add(node);
-               } catch (Exception e) {
-                  logger.error(e, "<< error applying instance(%s) [%s] destroying ", node.getId(),
-                           e.getMessage());
-                  destroyNode(node);
-               }
-               return null;
-            }
 
-         }), executor));
+         String idsString = Joiner.on(',').join(ids);
+         logger.debug("<< started instances(%s)", idsString);
+         Iterables.all(reservation, instanceStateRunning);
+         logger.debug("<< running instances(%s)", idsString);
+         Map<NodeMetadata, ListenableFuture<Void>> responses = Maps.newHashMap();
+         for (final NodeMetadata node : Iterables.transform(Iterables.concat(ec2Client
+                  .getInstanceServices().describeInstancesInRegion(region,
+                           Iterables.toArray(ids, String.class))), runningInstanceToNodeMetadata)) {
+            responses.put(node, makeListenable(executor.submit(new Callable<Void>() {
+               @Override
+               public Void call() throws Exception {
+                  try {
+                     utils.runOptionsOnNode(node, template.getOptions());
+                     logger.debug("<< options applied instance(%s)", node.getId());
+                     nodes.add(node);
+                  } catch (Exception e) {
+                     logger.error(e, "<< error applying instance(%s) [%s] destroying ", node
+                              .getId(), e.getMessage());
+                     destroyNode(node);
+                  }
+                  return null;
+               }
+
+            }), executor));
+         }
+         nodesToStart = awaitCompletion(responses, executor, null, logger, "nodes").size();
       }
-      ConcurrentUtils.awaitCompletion(responses, executor, null, logger, "nodes");
       return new NodeSetImpl(nodes);
    }
 
@@ -324,18 +326,25 @@ public class EC2ComputeService implements ComputeService {
    @Override
    public void destroyNodesWithTag(String tag) { // TODO parallel
       logger.debug(">> terminating servers by tag(%s)", tag);
-      Set<ListenableFuture<Void>> responses = Sets.newHashSet();
+      Iterable<NodeMetadata> nodesToDestroy = Iterables.filter(doGetNodes(tag),
-      for (final NodeMetadata node : doGetNodes(tag)) {
+               new Predicate<NodeMetadata>() {
-         if (node.getState() != NodeState.TERMINATED)
+                  @Override
-            responses.add(ConcurrentUtils.makeListenable(executor.submit(new Callable<Void>() {
+                  public boolean apply(NodeMetadata input) {
-               @Override
+                     return input.getState() != NodeState.TERMINATED;
-               public Void call() throws Exception {
+
-                  destroyNode(node);
+                  }
-                  return null;
+               });
-               }
+      Map<NodeMetadata, ListenableFuture<Void>> responses = Maps.newHashMap();
-            }), executor));
+      for (final NodeMetadata node : nodesToDestroy) {
+         responses.put(node, makeListenable(executor.submit(new Callable<Void>() {
+            @Override
+            public Void call() throws Exception {
+               destroyNode(node);
+               return null;
+            }
+         }), executor));
       }
-      ConcurrentUtils.awaitCompletion(responses, executor, null, logger, "nodes");
+      awaitCompletion(responses, executor, null, logger, "nodes");
       logger.debug("<< destroyed");
    }
 

aws/core/src/main/java/org/jclouds/aws/handlers/ParseAWSErrorFromXmlContent.java

@@ -34,7 +34,6 @@ import org.jclouds.http.HttpResponse;
 import org.jclouds.http.HttpResponseException;
 import org.jclouds.logging.Logger;
 import org.jclouds.rest.AuthorizationException;
-import org.jclouds.rest.ResourceNotFoundException;
 import org.jclouds.util.Utils;
 
 import com.google.common.io.Closeables;
@@ -58,40 +57,46 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
    }
 
    public void handleError(HttpCommand command, HttpResponse response) {
-      Exception exception = null;
+      Exception exception = new HttpResponseException(command, response);
       try {
+         AWSError error = parseErrorFromContentOrNull(command, response);
          switch (response.getStatusCode()) {
             case 401:
-               exception = new AuthorizationException(command.getRequest().getRequestLine());
+               exception = new AuthorizationException(command.getRequest(), error != null ? error
+                        .getMessage() : response.getStatusLine());
                break;
             case 404:
-               String container = command.getRequest().getEndpoint().getHost();
+               if (!command.getRequest().getMethod().equals("DELETE")) {
-               String key = command.getRequest().getEndpoint().getPath();
+                  String message = error != null ? error.getMessage() : String.format("%s -> %s",
-               if (key == null || key.equals("/"))
+                           command.getRequest().getRequestLine(), response.getStatusLine());
-                  exception = new ContainerNotFoundException(container);
+                  String container = command.getRequest().getEndpoint().getHost();
-               else
+                  String key = command.getRequest().getEndpoint().getPath();
-                  exception = new KeyNotFoundException(container, key);
+                  if (key == null || key.equals("/"))
+                     exception = new ContainerNotFoundException(container, message);
+                  else
+                     exception = new KeyNotFoundException(container, key, message);
+               }
                break;
             default:
-               if (response.getContent() != null) {
+               exception = error != null ? new AWSResponseException(command, response, error)
-                  try {
+                        : new HttpResponseException(command, response);
-                     String content = Utils.toStringAndClose(response.getContent());
-                     if (content.indexOf('<') >= 0) {
-                        AWSError error = utils.parseAWSErrorFromContent(command, response, content);
-                        exception = new AWSResponseException(command, response, error);
-                        if (error.getCode().indexOf("NotFound") >= 0)
-                           exception = new ResourceNotFoundException(exception);
-                     } else {
-                        exception = new HttpResponseException(command, response, content);
-                     }
-                  } catch (IOException e) {
-                     logger.warn(e, "exception reading error from response", response);
-                  }
-               }
          }
       } finally {
          Closeables.closeQuietly(response.getContent());
          command.setException(exception);
       }
    }
+
+   AWSError parseErrorFromContentOrNull(HttpCommand command, HttpResponse response) {
+      if (response.getContent() != null) {
+         try {
+            String content = Utils.toStringAndClose(response.getContent());
+            if (content != null && content.indexOf('<') >= 0)
+               return utils.parseAWSErrorFromContent(command, response, content);
+         } catch (IOException e) {
+            logger.warn(e, "exception reading error from response", response);
+         }
+      }
+      return null;
+   }
 }

aws/core/src/main/java/org/jclouds/aws/s3/S3AsyncClient.java

@@ -48,7 +48,7 @@ import org.jclouds.aws.s3.functions.BindRegionToXmlPayload;
 import org.jclouds.aws.s3.functions.ObjectKey;
 import org.jclouds.aws.s3.functions.ParseObjectFromHeadersAndHttpContent;
 import org.jclouds.aws.s3.functions.ParseObjectMetadataFromHeaders;
-import org.jclouds.aws.s3.functions.ReturnTrueIfBucketAlreadyOwnedByYou;
+import org.jclouds.aws.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYou;
 import org.jclouds.aws.s3.functions.ReturnTrueOn404OrNotFoundFalseIfNotEmpty;
 import org.jclouds.aws.s3.options.CopyObjectOptions;
 import org.jclouds.aws.s3.options.ListBucketOptions;
@@ -66,6 +66,7 @@ import org.jclouds.blobstore.attr.ConsistencyModel;
 import org.jclouds.blobstore.attr.ConsistencyModels;
 import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound;
 import org.jclouds.blobstore.functions.ReturnFalseOnKeyNotFound;
+import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
 import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
 import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404;
 import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
@@ -116,7 +117,7 @@ public interface S3AsyncClient {
     */
    @GET
    @Path("{key}")
-   @ExceptionParser(ThrowKeyNotFoundOn404.class)
+   @ExceptionParser(ReturnNullOnKeyNotFound.class)
    @ResponseParser(ParseObjectFromHeadersAndHttpContent.class)
    ListenableFuture<S3Object> getObject(@HostPrefixParam String bucketName,
             @PathParam("key") String key, GetOptions... options);
@@ -126,7 +127,7 @@ public interface S3AsyncClient {
     */
    @HEAD
    @Path("{key}")
-   @ExceptionParser(ThrowKeyNotFoundOn404.class)
+   @ExceptionParser(ReturnNullOnKeyNotFound.class)
    @ResponseParser(ParseObjectMetadataFromHeaders.class)
    ListenableFuture<ObjectMetadata> headObject(@HostPrefixParam String bucketName,
             @PathParam("key") String key);
@@ -165,7 +166,7 @@ public interface S3AsyncClient {
     */
    @PUT
    @Path("/")
-   @ExceptionParser(ReturnTrueIfBucketAlreadyOwnedByYou.class)
+   @ExceptionParser(ReturnFalseIfBucketAlreadyOwnedByYou.class)
    ListenableFuture<Boolean> putBucketInRegion(
             // TODO endpoint based on region
             @BinderParam(BindRegionToXmlPayload.class) Region region,

aws/core/src/main/java/org/jclouds/aws/s3/S3Client.java

@@ -185,7 +185,7 @@ public interface S3Client {
     * 
     * @param options
     *           for creating your bucket
-    * @return true, if the bucket was created or already exists
+    * @return true, if the bucket was created or false, if the container was already present
     * 
     * @see PutBucketOptions
     * @see <a

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3AsyncBlobStore.java

@@ -18,17 +18,15 @@
  */
 package org.jclouds.aws.s3.blobstore;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.util.concurrent.Futures.compose;
-import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
-import static org.jclouds.concurrent.ConcurrentUtils.convertExceptionToValue;
-import static org.jclouds.concurrent.ConcurrentUtils.makeListenable;
 
 import java.util.SortedSet;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
 import javax.inject.Named;
+import javax.inject.Singleton;
 
 import org.jclouds.Constants;
 import org.jclouds.aws.domain.Region;
@@ -40,27 +38,21 @@ import org.jclouds.aws.s3.blobstore.functions.BucketToResourceMetadata;
 import org.jclouds.aws.s3.blobstore.functions.ContainerToBucketListOptions;
 import org.jclouds.aws.s3.blobstore.functions.ObjectToBlob;
 import org.jclouds.aws.s3.blobstore.functions.ObjectToBlobMetadata;
-import org.jclouds.aws.s3.blobstore.internal.BaseS3BlobStore;
 import org.jclouds.aws.s3.domain.BucketMetadata;
 import org.jclouds.aws.s3.domain.ListBucketResponse;
 import org.jclouds.aws.s3.domain.ObjectMetadata;
 import org.jclouds.aws.s3.options.ListBucketOptions;
-import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.aws.s3.util.S3Utils;
-import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
-import org.jclouds.blobstore.domain.ListResponse;
 import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.Blob.Factory;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
-import org.jclouds.blobstore.domain.internal.ListResponseImpl;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
+import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
 import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
+import org.jclouds.blobstore.util.BlobStoreUtils;
-import org.jclouds.blobstore.strategy.GetDirectoryStrategy;
-import org.jclouds.blobstore.strategy.MkdirStrategy;
 import org.jclouds.http.options.GetOptions;
-import org.jclouds.logging.Logger.LoggerFactory;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
@@ -70,34 +62,52 @@ import com.google.common.util.concurrent.ListenableFuture;
  * 
  * @author Adrian Cole
  */
-public class S3AsyncBlobStore extends BaseS3BlobStore implements AsyncBlobStore {
+@Singleton
+public class S3AsyncBlobStore extends BaseAsyncBlobStore {
+
+   private final S3AsyncClient async;
+   private final S3Client sync;
+   private final BucketToResourceMetadata bucket2ResourceMd;
+   private final ContainerToBucketListOptions container2BucketListOptions;
+   private final BlobToHttpGetOptions blob2ObjectGetOptions;
+   private final BucketToResourceList bucket2ResourceList;
+   private final ObjectToBlob object2Blob;
+   private final BlobToObject blob2Object;
+   private final ObjectToBlobMetadata object2BlobMd;
 
    @Inject
-   public S3AsyncBlobStore(S3AsyncClient async, S3Client sync, Factory blobFactory,
+   S3AsyncBlobStore(BlobStoreUtils blobUtils,
-            LoggerFactory logFactory, ClearListStrategy clearContainerStrategy,
+            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, S3AsyncClient async,
-            ObjectToBlobMetadata object2BlobMd, ObjectToBlob object2Blob, BlobToObject blob2Object,
+            S3Client sync, BucketToResourceMetadata bucket2ResourceMd,
             ContainerToBucketListOptions container2BucketListOptions,
-            BlobToHttpGetOptions blob2ObjectGetOptions, GetDirectoryStrategy getDirectoryStrategy,
+            BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
-            MkdirStrategy mkdirStrategy, BucketToResourceMetadata bucket2ResourceMd,
+            BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
-            BucketToResourceList bucket2ResourceList,
+            ObjectToBlobMetadata object2BlobMd) {
-            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service) {
+      super(blobUtils, service);
-      super(async, sync, blobFactory, logFactory, clearContainerStrategy, object2BlobMd,
+      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
-               object2Blob, blob2Object, container2BucketListOptions, blob2ObjectGetOptions,
+      this.async = checkNotNull(async, "async");
-               getDirectoryStrategy, mkdirStrategy, bucket2ResourceMd, bucket2ResourceList, service);
+      this.sync = checkNotNull(sync, "sync");
+      this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd");
+      this.container2BucketListOptions = checkNotNull(container2BucketListOptions,
+               "container2BucketListOptions");
+      this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList");
+      this.object2Blob = checkNotNull(object2Blob, "object2Blob");
+      this.blob2Object = checkNotNull(blob2Object, "blob2Object");
+      this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
    }
 
    /**
     * This implementation invokes {@link S3AsyncClient#listOwnedBuckets}
     */
    @Override
-   public ListenableFuture<? extends ListResponse<? extends StorageMetadata>> list() {
+   public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list() {
       return compose(
                async.listOwnedBuckets(),
-               new Function<SortedSet<BucketMetadata>, org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata>>() {
+               new Function<SortedSet<BucketMetadata>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
-                  public org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata> apply(
+                  public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(
                            SortedSet<BucketMetadata> from) {
-                     return new ListResponseImpl<StorageMetadata>(Iterables.transform(from,
+                     return new PageSetImpl<StorageMetadata>(Iterables.transform(from,
-                              bucket2ResourceMd), null, null, false);
+                              bucket2ResourceMd), null);
                   }
                }, service);
    }
@@ -126,19 +136,6 @@ public class S3AsyncBlobStore extends BaseS3BlobStore implements AsyncBlobStore
       return async.putBucketInRegion(Region.fromValue(location), container);
    }
 
-   /**
-    * This implementation invokes
-    * {@link #list(String,org.jclouds.blobstore.options.ListContainerOptions)}
-    * 
-    * @param container
-    *           bucket name
-    */
-   @Override
-   public ListenableFuture<? extends ListContainerResponse<? extends StorageMetadata>> list(
-            String container) {
-      return this.list(container, org.jclouds.blobstore.options.ListContainerOptions.NONE);
-   }
-
    /**
     * This implementation invokes {@link S3AsyncClient#listBucket}
     * 
@@ -146,95 +143,18 @@ public class S3AsyncBlobStore extends BaseS3BlobStore implements AsyncBlobStore
     *           bucket name
     */
    @Override
-   public ListenableFuture<? extends ListContainerResponse<? extends StorageMetadata>> list(
+   public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(String container,
-            String container, ListContainerOptions options) {
+            ListContainerOptions options) {
       ListBucketOptions httpOptions = container2BucketListOptions.apply(options);
       ListenableFuture<ListBucketResponse> returnVal = async.listBucket(container, httpOptions);
       return compose(returnVal, bucket2ResourceList, service);
    }
 
    /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
+    * This implementation invokes {@link S3Utils#deleteAndVerifyContainerGone}
-    * {@link ListContainerOptions#recursive} option.
-    * 
-    * @param container
-    *           bucket name
     */
-   @Override
+   protected boolean deleteAndVerifyContainerGone(final String container) {
-   public ListenableFuture<Void> clearContainer(final String container) {
+      return S3Utils.deleteAndVerifyContainerGone(sync, container);
-      return makeListenable(service.submit(new Callable<Void>() {
-
-         public Void call() throws Exception {
-            clearContainerStrategy.execute(container, recursive());
-            return null;
-         }
-
-      }), service);
-   }
-
-   /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
-    * {@link ListContainerOptions#recursive} option. Then, it invokes
-    * {@link S3AsyncClient#deleteBucketIfEmpty}
-    * 
-    * @param container
-    *           bucket name
-    */
-   @Override
-   public ListenableFuture<Void> deleteContainer(final String container) {
-      return makeListenable(service.submit(new Callable<Void>() {
-
-         public Void call() throws Exception {
-            clearContainerStrategy.execute(container, recursive());
-            async.deleteBucketIfEmpty(container).get();
-            return null;
-         }
-
-      }), service);
-   }
-
-   /**
-    * This implementation invokes {@link GetDirectoryStrategy#execute}
-    * 
-    * @param container
-    *           bucket name
-    * @param directory
-    *           virtual path
-    */
-   @Override
-   public ListenableFuture<Boolean> directoryExists(final String container, final String directory) {
-      return makeListenable(service.submit(new Callable<Boolean>() {
-
-         public Boolean call() throws Exception {
-            try {
-               getDirectoryStrategy.execute(container, directory);
-               return true;
-            } catch (KeyNotFoundException e) {
-               return false;
-            }
-         }
-
-      }), service);
-   }
-
-   /**
-    * This implementation invokes {@link MkdirStrategy#execute}
-    * 
-    * @param container
-    *           bucket name
-    * @param directory
-    *           virtual path
-    */
-   @Override
-   public ListenableFuture<Void> createDirectory(final String container, final String directory) {
-      return makeListenable(service.submit(new Callable<Void>() {
-
-         public Void call() throws Exception {
-            mkdirStrategy.execute(container, directory);
-            return null;
-         }
-
-      }), service);
    }
 
    /**
@@ -260,29 +180,15 @@ public class S3AsyncBlobStore extends BaseS3BlobStore implements AsyncBlobStore
     */
    @Override
    public ListenableFuture<BlobMetadata> blobMetadata(String container, String key) {
-      return compose(convertExceptionToValue(async.headObject(container, key),
+      return compose(async.headObject(container, key),
-               KeyNotFoundException.class, null), new Function<ObjectMetadata, BlobMetadata>() {
+               new Function<ObjectMetadata, BlobMetadata>() {
 
-         @Override
+                  @Override
-         public BlobMetadata apply(ObjectMetadata from) {
+                  public BlobMetadata apply(ObjectMetadata from) {
-            return object2BlobMd.apply(from);
+                     return object2BlobMd.apply(from);
-         }
+                  }
 
-      }, service);
+               }, service);
-   }
-
-   /**
-    * This implementation invokes
-    * {@link #getBlob(String,String,org.jclouds.blobstore.options.GetOptions)}
-    * 
-    * @param container
-    *           bucket name
-    * @param key
-    *           object key
-    */
-   @Override
-   public ListenableFuture<Blob> getBlob(String container, String key) {
-      return getBlob(container, key, org.jclouds.blobstore.options.GetOptions.NONE);
    }
 
    /**
@@ -297,8 +203,7 @@ public class S3AsyncBlobStore extends BaseS3BlobStore implements AsyncBlobStore
    public ListenableFuture<Blob> getBlob(String container, String key,
             org.jclouds.blobstore.options.GetOptions options) {
       GetOptions httpOptions = blob2ObjectGetOptions.apply(options);
-      return compose(convertExceptionToValue(async.getObject(container, key, httpOptions),
+      return compose(async.getObject(container, key, httpOptions), object2Blob, service);
-               KeyNotFoundException.class, null), object2Blob, service);
    }
 
    /**

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java

@@ -18,18 +18,14 @@
  */
 package org.jclouds.aws.s3.blobstore;
 
-import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
+import static com.google.common.base.Preconditions.checkNotNull;
-import static org.jclouds.blobstore.util.BlobStoreUtils.keyNotFoundToNullOrPropagate;
 
 import java.util.SortedSet;
-import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
-import javax.inject.Named;
+import javax.inject.Singleton;
 
-import org.jclouds.Constants;
 import org.jclouds.aws.domain.Region;
-import org.jclouds.aws.s3.S3AsyncClient;
 import org.jclouds.aws.s3.S3Client;
 import org.jclouds.aws.s3.blobstore.functions.BlobToObject;
 import org.jclouds.aws.s3.blobstore.functions.BucketToResourceList;
@@ -37,55 +33,68 @@ import org.jclouds.aws.s3.blobstore.functions.BucketToResourceMetadata;
 import org.jclouds.aws.s3.blobstore.functions.ContainerToBucketListOptions;
 import org.jclouds.aws.s3.blobstore.functions.ObjectToBlob;
 import org.jclouds.aws.s3.blobstore.functions.ObjectToBlobMetadata;
-import org.jclouds.aws.s3.blobstore.internal.BaseS3BlobStore;
 import org.jclouds.aws.s3.domain.BucketMetadata;
 import org.jclouds.aws.s3.options.ListBucketOptions;
-import org.jclouds.blobstore.BlobStore;
+import org.jclouds.aws.s3.util.S3Utils;
-import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
-import org.jclouds.blobstore.domain.ListResponse;
 import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.Blob.Factory;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
-import org.jclouds.blobstore.domain.internal.ListResponseImpl;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
+import org.jclouds.blobstore.internal.BaseBlobStore;
 import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
+import org.jclouds.blobstore.util.BlobStoreUtils;
-import org.jclouds.blobstore.strategy.GetDirectoryStrategy;
-import org.jclouds.blobstore.strategy.MkdirStrategy;
 import org.jclouds.http.options.GetOptions;
-import org.jclouds.logging.Logger.LoggerFactory;
+import org.jclouds.util.Utils;
 
 import com.google.common.base.Function;
+import com.google.common.base.Supplier;
 import com.google.common.collect.Iterables;
 
-public class S3BlobStore extends BaseS3BlobStore implements BlobStore {
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class S3BlobStore extends BaseBlobStore {
+   private final S3Client sync;
+   private final BucketToResourceMetadata bucket2ResourceMd;
+   private final ContainerToBucketListOptions container2BucketListOptions;
+   private final BucketToResourceList bucket2ResourceList;
+   private final ObjectToBlob object2Blob;
+   private final BlobToObject blob2Object;
+   private final ObjectToBlobMetadata object2BlobMd;
+   private final BlobToHttpGetOptions blob2ObjectGetOptions;
 
    @Inject
-   public S3BlobStore(S3AsyncClient async, S3Client sync, Factory blobFactory,
+   S3BlobStore(BlobStoreUtils blobUtils, S3Client sync, BucketToResourceMetadata bucket2ResourceMd,
-            LoggerFactory logFactory, ClearListStrategy clearContainerStrategy,
-            ObjectToBlobMetadata object2BlobMd, ObjectToBlob object2Blob, BlobToObject blob2Object,
             ContainerToBucketListOptions container2BucketListOptions,
-            BlobToHttpGetOptions blob2ObjectGetOptions, GetDirectoryStrategy getDirectoryStrategy,
+            BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
-            MkdirStrategy mkdirStrategy, BucketToResourceMetadata bucket2ResourceMd,
+            BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
-            BucketToResourceList bucket2ResourceList,
+            ObjectToBlobMetadata object2BlobMd) {
-            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service) {
+      super(blobUtils);
-      super(async, sync, blobFactory, logFactory, clearContainerStrategy, object2BlobMd,
+      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
-               object2Blob, blob2Object, container2BucketListOptions, blob2ObjectGetOptions,
+      this.sync = checkNotNull(sync, "sync");
-               getDirectoryStrategy, mkdirStrategy, bucket2ResourceMd, bucket2ResourceList, service);
+      this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd");
+      this.container2BucketListOptions = checkNotNull(container2BucketListOptions,
+               "container2BucketListOptions");
+      this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList");
+      this.object2Blob = checkNotNull(object2Blob, "object2Blob");
+      this.blob2Object = checkNotNull(blob2Object, "blob2Object");
+      this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
    }
 
    /**
     * This implementation invokes {@link S3Client#listOwnedBuckets}
     */
    @Override
-   public ListResponse<? extends StorageMetadata> list() {
+   public PageSet<? extends StorageMetadata> list() {
-      return new Function<SortedSet<BucketMetadata>, org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata>>() {
+      return new Function<SortedSet<BucketMetadata>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
-         public org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata> apply(
+         public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(
                   SortedSet<BucketMetadata> from) {
-            return new ListResponseImpl<StorageMetadata>(Iterables.transform(from,
+            return new PageSetImpl<StorageMetadata>(Iterables.transform(from, bucket2ResourceMd),
-                     bucket2ResourceMd), null, null, false);
+                     null);
          }
       }.apply(sync.listOwnedBuckets());
    }
@@ -111,19 +120,7 @@ public class S3BlobStore extends BaseS3BlobStore implements BlobStore {
     */
    @Override
    public boolean createContainerInLocation(String location, String container) {
-      return sync.putBucketInRegion(Region.DEFAULT, container);// TODO parameterize
+      return sync.putBucketInRegion(Region.fromValue(location), container);
-   }
-
-   /**
-    * This implementation invokes
-    * {@link #list(String,org.jclouds.blobstore.options.ListContainerOptions)}
-    * 
-    * @param container
-    *           bucket name
-    */
-   @Override
-   public ListContainerResponse<? extends StorageMetadata> list(String container) {
-      return this.list(container, org.jclouds.blobstore.options.ListContainerOptions.NONE);
    }
 
    /**
@@ -133,67 +130,39 @@ public class S3BlobStore extends BaseS3BlobStore implements BlobStore {
     *           bucket name
     */
    @Override
-   public ListContainerResponse<? extends StorageMetadata> list(String container,
+   public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions optionsList) {
-            ListContainerOptions optionsList) {
       ListBucketOptions httpOptions = container2BucketListOptions.apply(optionsList);
       return bucket2ResourceList.apply(sync.listBucket(container, httpOptions));
    }
 
    /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
+    * This implementation invokes {@link #deleteAndEnsurePathGone}
-    * {@link ListContainerOptions#recursive} option.
-    * 
-    * @param container
-    *           bucket name
-    */
-   @Override
-   public void clearContainer(String container) {
-      clearContainerStrategy.execute(container, recursive());
-   }
-
-   /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
-    * {@link ListContainerOptions#recursive} option. Then, it invokes
-    * {@link S3Client#deleteBucketIfEmpty}
     * 
     * @param container
     *           bucket name
     */
    @Override
    public void deleteContainer(String container) {
-      clearContainer(container);
+      deleteAndEnsurePathGone(container);
-      sync.deleteBucketIfEmpty(container);
-   }
-   
-   /**
-    * This implementation invokes {@link GetDirectoryStrategy#execute}
-    * 
-    * @param container
-    *           bucket name
-    * @param directory
-    *           virtual path
-    */
-   @Override
-   public boolean directoryExists(String containerName, String directory) {
-      try {
-         getDirectoryStrategy.execute(containerName, directory);
-         return true;
-      } catch (KeyNotFoundException e) {
-         return false;
-      }
    }
 
    /**
-    * This implementation invokes {@link MkdirStrategy#execute}
+    * This implementation invokes {@link #clearContainer} then {@link S3Client#deleteBucketIfEmpty}
-    * 
+    * until it is true.
-    * @param container
-    *           bucket name
-    * @param directory
-    *           virtual path
     */
-   @Override
+   public void deleteAndEnsurePathGone(final String container) {
-   public void createDirectory(String containerName, String directory) {
+      try {
-      mkdirStrategy.execute(containerName, directory);
+         if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
+            public Boolean get() {
+               clearContainer(container);
+               return sync.deleteBucketIfEmpty(container);
+            }
+         }, 30000)) {
+            throw new IllegalStateException(container + " still exists after deleting!");
+         }
+      } catch (InterruptedException e) {
+         new IllegalStateException(container + " interrupted during deletion!", e);
+      }
    }
 
    /**
@@ -219,25 +188,8 @@ public class S3BlobStore extends BaseS3BlobStore implements BlobStore {
     */
    @Override
    public BlobMetadata blobMetadata(String container, String key) {
-      try {
+      return object2BlobMd.apply(sync.headObject(container, key));
-         return object2BlobMd.apply(sync.headObject(container, key));
-      } catch (Exception e) {
-         return keyNotFoundToNullOrPropagate(e);
-      }
-   }
 
-   /**
-    * This implementation invokes
-    * {@link #getBlob(String,String,org.jclouds.blobstore.options.GetOptions)}
-    * 
-    * @param container
-    *           bucket name
-    * @param key
-    *           object key
-    */
-   @Override
-   public Blob getBlob(String container, String key) {
-      return getBlob(container, key, org.jclouds.blobstore.options.GetOptions.NONE);
    }
 
    /**
@@ -252,11 +204,7 @@ public class S3BlobStore extends BaseS3BlobStore implements BlobStore {
    public Blob getBlob(String container, String key,
             org.jclouds.blobstore.options.GetOptions optionsList) {
       GetOptions httpOptions = blob2ObjectGetOptions.apply(optionsList);
-      try {
+      return object2Blob.apply(sync.getObject(container, key, httpOptions));
-         return object2Blob.apply(sync.getObject(container, key, httpOptions));
-      } catch (Exception e) {
-         return keyNotFoundToNullOrPropagate(e);
-      }
    }
 
    /**
@@ -285,4 +233,10 @@ public class S3BlobStore extends BaseS3BlobStore implements BlobStore {
       sync.deleteObject(container, key);
    }
 
+   /**
+    * This implementation invokes {@link S3Utils#deleteAndVerifyContainerGone}
+    */
+   protected boolean deleteAndVerifyContainerGone(final String container) {
+      return S3Utils.deleteAndVerifyContainerGone(sync, container);
+   }
 }

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/BlobToObject.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.aws.s3.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -46,7 +48,7 @@ public class BlobToObject implements Function<Blob, S3Object> {
       S3Object object = objectProvider.create(blob2ObjectMd.apply(from.getMetadata()));
       if (from.getContentLength() != null)
          object.setContentLength(from.getContentLength());
-      object.setPayload(from.getPayload());
+      object.setPayload(checkNotNull(from.getPayload(), "payload: " + from));
       object.setAllHeaders(from.getAllHeaders());
       return object;
    }

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/BucketToResourceList.java

@@ -18,18 +18,21 @@
  */
 package org.jclouds.aws.s3.blobstore.functions;
 
-import java.util.SortedSet;
+import java.util.Map;
+import java.util.Set;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
 import org.jclouds.aws.s3.domain.ListBucketResponse;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.internal.ListContainerResponseImpl;
+import org.jclouds.blobstore.domain.StorageType;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
 /**
@@ -37,10 +40,17 @@ import com.google.common.collect.Sets;
  */
 @Singleton
 public class BucketToResourceList implements
-         Function<ListBucketResponse, ListContainerResponse<? extends StorageMetadata>> {
+         Function<ListBucketResponse, PageSet<? extends StorageMetadata>> {
    private final ObjectToBlobMetadata object2blobMd;
    private final CommonPrefixesToResourceMetadata prefix2ResourceMd;
 
+   protected final Function<StorageMetadata, String> indexer = new Function<StorageMetadata, String>() {
+      @Override
+      public String apply(StorageMetadata from) {
+         return from.getName();
+      }
+   };
+
    @Inject
    public BucketToResourceList(ObjectToBlobMetadata object2blobMd,
             CommonPrefixesToResourceMetadata prefix2ResourceMd) {
@@ -48,11 +58,17 @@ public class BucketToResourceList implements
       this.prefix2ResourceMd = prefix2ResourceMd;
    }
 
-   public ListContainerResponse<? extends StorageMetadata> apply(ListBucketResponse from) {
+   public PageSet<? extends StorageMetadata> apply(ListBucketResponse from) {
-      SortedSet<StorageMetadata> contents = Sets.newTreeSet(Iterables.concat(Iterables.transform(
+      Set<StorageMetadata> contents = Sets.<StorageMetadata> newHashSet(Iterables.transform(from,
-               from, object2blobMd), prefix2ResourceMd.apply(from.getCommonPrefixes())));
+               object2blobMd));
-      return new ListContainerResponseImpl<StorageMetadata>(contents, from.getPrefix(), from.getMarker(),
-               from.getMaxKeys(), from.isTruncated());
 
+      Map<String, StorageMetadata> nameToMd = Maps.uniqueIndex(contents, indexer);
+      for (String prefix : from.getCommonPrefixes()) {
+         prefix = prefix.endsWith("/") ? prefix.substring(0, prefix.lastIndexOf('/')) : prefix;
+         if (!nameToMd.containsKey(prefix)
+                  || nameToMd.get(prefix).getType() != StorageType.RELATIVE_PATH)
+            contents.add(prefix2ResourceMd.apply(prefix));
+      }
+      return new PageSetImpl<StorageMetadata>(contents, from.getNextMarker());
    }
 }

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/CommonPrefixesToResourceMetadata.java

@@ -26,24 +26,18 @@ import org.jclouds.blobstore.domain.StorageType;
 import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl;
 
 import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
 
 /**
  * @author Adrian Cole
  */
 @Singleton
-public class CommonPrefixesToResourceMetadata implements
+public class CommonPrefixesToResourceMetadata implements Function<String, StorageMetadata> {
-         Function<Iterable<String>, Iterable<StorageMetadata>> {
-   public Iterable<StorageMetadata> apply(
 
-   Iterable<String> prefixes) {
+   public StorageMetadata apply(String from) {
-      return Iterables.transform(prefixes, new Function<String, StorageMetadata>() {
+      MutableStorageMetadata returnVal = new MutableStorageMetadataImpl();
-         public StorageMetadata apply(String from) {
+      returnVal.setType(StorageType.RELATIVE_PATH);
-            MutableStorageMetadata returnVal = new MutableStorageMetadataImpl();
+      returnVal.setName(from);
-            returnVal.setType(StorageType.RELATIVE_PATH);
+      return returnVal;
-            returnVal.setName(from);
-            return returnVal;
-         }
-      });
    }
+
 }

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/ContainerToBucketListOptions.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.aws.s3.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Singleton;
 
 import org.jclouds.aws.s3.options.ListBucketOptions;
@@ -32,23 +34,22 @@ import com.google.common.base.Function;
 public class ContainerToBucketListOptions implements
          Function<ListContainerOptions, ListBucketOptions> {
    public ListBucketOptions apply(ListContainerOptions from) {
+      checkNotNull(from, "set options to instance NONE instead of passing null");
       ListBucketOptions httpOptions = new ListBucketOptions();
-      if (from != null && from != ListContainerOptions.NONE) {
+      if (!from.isRecursive()) {
-         if (!from.isRecursive()) {
+         httpOptions.delimiter("/");
-            httpOptions.delimiter("/");
+      }
-         }
+      if (from.getDir() != null) {// TODO unit test
-         if (from.getDir() != null) {// TODO unit test
+         String path = from.getDir();
-            String path = from.getDir();
+         if (!path.endsWith("/"))
-            if (!path.endsWith("/"))
+            path = path + "/";
-               path = path + "/";
+         httpOptions.withPrefix(path);
-            httpOptions.withPrefix(path);
+      }
-         }
+      if (from.getMarker() != null) {
-         if (from.getMarker() != null) {
+         httpOptions.afterMarker(from.getMarker());
-            httpOptions.afterMarker(from.getMarker());
+      }
-         }
+      if (from.getMaxResults() != null) {
-         if (from.getMaxResults() != null) {
+         httpOptions.maxResults(from.getMaxResults());
-            httpOptions.maxResults(from.getMaxResults());
-         }
       }
       return httpOptions;
    }

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/ObjectToBlob.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.aws.s3.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -47,8 +49,7 @@ public class ObjectToBlob implements Function<S3Object, Blob> {
       Blob blob = blobFactory.create(object2BlobMd.apply(from.getMetadata()));
       if (from.getContentLength() != null)
          blob.setContentLength(from.getContentLength());
-      if (from.getPayload() != null)
+      blob.setPayload(checkNotNull(from.getPayload(), "payload: " + from));
-         blob.setPayload(from.getPayload());
       blob.setAllHeaders(from.getAllHeaders());
       return blob;
    }

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/ObjectToBlobMetadata.java

@@ -25,7 +25,7 @@ import org.jclouds.aws.s3.domain.ObjectMetadata;
 import org.jclouds.blobstore.domain.MutableBlobMetadata;
 import org.jclouds.blobstore.domain.StorageType;
 import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
-import org.jclouds.blobstore.strategy.IsDirectoryStrategy;
+import org.jclouds.blobstore.strategy.IfDirectoryReturnNameStrategy;
 
 import com.google.common.base.Function;
 
@@ -34,28 +34,32 @@ import com.google.common.base.Function;
  */
 @Singleton
 public class ObjectToBlobMetadata implements Function<ObjectMetadata, MutableBlobMetadata> {
-   private final IsDirectoryStrategy isDirectoryStrategy;
+   private final IfDirectoryReturnNameStrategy ifDirectoryReturnName;
 
    @Inject
-   public ObjectToBlobMetadata(IsDirectoryStrategy isDirectoryStrategy) {
+   public ObjectToBlobMetadata(IfDirectoryReturnNameStrategy ifDirectoryReturnName) {
-      this.isDirectoryStrategy = isDirectoryStrategy;
+      this.ifDirectoryReturnName = ifDirectoryReturnName;
    }
 
    public MutableBlobMetadata apply(ObjectMetadata from) {
       if (from == null)
          return null;
       MutableBlobMetadata to = new MutableBlobMetadataImpl();
-      to.setContentMD5(from.getContentMD5());
+      if (from.getContentMD5() != null)
+         to.setContentMD5(from.getContentMD5());
       if (from.getContentType() != null)
          to.setContentType(from.getContentType());
       to.setETag(from.getETag());
       to.setName(from.getKey());
       to.setSize(from.getSize());
-      to.setType(StorageType.BLOB);
       to.setLastModified(from.getLastModified());
       to.setUserMetadata(from.getUserMetadata());
-      if (isDirectoryStrategy.execute(to)) {
+      String directoryName = ifDirectoryReturnName.execute(to);
+      if (directoryName != null) {
+         to.setName(directoryName);
          to.setType(StorageType.RELATIVE_PATH);
+      } else {
+         to.setType(StorageType.BLOB);
       }
       return to;
    }

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/internal/BaseS3BlobStore.java

@@ -1,92 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
- *
- * ====================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.blobstore.internal;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.concurrent.ExecutorService;
-
-import javax.inject.Inject;
-
-import org.jclouds.aws.s3.S3AsyncClient;
-import org.jclouds.aws.s3.S3Client;
-import org.jclouds.aws.s3.blobstore.functions.BlobToObject;
-import org.jclouds.aws.s3.blobstore.functions.BucketToResourceList;
-import org.jclouds.aws.s3.blobstore.functions.BucketToResourceMetadata;
-import org.jclouds.aws.s3.blobstore.functions.ContainerToBucketListOptions;
-import org.jclouds.aws.s3.blobstore.functions.ObjectToBlob;
-import org.jclouds.aws.s3.blobstore.functions.ObjectToBlobMetadata;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
-import org.jclouds.blobstore.strategy.GetDirectoryStrategy;
-import org.jclouds.blobstore.strategy.MkdirStrategy;
-import org.jclouds.logging.Logger.LoggerFactory;
-
-public class BaseS3BlobStore {
-   protected final S3AsyncClient async;
-   protected final S3Client sync;
-   protected final Blob.Factory blobFactory;
-   protected final LoggerFactory logFactory;
-   protected final ClearListStrategy clearContainerStrategy;
-   protected final ObjectToBlobMetadata object2BlobMd;
-   protected final ObjectToBlob object2Blob;
-   protected final BlobToObject blob2Object;
-   protected final ContainerToBucketListOptions container2BucketListOptions;
-   protected final BlobToHttpGetOptions blob2ObjectGetOptions;
-   protected final BucketToResourceMetadata bucket2ResourceMd;
-   protected final BucketToResourceList bucket2ResourceList;
-   protected final ExecutorService service;
-   protected final GetDirectoryStrategy getDirectoryStrategy;
-   protected final MkdirStrategy mkdirStrategy;
-
-   @Inject
-   protected BaseS3BlobStore(S3AsyncClient async, S3Client sync, Blob.Factory blobFactory,
-            LoggerFactory logFactory, ClearListStrategy clearContainerStrategy,
-            ObjectToBlobMetadata object2BlobMd, ObjectToBlob object2Blob, BlobToObject blob2Object,
-            ContainerToBucketListOptions container2BucketListOptions,
-            BlobToHttpGetOptions blob2ObjectGetOptions,
-            GetDirectoryStrategy getDirectoryStrategy, MkdirStrategy mkdirStrategy,
-            BucketToResourceMetadata bucket2ResourceMd, BucketToResourceList bucket2ResourceList,
-            ExecutorService service) {
-      this.async = checkNotNull(async, "async");
-      this.sync = checkNotNull(sync, "sync");
-      this.blobFactory = checkNotNull(blobFactory, "blobFactory");
-      this.logFactory = checkNotNull(logFactory, "logFactory");
-      this.clearContainerStrategy = checkNotNull(clearContainerStrategy, "clearContainerStrategy");
-      this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
-      this.object2Blob = checkNotNull(object2Blob, "object2Blob");
-      this.blob2Object = checkNotNull(blob2Object, "blob2Object");
-      this.container2BucketListOptions = checkNotNull(container2BucketListOptions,
-               "container2BucketListOptions");
-      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
-      this.getDirectoryStrategy = checkNotNull(getDirectoryStrategy, "getDirectoryStrategy");
-      this.mkdirStrategy = checkNotNull(mkdirStrategy, "mkdirStrategy");
-      this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd");
-      this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList");
-      this.service = checkNotNull(service, "service");
-   }
-
-   public Blob newBlob(String name) {
-      Blob blob = blobFactory.create(null);
-      blob.getMetadata().setName(name);
-      return blob;
-   }
-
-}

aws/core/src/main/java/org/jclouds/aws/s3/domain/ListBucketResponse.java

@@ -18,7 +18,7 @@
  */
 package org.jclouds.aws.s3.domain;
 
-import java.util.SortedSet;
+import java.util.Set;
 
 /**
  * A container that provides namespace, access control and aggregation of {@link S3Object}s
@@ -43,7 +43,7 @@ import java.util.SortedSet;
  * @author Adrian Cole
  * @see <a href="http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html" />
  */
-public interface ListBucketResponse extends SortedSet<ObjectMetadata> {
+public interface ListBucketResponse extends Set<ObjectMetadata> {
 
    /**
     * Limits the response to keys which begin with the indicated prefix. You can use prefixes to
@@ -57,6 +57,8 @@ public interface ListBucketResponse extends SortedSet<ObjectMetadata> {
     * lexicographically after marker. This is convenient for pagination: To get the next page of
     * results use the last key of the current page as the marker.
     */
+   String getNextMarker();
+
    String getMarker();
 
    /**
@@ -93,12 +95,11 @@ public interface ListBucketResponse extends SortedSet<ObjectMetadata> {
     * 
     * @see org.jclouds.aws.s3.options.ListBucketOptions#getPrefix()
     */
-   SortedSet<String> getCommonPrefixes();
+   Set<String> getCommonPrefixes();
 
    /**
-    * name of the Bucket FIXME Comment this
+    * name of the Bucket
-    * 
-    * @return
     */
    String getName();
+
 }

aws/core/src/main/java/org/jclouds/aws/s3/domain/internal/ListBucketResponseImpl.java

@@ -18,8 +18,8 @@
  */
 package org.jclouds.aws.s3.domain.internal;
 
-import java.util.SortedSet;
+import java.util.HashSet;
-import java.util.TreeSet;
+import java.util.Set;
 
 import org.jclouds.aws.s3.domain.ListBucketResponse;
 import org.jclouds.aws.s3.domain.ObjectMetadata;
@@ -31,25 +31,27 @@ import com.google.common.collect.Iterables;
  * @author Adrian Cole
  * 
  */
-public class TreeSetListBucketResponse extends TreeSet<ObjectMetadata> implements
+public class ListBucketResponseImpl extends HashSet<ObjectMetadata> implements
          ListBucketResponse {
    /** The serialVersionUID */
    private static final long serialVersionUID = -4475709781001190244L;
-   private final String name;
+   protected final String name;
    protected final String prefix;
    protected final int maxKeys;
-   private final String delimiter;
+   protected final String delimiter;
    protected final String marker;
-   private final SortedSet<String> commonPrefixes;
+   protected final String nextMarker;
+   protected final Set<String> commonPrefixes;
    protected final boolean truncated;
 
-   public TreeSetListBucketResponse(String name, Iterable<ObjectMetadata> contents, String prefix,
+   public ListBucketResponseImpl(String name, Iterable<ObjectMetadata> contents, String prefix,
-            String marker, int maxKeys, String delimiter, boolean isTruncated,
+            String marker, String nextMarker, int maxKeys, String delimiter, boolean isTruncated,
-            SortedSet<String> commonPrefixes) {
+            Set<String> commonPrefixes) {
       Iterables.addAll(this, contents);
       this.name = name;
       this.prefix = prefix;
       this.marker = marker;
+      this.nextMarker = nextMarker;
       this.maxKeys = maxKeys;
       this.truncated = isTruncated;
       this.delimiter = delimiter;
@@ -59,13 +61,15 @@ public class TreeSetListBucketResponse extends TreeSet<ObjectMetadata> implement
    /**
     * {@inheritDoc}
     */
-   public SortedSet<String> getCommonPrefixes() {
+   @Override
+   public Set<String> getCommonPrefixes() {
       return commonPrefixes;
    }
 
    /**
     * {@inheritDoc}
     */
+   @Override
    public String getDelimiter() {
       return delimiter;
    }
@@ -73,6 +77,7 @@ public class TreeSetListBucketResponse extends TreeSet<ObjectMetadata> implement
    /**
     * {@inheritDoc}
     */
+   @Override
    public String getMarker() {
       return marker;
    }
@@ -80,6 +85,15 @@ public class TreeSetListBucketResponse extends TreeSet<ObjectMetadata> implement
    /**
     * {@inheritDoc}
     */
+   @Override
+   public String getNextMarker() {
+      return nextMarker;
+   }
+
+   /**
+    * {@inheritDoc}
+    */
+   @Override
    public int getMaxKeys() {
       return maxKeys;
    }
@@ -87,6 +101,7 @@ public class TreeSetListBucketResponse extends TreeSet<ObjectMetadata> implement
    /**
     * {@inheritDoc}
     */
+   @Override
    public String getPrefix() {
       return prefix;
    }
@@ -94,6 +109,7 @@ public class TreeSetListBucketResponse extends TreeSet<ObjectMetadata> implement
    /**
     * {@inheritDoc}
     */
+   @Override
    public boolean isTruncated() {
       return truncated;
    }
@@ -101,6 +117,7 @@ public class TreeSetListBucketResponse extends TreeSet<ObjectMetadata> implement
    /**
     * {@inheritDoc}
     */
+   @Override
    public String getName() {
       return name;
    }
@@ -127,7 +144,7 @@ public class TreeSetListBucketResponse extends TreeSet<ObjectMetadata> implement
          return false;
       if (getClass() != obj.getClass())
          return false;
-      TreeSetListBucketResponse other = (TreeSetListBucketResponse) obj;
+      ListBucketResponseImpl other = (ListBucketResponseImpl) obj;
       if (commonPrefixes == null) {
          if (other.commonPrefixes != null)
             return false;

aws/core/src/main/java/org/jclouds/aws/s3/functions/ParseObjectFromHeadersAndHttpContent.java

@@ -23,12 +23,10 @@ import javax.ws.rs.core.HttpHeaders;
 
 import org.jclouds.aws.s3.domain.S3Object;
 import org.jclouds.blobstore.functions.ParseSystemAndUserMetadataFromHeaders;
-import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpResponse;
 import org.jclouds.rest.InvocationContext;
 import org.jclouds.rest.internal.GeneratedHttpRequest;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 
 /**
@@ -60,16 +58,8 @@ public class ParseObjectFromHeadersAndHttpContent implements Function<HttpRespon
     */
    public S3Object apply(HttpResponse from) {
       S3Object object = objectProvider.create(metadataParser.apply(from));
-      addAllHeadersTo(from, object);
+      object.getAllHeaders().putAll(from.getHeaders());
-      if (from.getContent() != null)
-         object.setPayload(from.getContent());
-      attemptToParseSizeAndRangeFromHeaders(from, object);
-      return object;
-   }
 
-   @VisibleForTesting
-   void attemptToParseSizeAndRangeFromHeaders(HttpResponse from, S3Object object)
-            throws HttpException {
       String contentLength = from.getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH);
       String contentRange = from.getFirstHeaderOrNull("Content-Range");
 
@@ -77,17 +67,21 @@ public class ParseObjectFromHeadersAndHttpContent implements Function<HttpRespon
          object.setContentLength(Long.parseLong(contentLength));
       }
 
+      if (from.getContent() != null) {
+         object.setPayload(from.getContent());
+      } else if (object.getContentLength() != null && object.getContentLength() == 0) {
+         object.setPayload(new byte[0]);
+      } else {
+         assert false : "no content in " + from;
+      }
+
       if (contentRange == null && contentLength != null) {
          object.getMetadata().setSize(object.getContentLength());
       } else if (contentRange != null) {
          object.getMetadata().setSize(
                   Long.parseLong(contentRange.substring(contentRange.lastIndexOf('/') + 1)));
       }
-   }
+      return object;
-
-   @VisibleForTesting
-   void addAllHeadersTo(HttpResponse from, S3Object object) {
-      object.getAllHeaders().putAll(from.getHeaders());
    }
 
    public void setContext(GeneratedHttpRequest<?> request) {

aws/core/src/main/java/org/jclouds/aws/s3/functions/ReturnFalseIfBucketAlreadyOwnedByYou.java

@@ -31,13 +31,13 @@ import com.google.common.base.Function;
  * @author Adrian Cole
  */
 @Singleton
-public class ReturnTrueIfBucketAlreadyOwnedByYou implements Function<Exception, Boolean> {
+public class ReturnFalseIfBucketAlreadyOwnedByYou implements Function<Exception, Boolean> {
 
    public Boolean apply(Exception from) {
       if (from instanceof AWSResponseException) {
          AWSResponseException responseException = (AWSResponseException) from;
          if ("BucketAlreadyOwnedByYou".equals(responseException.getError().getCode())) {
-            return true;
+            return false;
          }
       }
       return Boolean.class.cast(propagateOrNull(from));

aws/core/src/main/java/org/jclouds/aws/s3/functions/ReturnTrueOn404OrNotFoundFalseIfNotEmpty.java

@@ -20,28 +20,49 @@ package org.jclouds.aws.s3.functions;
 
 import static org.jclouds.util.Utils.propagateOrNull;
 
+import java.util.List;
+
 import javax.inject.Singleton;
 
 import org.jclouds.aws.AWSResponseException;
 import org.jclouds.blobstore.ContainerNotFoundException;
+import org.jclouds.http.HttpResponseException;
 
 import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Iterables;
 
+/**
+ * 
+ * @author Adrian Cole
+ */
 @Singleton
 public class ReturnTrueOn404OrNotFoundFalseIfNotEmpty implements Function<Exception, Boolean> {
 
    public Boolean apply(Exception from) {
-      if (from instanceof ContainerNotFoundException) {
+      List<Throwable> throwables = Throwables.getCausalChain(from);
-         return true;
+
-      } else if (from instanceof AWSResponseException) {
+      Iterable<AWSResponseException> matchingAWSResponseException = Iterables.filter(throwables,
-         AWSResponseException responseException = (AWSResponseException) from;
+               AWSResponseException.class);
-         if (responseException.getResponse().getStatusCode() == 404) {
+      if (Iterables.size(matchingAWSResponseException) >= 1) {
-            return true;
+         if (Iterables.get(matchingAWSResponseException, 0).getError().getCode().equals(
-         } else if ("BucketNotEmpty".equals(responseException.getError().getCode())
+                  "BucketNotEmpty"))
-                  || responseException.getResponse().getStatusCode() == 409) {
             return false;
-         }
       }
+      
+      Iterable<ContainerNotFoundException> matchingContainerNotFoundException = Iterables.filter(
+               throwables, ContainerNotFoundException.class);
+      if (Iterables.size(matchingContainerNotFoundException) >= 1) {
+         return true;
+      }
+
+      Iterable<HttpResponseException> matchingHttpResponseException = Iterables.filter(throwables,
+               HttpResponseException.class);
+      if (Iterables.size(matchingHttpResponseException) >= 1) {
+         if (Iterables.get(matchingHttpResponseException, 0).getResponse().getStatusCode() == 404)
+            return true;
+      }
+
       return Boolean.class.cast(propagateOrNull(from));
    }
 }

aws/core/src/main/java/org/jclouds/aws/s3/util/S3Utils.java

@@ -26,11 +26,12 @@ import java.io.InputStream;
 import java.util.regex.Pattern;
 
 import javax.inject.Inject;
+import javax.inject.Singleton;
 
 import org.jclouds.aws.domain.AWSError;
+import org.jclouds.aws.s3.S3Client;
 import org.jclouds.aws.s3.reference.S3Headers;
 import org.jclouds.aws.util.AWSUtils;
-import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.http.HttpCommand;
 import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpResponse;
@@ -41,7 +42,8 @@ import org.jclouds.util.Patterns;
  * 
  * @author Adrian Cole
  */
-public class S3Utils extends BlobStoreUtils {
+@Singleton
+public class S3Utils {
 
    @Inject
    AWSUtils util;
@@ -76,4 +78,12 @@ public class S3Utils extends BlobStoreUtils {
       return bucketName;
    }
 
+   /**
+    * This implementation invokes {@link S3Client#deleteBucketIfEmpty} followed by
+    * {@link S3Client#bucketExists} until it is true.
+    */
+   public static boolean deleteAndVerifyContainerGone(S3Client sync, String container) {
+      sync.deleteBucketIfEmpty(container);
+      return sync.bucketExists(container);
+   }
 }

aws/core/src/main/java/org/jclouds/aws/s3/xml/ListBucketHandler.java

@@ -28,7 +28,7 @@ import org.jclouds.aws.s3.domain.ListBucketResponse;
 import org.jclouds.aws.s3.domain.ObjectMetadata;
 import org.jclouds.aws.s3.domain.ObjectMetadata.StorageClass;
 import org.jclouds.aws.s3.domain.internal.BucketListObjectMetadata;
-import org.jclouds.aws.s3.domain.internal.TreeSetListBucketResponse;
+import org.jclouds.aws.s3.domain.internal.ListBucketResponseImpl;
 import org.jclouds.date.DateService;
 import org.jclouds.encryption.EncryptionService;
 import org.jclouds.http.functions.ParseSax;
@@ -72,7 +72,7 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<ListBucketResp
    }
 
    public ListBucketResponse getResult() {
-      return new TreeSetListBucketResponse(bucketName, contents, prefix, marker, maxResults,
+      return new ListBucketResponseImpl(bucketName, contents, prefix, marker, nextMarker, maxResults,
                delimiter, isTruncated, commonPrefixes);
    }
 
@@ -83,6 +83,7 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<ListBucketResp
    private byte[] currentMD5;
    private long currentSize;
    private StorageClass currentStorageClass;
+   private String nextMarker;
 
    public void startElement(String uri, String name, String qName, Attributes attrs) {
       if (qName.equals("CommonPrefixes")) {
@@ -124,6 +125,9 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<ListBucketResp
       } else if (qName.equals("Marker")) {
          if (!currentText.toString().equals(""))
             this.marker = currentText.toString().trim();
+      } else if (qName.equals("NextMarker")) {
+         if (!currentText.toString().equals(""))
+            this.nextMarker = currentText.toString().trim();
       } else if (qName.equals("MaxKeys")) {
          this.maxResults = Integer.parseInt(currentText.toString().trim());
       } else if (qName.equals("IsTruncated")) {

aws/core/src/test/java/org/jclouds/aws/s3/S3AsyncClientTest.java

@@ -39,7 +39,7 @@ import org.jclouds.aws.s3.domain.AccessControlList.Permission;
 import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
 import org.jclouds.aws.s3.functions.ParseObjectFromHeadersAndHttpContent;
 import org.jclouds.aws.s3.functions.ParseObjectMetadataFromHeaders;
-import org.jclouds.aws.s3.functions.ReturnTrueIfBucketAlreadyOwnedByYou;
+import org.jclouds.aws.s3.functions.ReturnFalseIfBucketAlreadyOwnedByYou;
 import org.jclouds.aws.s3.functions.ReturnTrueOn404OrNotFoundFalseIfNotEmpty;
 import org.jclouds.aws.s3.options.CopyObjectOptions;
 import org.jclouds.aws.s3.options.ListBucketOptions;
@@ -57,6 +57,7 @@ import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
 import org.jclouds.blobstore.config.BlobStoreObjectModule;
 import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound;
 import org.jclouds.blobstore.functions.ReturnFalseOnKeyNotFound;
+import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
 import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
 import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404;
 import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
@@ -275,7 +276,7 @@ public class S3AsyncClientTest extends RestClientTest<S3AsyncClient> {
       assertResponseParserClassEquals(method, httpMethod,
                ParseObjectFromHeadersAndHttpContent.class);
       assertSaxResponseParserClassEquals(method, null);
-      assertExceptionParserClassEquals(method, ThrowKeyNotFoundOn404.class);
+      assertExceptionParserClassEquals(method, ReturnNullOnKeyNotFound.class);
 
       checkFilters(httpMethod);
    }
@@ -326,7 +327,7 @@ public class S3AsyncClientTest extends RestClientTest<S3AsyncClient> {
 
       assertResponseParserClassEquals(method, httpMethod, ParseObjectMetadataFromHeaders.class);
       assertSaxResponseParserClassEquals(method, null);
-      assertExceptionParserClassEquals(method, ThrowKeyNotFoundOn404.class);
+      assertExceptionParserClassEquals(method, ReturnNullOnKeyNotFound.class);
 
       checkFilters(httpMethod);
    }
@@ -384,7 +385,7 @@ public class S3AsyncClientTest extends RestClientTest<S3AsyncClient> {
 
       assertResponseParserClassEquals(method, httpMethod, ReturnTrueIf2xx.class);
       assertSaxResponseParserClassEquals(method, null);
-      assertExceptionParserClassEquals(method, ReturnTrueIfBucketAlreadyOwnedByYou.class);
+      assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYou.class);
 
       checkFilters(httpMethod);
    }
@@ -405,7 +406,7 @@ public class S3AsyncClientTest extends RestClientTest<S3AsyncClient> {
 
       assertResponseParserClassEquals(method, httpMethod, ReturnTrueIf2xx.class);
       assertSaxResponseParserClassEquals(method, null);
-      assertExceptionParserClassEquals(method, ReturnTrueIfBucketAlreadyOwnedByYou.class);
+      assertExceptionParserClassEquals(method, ReturnFalseIfBucketAlreadyOwnedByYou.class);
 
       checkFilters(httpMethod);
    }

aws/core/src/test/java/org/jclouds/aws/s3/S3ContextBuilderTest.java

@@ -0,0 +1,111 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.aws.s3;
+
+import static com.google.common.util.concurrent.Executors.sameThreadExecutor;
+import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AWS_ACCESSKEYID;
+import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AWS_SECRETACCESSKEY;
+import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX;
+import static org.testng.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.jclouds.aws.s3.S3AsyncClient;
+import org.jclouds.aws.s3.S3ContextBuilder;
+import org.jclouds.aws.s3.S3PropertiesBuilder;
+import org.jclouds.aws.s3.blobstore.config.S3BlobStoreContextModule;
+import org.jclouds.aws.s3.config.S3RestClientModule;
+import org.jclouds.aws.s3.config.S3StubClientModule;
+import org.jclouds.aws.s3.domain.S3Object;
+import org.jclouds.aws.s3.domain.internal.S3ObjectImpl;
+import org.jclouds.aws.s3.internal.StubS3AsyncClient;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.internal.BlobImpl;
+import org.jclouds.blobstore.internal.BlobStoreContextImpl;
+import org.jclouds.concurrent.config.ExecutorServiceModule;
+import org.testng.annotations.Test;
+
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+/**
+ * Tests behavior of modules configured in S3ContextBuilder
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "s3.S3ContextBuilderTest")
+public class S3ContextBuilderTest {
+
+   public void testNewBuilder() {
+      S3ContextBuilder builder = new S3ContextBuilder(new S3PropertiesBuilder(
+               "id", "secret").build());
+      assertEquals(builder.getProperties().getProperty(PROPERTY_USER_METADATA_PREFIX),
+               "x-amz-meta-");
+      assertEquals(builder.getProperties().getProperty(PROPERTY_AWS_ACCESSKEYID), "id");
+      assertEquals(builder.getProperties().getProperty(PROPERTY_AWS_SECRETACCESSKEY), "secret");
+   }
+
+   public void testBuildContext() {
+      BlobStoreContext context = new S3ContextBuilder(new S3PropertiesBuilder("id",
+               "secret").build()).withModules(
+               new ExecutorServiceModule(sameThreadExecutor(), sameThreadExecutor()),
+               new S3StubClientModule()).buildBlobStoreContext();
+      assertEquals(context.getClass(), BlobStoreContextImpl.class);
+      assertEquals(context.getProviderSpecificContext().getAsyncApi().getClass(),
+               StubS3AsyncClient.class);
+      // assertEquals(context.getAsyncBlobStore().getClass(), S3AsyncBlobStore.class);
+      assertEquals(((S3AsyncClient) context.getProviderSpecificContext().getAsyncApi())
+               .newS3Object().getClass(), S3ObjectImpl.class);
+      assertEquals(context.getAsyncBlobStore().newBlob(null).getClass(), BlobImpl.class);
+      assertEquals(context.getProviderSpecificContext().getAccount(), "id");
+      assertEquals(context.getProviderSpecificContext().getEndPoint(), URI
+               .create("https://localhost/s3stub"));
+   }
+
+   public void testBuildInjector() {
+      Injector i = new S3ContextBuilder(new S3PropertiesBuilder("id", "secret").build())
+               .withModules(new S3StubClientModule()).buildInjector();
+      assert i.getInstance(BlobStoreContext.class) != null;
+      assert i.getInstance(S3Object.class) != null;
+      assert i.getInstance(Blob.class) != null;
+   }
+
+   protected void testAddContextModule() {
+      List<Module> modules = new ArrayList<Module>();
+      S3ContextBuilder builder = (S3ContextBuilder) new S3ContextBuilder(
+               new S3PropertiesBuilder("id", "secret").build())
+               .withModules(new ExecutorServiceModule(sameThreadExecutor(), sameThreadExecutor()));
+      builder.addContextModule(modules);
+      assertEquals(modules.size(), 1);
+      assertEquals(modules.get(0).getClass(), S3BlobStoreContextModule.class);
+   }
+
+   protected void addClientModule() {
+      List<Module> modules = new ArrayList<Module>();
+      S3ContextBuilder builder = new S3ContextBuilder(new S3PropertiesBuilder(
+               "id", "secret").build());
+      builder.addClientModule(modules);
+      assertEquals(modules.size(), 1);
+      assertEquals(modules.get(0).getClass(), S3RestClientModule.class);
+   }
+
+}

aws/core/src/test/java/org/jclouds/aws/s3/blobstore/functions/ResourceToBucketList.java

@@ -26,9 +26,9 @@ import javax.inject.Singleton;
 import org.jclouds.aws.s3.domain.ListBucketResponse;
 import org.jclouds.aws.s3.domain.MutableObjectMetadata;
 import org.jclouds.aws.s3.domain.ObjectMetadata;
-import org.jclouds.aws.s3.domain.internal.TreeSetListBucketResponse;
+import org.jclouds.aws.s3.domain.internal.ListBucketResponseImpl;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.domain.StorageType;
 
@@ -42,7 +42,7 @@ import com.google.common.collect.Sets;
  */
 @Singleton
 public class ResourceToBucketList implements
-         Function<ListContainerResponse<? extends StorageMetadata>, ListBucketResponse> {
+         Function<PageSet<? extends StorageMetadata>, ListBucketResponse> {
    private final BlobToObjectMetadata blob2ObjectMd;
 
    @Inject
@@ -50,10 +50,10 @@ public class ResourceToBucketList implements
       this.blob2ObjectMd = blob2ObjectMd;
    }
 
-   public ListBucketResponse apply(ListContainerResponse<? extends StorageMetadata> list) {
+   public ListBucketResponse apply(PageSet<? extends StorageMetadata> list) {
 
-      Iterable<ObjectMetadata> contents = Iterables.transform(Iterables.filter(
+      Iterable<ObjectMetadata> contents = Iterables.transform(Iterables.filter(list,
-               list, new Predicate<StorageMetadata>() {
+               new Predicate<StorageMetadata>() {
 
                   public boolean apply(StorageMetadata input) {
                      return input.getType() == StorageType.BLOB;
@@ -81,7 +81,7 @@ public class ResourceToBucketList implements
          }
 
       }));
-      return new TreeSetListBucketResponse(null, contents, list.getPath(), list.getMarker(), list
+      return new ListBucketResponseImpl(null, contents, null, null, list.getNextMarker(), 0, "/",
-               .getMaxResults(), "/", Iterables.size(contents) == list.getMaxResults(), commonPrefixes);
+               list.getNextMarker() != null, commonPrefixes);
    }
 }

aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3BlobMapIntegrationTest.java

@@ -27,4 +27,9 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "s3.S3BlobMapIntegrationTest")
 public class S3BlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
 
+   @Override
+   protected int maxList() {
+      return 1000;
+   }
+
 }

aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3InputStreamMapIntegrationTest.java

@@ -26,5 +26,8 @@ import org.testng.annotations.Test;
  */
 @Test(groups = { "integration", "live" }, testName = "s3.S3InputStreamMapIntegrationTest")
 public class S3InputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
-
+   @Override
+   protected int maxList() {
+      return 1000;
+   }
 }

aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3TestInitializer.java

@@ -27,7 +27,7 @@ import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.BlobStoreContextFactory;
 import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
 import org.jclouds.blobstore.integration.internal.BaseTestInitializer;
-import org.jclouds.logging.config.ConsoleLoggingModule;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Module;
@@ -43,7 +43,7 @@ public class S3TestInitializer extends BaseTestInitializer {
             String account, String key) throws IOException {
       BaseBlobStoreIntegrationTest.SANITY_CHECK_RETURNED_BUCKET_NAME = true;
       return new BlobStoreContextFactory().createContext("s3", account, key, ImmutableSet.of(
-               configurationModule, new ConsoleLoggingModule()), new Properties());
+               configurationModule, new Log4JLoggingModule()), new Properties());
    }
 
    @Override

aws/core/src/test/java/org/jclouds/aws/s3/functions/ReturnTrueIfBucketAlreadyOwnedByYouTest.java

@@ -31,13 +31,13 @@ public class ReturnTrueIfBucketAlreadyOwnedByYouTest {
    @Test
    void testBucketAlreadyOwnedByYouIsOk() throws Exception {
       Exception e = getErrorWithCode("BucketAlreadyOwnedByYou");
-      assert new ReturnTrueIfBucketAlreadyOwnedByYou().apply(e);
+      assert !new ReturnFalseIfBucketAlreadyOwnedByYou().apply(e);
    }
 
    @Test(expectedExceptions = AWSResponseException.class)
    void testBlahIsNotOk() throws Exception {
       Exception e = getErrorWithCode("blah");
-      new ReturnTrueIfBucketAlreadyOwnedByYou().apply(e);
+      new ReturnFalseIfBucketAlreadyOwnedByYou().apply(e);
    }
 
    private Exception getErrorWithCode(String code) {

aws/core/src/test/java/org/jclouds/aws/s3/internal/StubS3AsyncClient.java

@@ -189,7 +189,8 @@ public class StubS3AsyncClient implements S3AsyncClient {
          return immediateFuture((ObjectMetadata) blob2ObjectMetadata.apply(StubAsyncBlobStore
                   .copy(newMd)));
       }
-      return immediateFailedFuture(new KeyNotFoundException(sourceBucket, sourceObject));
+      return immediateFailedFuture(new KeyNotFoundException(sourceBucket, sourceObject,
+               sourceBucket + "/" + sourceObject));
    }
 
    public ListenableFuture<String> putObject(final String bucketName, final S3Object object,

aws/core/src/test/java/org/jclouds/aws/s3/options/CopyObjectOptionsTest.java

@@ -70,7 +70,6 @@ public class CopyObjectOptionsTest {
    void testGoodMetaStatic() {
       CopyObjectOptions options = overrideMetadataWith(goodMeta);
       options.setMetadataPrefix("x-amz-meta-");
-
       assertGoodMeta(options);
    }
 

aws/core/src/test/java/org/jclouds/aws/s3/xml/ListBucketHandlerTest.java

@@ -28,7 +28,7 @@ import org.jclouds.aws.s3.domain.ListBucketResponse;
 import org.jclouds.aws.s3.domain.ObjectMetadata;
 import org.jclouds.aws.s3.domain.ObjectMetadata.StorageClass;
 import org.jclouds.aws.s3.domain.internal.BucketListObjectMetadata;
-import org.jclouds.aws.s3.domain.internal.TreeSetListBucketResponse;
+import org.jclouds.aws.s3.domain.internal.ListBucketResponseImpl;
 import org.jclouds.date.DateService;
 import org.jclouds.encryption.EncryptionService;
 import org.jclouds.encryption.internal.JCEEncryptionService;
@@ -60,7 +60,7 @@ public class ListBucketHandlerTest extends BaseHandlerTest {
       InputStream is = getClass().getResourceAsStream("/s3/list_bucket.xml");
       CanonicalUser owner = new CanonicalUser(
                "e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0", "ferncam");
-      ListBucketResponse expected = new TreeSetListBucketResponse(
+      ListBucketResponse expected = new ListBucketResponseImpl(
                "adriancole.org.jclouds.aws.s3.amazons3testdelimiter", ImmutableList.of(
                         (ObjectMetadata) new BucketListObjectMetadata("apps/0", dateService
                                  .iso8601DateParse("2009-05-07T18:27:08.000Z"),
@@ -111,8 +111,8 @@ public class ListBucketHandlerTest extends BaseHandlerTest {
                                  .iso8601DateParse("2009-05-07T18:27:10.000Z"),
                                  "\"cd8a19b26fea8a827276df0ad11c580d\"", encryptionService
                                           .fromHexString("cd8a19b26fea8a827276df0ad11c580d"), 8,
-                                 owner, StorageClass.STANDARD)), "apps/", null, 1000, null, false,
+                                 owner, StorageClass.STANDARD)), "apps/", null, null, 1000, null,
-               new TreeSet<String>());
+               false, new TreeSet<String>());
 
       ListBucketResponse result = (ListBucketResponse) factory.create(
                injector.getInstance(ListBucketHandler.class)).parse(is);

aws/core/src/test/resources/log4j.xml

@@ -1,4 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+    <!--
+
+
+        Copyright (C) 2009 Cloud Conscious, LLC.
+        <info@cloudconscious.com>
+
+        ====================================================================
+        Licensed under the Apache License, Version 2.0 (the "License");
+        you may not use this file except in compliance with the License.
+        You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+        applicable law or agreed to in writing, software distributed
+        under the License is distributed on an "AS IS" BASIS, WITHOUT
+        WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        See the License for the specific language governing permissions
+        and limitations under the License.
+        ====================================================================
+    -->
 <!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+    <!--
+        For more configuration infromation and examples see the Apache
+        Log4j website: http://logging.apache.org/log4j/
+    -->
 <log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
     debug="false">
 
@@ -45,6 +70,17 @@
             -->
         </layout>
     </appender>
+
+    <!-- A time/date based rolling appender -->
+    <appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="target/test-data/jclouds-blobstore.log" />
+        <param name="Append" value="true" />
+        <param name="DatePattern" value="'.'yyyy-MM-dd" />
+        <param name="Threshold" value="TRACE" />
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+        </layout>
+    </appender>
     
     <!-- A time/date based rolling appender -->
     <appender name="COMPUTEFILE" class="org.apache.log4j.DailyRollingFileAppender">
@@ -61,17 +97,16 @@
             <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
 
             <!--
-                The full pattern: Date MS Priority [Category] (Thread:NDC) Message\n
+                The full pattern: Date MS Priority [Category]
-                <param name="ConversionPattern" value="%d %-5r %-5p [%c] (%t:%x)
+                (Thread:NDC) Message\n <param name="ConversionPattern"
-                %m%n"/>
+                value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
             -->
         </layout>
     </appender>
-    
+
     <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
         <appender-ref ref="COMPUTEFILE" />
-    </appender> 
+    </appender>
-
     <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
         <appender-ref ref="FILE" />
     </appender>
@@ -80,9 +115,16 @@
         <appender-ref ref="WIREFILE" />
     </appender>
 
+    <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+        <appender-ref ref="BLOBSTOREFILE" />
+    </appender>
     <!-- ================ -->
     <!-- Limit categories -->
     <!-- ================ -->
+    <category name="jclouds.blobstore">
+        <priority value="TRACE" />
+        <appender-ref ref="ASYNCBLOBSTORE" />
+    </category>
 
     <category name="org.jclouds">
         <priority value="DEBUG" />
@@ -93,18 +135,16 @@
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCWIRE" />
     </category>
+    <category name="jclouds.compute">
+        <priority value="TRACE" />
+        <appender-ref ref="ASYNCCOMPUTE" />
+    </category><!--
 
     <category name="jclouds.wire">
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCWIRE" />
     </category>
-    
+    --><!--  ======================= -->
-    <category name="jclouds.compute">
-        <priority value="TRACE" />
-        <appender-ref ref="ASYNCCOMPUTE" />
-    </category>
-    
-    <!--======================= -->
     <!-- Setup the Root category -->
     <!-- ======================= -->
 

aws/perftest/src/test/java/org/jclouds/aws/s3/BasePerformanceLiveTest.java

@@ -23,7 +23,7 @@ import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.InputStream;
-import java.util.Set;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
@@ -36,7 +36,7 @@ import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
 import org.jclouds.logging.Logger;
 import org.testng.annotations.Test;
 
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 
@@ -146,11 +146,11 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
 
    private void doParallel(Provider<ListenableFuture<?>> provider, int loopCount,
             String containerName) throws InterruptedException, ExecutionException, TimeoutException {
-      Set<ListenableFuture<?>> responses = Sets.newHashSet();
+      Map<Integer, ListenableFuture<?>> responses = Maps.newHashMap();
       for (int i = 0; i < loopCount; i++)
-         responses.add(provider.get());
+         responses.put(i, provider.get());
-      awaitCompletion(responses, exec, null, logger, String.format(
+      assert awaitCompletion(responses, exec, null, logger,
-               "putting into containerName: %s", containerName));
+               String.format("putting into containerName: %s", containerName)).size() == 0;
    }
 
    class PutBytesFuture implements Provider<ListenableFuture<?>> {

azure/src/main/java/org/jclouds/azure/storage/blob/AzureBlobAsyncClient.java

@@ -37,7 +37,7 @@ import org.jclouds.azure.storage.blob.functions.BlobName;
 import org.jclouds.azure.storage.blob.functions.ParseBlobFromHeadersAndHttpContent;
 import org.jclouds.azure.storage.blob.functions.ParseBlobPropertiesFromHeaders;
 import org.jclouds.azure.storage.blob.functions.ParseContainerPropertiesFromHeaders;
-import org.jclouds.azure.storage.blob.functions.ReturnTrueIfContainerAlreadyExists;
+import org.jclouds.azure.storage.blob.functions.ReturnFalseIfContainerAlreadyExists;
 import org.jclouds.azure.storage.blob.options.CreateContainerOptions;
 import org.jclouds.azure.storage.blob.options.ListBlobsOptions;
 import org.jclouds.azure.storage.blob.xml.AccountNameEnumerationResultsHandler;
@@ -51,8 +51,9 @@ import org.jclouds.blobstore.attr.ConsistencyModels;
 import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix;
 import org.jclouds.blobstore.functions.ReturnFalseOnContainerNotFound;
 import org.jclouds.blobstore.functions.ReturnFalseOnKeyNotFound;
+import org.jclouds.blobstore.functions.ReturnNullOnContainerNotFound;
+import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
 import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
-import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
 import org.jclouds.http.functions.ParseETagHeader;
 import org.jclouds.http.functions.ReturnTrueOn404;
 import org.jclouds.http.options.GetOptions;
@@ -104,7 +105,7 @@ public interface AzureBlobAsyncClient {
     */
    @PUT
    @Path("{container}")
-   @ExceptionParser(ReturnTrueIfContainerAlreadyExists.class)
+   @ExceptionParser(ReturnFalseIfContainerAlreadyExists.class)
    @QueryParams(keys = "restype", values = "container")
    ListenableFuture<Boolean> createContainer(@PathParam("container") String container,
             CreateContainerOptions... options);
@@ -116,6 +117,7 @@ public interface AzureBlobAsyncClient {
    @Path("{container}")
    @QueryParams(keys = "restype", values = "container")
    @ResponseParser(ParseContainerPropertiesFromHeaders.class)
+   @ExceptionParser(ReturnNullOnContainerNotFound.class)
    ListenableFuture<ContainerProperties> getContainerProperties(
             @PathParam("container") String container);
 
@@ -151,7 +153,7 @@ public interface AzureBlobAsyncClient {
     */
    @PUT
    @Path("$root")
-   @ExceptionParser(ReturnTrueIfContainerAlreadyExists.class)
+   @ExceptionParser(ReturnFalseIfContainerAlreadyExists.class)
    @QueryParams(keys = "restype", values = "container")
    ListenableFuture<Boolean> createRootContainer(CreateContainerOptions... options);
 
@@ -198,7 +200,7 @@ public interface AzureBlobAsyncClient {
     */
    @GET
    @ResponseParser(ParseBlobFromHeadersAndHttpContent.class)
-   @ExceptionParser(ThrowKeyNotFoundOn404.class)
+   @ExceptionParser(ReturnNullOnKeyNotFound.class)
    @Path("{container}/{name}")
    ListenableFuture<org.jclouds.azure.storage.blob.domain.AzureBlob> getBlob(
             @PathParam("container") String container, @PathParam("name") String name,
@@ -209,7 +211,7 @@ public interface AzureBlobAsyncClient {
     */
    @HEAD
    @ResponseParser(ParseBlobPropertiesFromHeaders.class)
-   @ExceptionParser(ThrowKeyNotFoundOn404.class)
+   @ExceptionParser(ReturnNullOnKeyNotFound.class)
    @Path("{container}/{name}")
    ListenableFuture<BlobProperties> getBlobProperties(@PathParam("container") String container,
             @PathParam("name") String name);

azure/src/main/java/org/jclouds/azure/storage/blob/AzureBlobClient.java

@@ -67,6 +67,8 @@ public interface AzureBlobClient {
     * The container resource includes metadata and properties for that container. It does not
     * include a list of the blobs contained by the container.
     * 
+    * @return true, if the bucket was created or false, if the container was already present
+    * 
     * @see CreateContainerOptions
     * 
     */
@@ -176,8 +178,10 @@ public interface AzureBlobClient {
     * <p/>
     * Blobs are listed in alphabetical order in the response body.
     */
+   @Timeout(duration = 2, timeUnit = TimeUnit.MINUTES)
    ListBlobsResponse listBlobs(String container, ListBlobsOptions... options);
 
+   @Timeout(duration = 2, timeUnit = TimeUnit.MINUTES)
    ListBlobsResponse listBlobs(ListBlobsOptions... options);
 
    /**

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/AzureAsyncBlobStore.java

@@ -18,74 +18,78 @@
  */
 package org.jclouds.azure.storage.blob.blobstore;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.util.concurrent.Futures.compose;
 import static org.jclouds.azure.storage.options.ListOptions.Builder.includeMetadata;
-import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
-import static org.jclouds.concurrent.ConcurrentUtils.convertExceptionToValue;
-import static org.jclouds.concurrent.ConcurrentUtils.makeListenable;
 
-import java.util.Set;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
 import javax.inject.Named;
+import javax.inject.Singleton;
 
 import org.jclouds.Constants;
 import org.jclouds.azure.storage.blob.AzureBlobAsyncClient;
-import org.jclouds.azure.storage.blob.AzureBlobClient;
 import org.jclouds.azure.storage.blob.blobstore.functions.AzureBlobToBlob;
 import org.jclouds.azure.storage.blob.blobstore.functions.BlobPropertiesToBlobMetadata;
 import org.jclouds.azure.storage.blob.blobstore.functions.BlobToAzureBlob;
 import org.jclouds.azure.storage.blob.blobstore.functions.ContainerToResourceMetadata;
 import org.jclouds.azure.storage.blob.blobstore.functions.ListBlobsResponseToResourceList;
 import org.jclouds.azure.storage.blob.blobstore.functions.ListOptionsToListBlobsOptions;
-import org.jclouds.azure.storage.blob.blobstore.internal.BaseAzureBlobStore;
 import org.jclouds.azure.storage.blob.domain.AzureBlob;
 import org.jclouds.azure.storage.blob.domain.BlobProperties;
 import org.jclouds.azure.storage.blob.domain.ContainerProperties;
 import org.jclouds.azure.storage.blob.domain.ListBlobsResponse;
 import org.jclouds.azure.storage.blob.options.ListBlobsOptions;
-import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.azure.storage.domain.BoundedSet;
-import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.Blob.Factory;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
-import org.jclouds.blobstore.domain.internal.ListResponseImpl;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
+import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
 import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
+import org.jclouds.blobstore.util.BlobStoreUtils;
-import org.jclouds.blobstore.strategy.GetDirectoryStrategy;
-import org.jclouds.blobstore.strategy.MkdirStrategy;
 import org.jclouds.http.options.GetOptions;
-import org.jclouds.logging.Logger.LoggerFactory;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.ListenableFuture;
 
 /**
- * 
  * @author Adrian Cole
  */
-public class AzureAsyncBlobStore extends BaseAzureBlobStore implements AsyncBlobStore {
+@Singleton
+public class AzureAsyncBlobStore extends BaseAsyncBlobStore {
+   private final AzureBlobAsyncClient async;
+   private final ContainerToResourceMetadata container2ResourceMd;
+   private final ListOptionsToListBlobsOptions blobStore2AzureContainerListOptions;
+   private final ListBlobsResponseToResourceList azure2BlobStoreResourceList;
+   private final AzureBlobToBlob azureBlob2Blob;
+   private final BlobToAzureBlob blob2AzureBlob;
+   private final BlobPropertiesToBlobMetadata blob2BlobMd;
+   private final BlobToHttpGetOptions blob2ObjectGetOptions;
 
    @Inject
-   public AzureAsyncBlobStore(AzureBlobAsyncClient async, AzureBlobClient sync,
+   AzureAsyncBlobStore(BlobStoreUtils blobUtils,
-            Factory blobFactory, LoggerFactory logFactory,
+            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service,
-            ClearListStrategy clearContainerStrategy, BlobPropertiesToBlobMetadata blob2BlobMd,
+            AzureBlobAsyncClient async, ContainerToResourceMetadata container2ResourceMd,
-            AzureBlobToBlob blob2Blob, BlobToAzureBlob blob2Object,
+            ListOptionsToListBlobsOptions blobStore2AzureContainerListOptions,
-            ListOptionsToListBlobsOptions container2ContainerListOptions,
+            ListBlobsResponseToResourceList azure2BlobStoreResourceList,
-            BlobToHttpGetOptions blob2ObjectGetOptions, GetDirectoryStrategy getDirectoryStrategy,
+            AzureBlobToBlob azureBlob2Blob, BlobToAzureBlob blob2AzureBlob,
-            MkdirStrategy mkdirStrategy, ContainerToResourceMetadata container2ResourceMd,
+            BlobPropertiesToBlobMetadata blob2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions) {
-            ListBlobsResponseToResourceList container2ResourceList,
+      super(blobUtils, service);
-            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service) {
+      this.async = checkNotNull(async, "async");
-      super(async, sync, blobFactory, logFactory, clearContainerStrategy, blob2BlobMd, blob2Blob,
+      this.container2ResourceMd = checkNotNull(container2ResourceMd, "container2ResourceMd");
-               blob2Object, container2ContainerListOptions, blob2ObjectGetOptions,
+      this.blobStore2AzureContainerListOptions = checkNotNull(blobStore2AzureContainerListOptions,
-               getDirectoryStrategy, mkdirStrategy, container2ResourceMd, container2ResourceList,
+               "blobStore2AzureContainerListOptions");
-               service);
+      this.azure2BlobStoreResourceList = checkNotNull(azure2BlobStoreResourceList,
+               "azure2BlobStoreResourceList");
+      this.azureBlob2Blob = checkNotNull(azureBlob2Blob, "azureBlob2Blob");
+      this.blob2AzureBlob = checkNotNull(blob2AzureBlob, "blob2AzureBlob");
+      this.blob2BlobMd = checkNotNull(blob2BlobMd, "blob2BlobMd");
+      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
    }
 
    /**
@@ -93,14 +97,14 @@ public class AzureAsyncBlobStore extends BaseAzureBlobStore implements AsyncBlob
     * {@link org.jclouds.azure.storage.options.ListOptions#includeMetadata} option.
     */
    @Override
-   public ListenableFuture<? extends org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata>> list() {
+   public ListenableFuture<? extends org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>> list() {
       return compose(
                async.listContainers(includeMetadata()),
-               new Function<Set<ContainerProperties>, org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata>>() {
+               new Function<BoundedSet<ContainerProperties>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
-                  public org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata> apply(
+                  public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(
-                           Set<ContainerProperties> from) {
+                           BoundedSet<ContainerProperties> from) {
-                     return new ListResponseImpl<StorageMetadata>(Iterables.transform(from,
+                     return new PageSetImpl<StorageMetadata>(Iterables.transform(from,
-                              container2ResourceMd), null, null, false);
+                              container2ResourceMd), from.getNextMarker());
                   }
                }, service);
    }
@@ -129,19 +133,6 @@ public class AzureAsyncBlobStore extends BaseAzureBlobStore implements AsyncBlob
       return async.createContainer(container);
    }
 
-   /**
-    * This implementation invokes
-    * {@link #list(String,org.jclouds.blobstore.options.ListContainerOptions)}
-    * 
-    * @param container
-    *           container name
-    */
-   @Override
-   public ListenableFuture<? extends ListContainerResponse<? extends StorageMetadata>> list(
-            String container) {
-      return this.list(container, org.jclouds.blobstore.options.ListContainerOptions.NONE);
-   }
-
    /**
     * This implementation invokes {@link AzureBlobAsyncClient#listBucket}
     * 
@@ -149,31 +140,12 @@ public class AzureAsyncBlobStore extends BaseAzureBlobStore implements AsyncBlob
     *           container name
     */
    @Override
-   public ListenableFuture<? extends ListContainerResponse<? extends StorageMetadata>> list(
+   public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(String container,
-            String container, ListContainerOptions options) {
+            ListContainerOptions options) {
-      ListBlobsOptions azureOptions = container2ContainerListOptions.apply(options);
+      ListBlobsOptions azureOptions = blobStore2AzureContainerListOptions.apply(options);
       ListenableFuture<ListBlobsResponse> returnVal = async.listBlobs(container, azureOptions
                .includeMetadata());
-      return compose(returnVal, container2ResourceList, service);
+      return compose(returnVal, azure2BlobStoreResourceList, service);
-   }
-
-   /**
-    * This implementation invokes {@link ClearListStrategy#execute} with the
-    * {@link ListContainerOptions#recursive} option.
-    * 
-    * @param container
-    *           container name
-    */
-   @Override
-   public ListenableFuture<Void> clearContainer(final String container) {
-      return makeListenable(service.submit(new Callable<Void>() {
-
-         public Void call() throws Exception {
-            clearContainerStrategy.execute(container, recursive());
-            return null;
-         }
-
-      }), service);
    }
 
    /**
@@ -188,47 +160,45 @@ public class AzureAsyncBlobStore extends BaseAzureBlobStore implements AsyncBlob
    }
 
    /**
-    * This implementation invokes {@link GetDirectoryStrategy#execute}
+    * This implementation invokes {@link AzureBlobAsyncClient#getBlob}
     * 
     * @param container
     *           container name
-    * @param directory
+    * @param key
-    *           virtual path
+    *           blob key
     */
    @Override
-   public ListenableFuture<Boolean> directoryExists(final String container, final String directory) {
+   public ListenableFuture<Blob> getBlob(String container, String key,
-      return makeListenable(service.submit(new Callable<Boolean>() {
+            org.jclouds.blobstore.options.GetOptions options) {
-
+      GetOptions azureOptions = blob2ObjectGetOptions.apply(options);
-         public Boolean call() throws Exception {
+      ListenableFuture<AzureBlob> returnVal = async.getBlob(container, key, azureOptions);
-            try {
+      return compose(returnVal, azureBlob2Blob, service);
-               getDirectoryStrategy.execute(container, directory);
-               return true;
-            } catch (KeyNotFoundException e) {
-               return false;
-            }
-         }
-
-      }), service);
    }
 
    /**
-    * This implementation invokes {@link MkdirStrategy#execute}
+    * This implementation invokes {@link AzureBlobAsyncClient#putBlob}
     * 
     * @param container
     *           container name
-    * @param directory
+    * @param blob
-    *           virtual path
+    *           blob
     */
    @Override
-   public ListenableFuture<Void> createDirectory(final String container, final String directory) {
+   public ListenableFuture<String> putBlob(String container, Blob blob) {
-      return makeListenable(service.submit(new Callable<Void>() {
+      return async.putBlob(container, blob2AzureBlob.apply(blob));
+   }
 
-         public Void call() throws Exception {
+   /**
-            mkdirStrategy.execute(container, directory);
+    * This implementation invokes {@link AzureBlobAsyncClient#deleteObject}
-            return null;
+    * 
-         }
+    * @param container
-
+    *           container name
-      }), service);
+    * @param key
+    *           blob key
+    */
+   @Override
+   public ListenableFuture<Void> removeBlob(String container, String key) {
+      return async.deleteBlob(container, key);
    }
 
    /**
@@ -254,8 +224,7 @@ public class AzureAsyncBlobStore extends BaseAzureBlobStore implements AsyncBlob
     */
    @Override
    public ListenableFuture<BlobMetadata> blobMetadata(String container, String key) {
-      return compose(convertExceptionToValue(async.getBlobProperties(container, key),
+      return compose(async.getBlobProperties(container, key), new Function<BlobProperties, BlobMetadata>() {
-               KeyNotFoundException.class, null), new Function<BlobProperties, BlobMetadata>() {
 
          @Override
          public BlobMetadata apply(BlobProperties from) {
@@ -265,61 +234,9 @@ public class AzureAsyncBlobStore extends BaseAzureBlobStore implements AsyncBlob
       }, service);
    }
 
-   /**
-    * This implementation invokes
-    * {@link #getBlob(String,String,org.jclouds.blobstore.options.GetOptions)}
-    * 
-    * @param container
-    *           container name
-    * @param key
-    *           blob key
-    */
    @Override
-   public ListenableFuture<Blob> getBlob(String container, String key) {
+   protected boolean deleteAndVerifyContainerGone(String container) {
-      return getBlob(container, key, org.jclouds.blobstore.options.GetOptions.NONE);
+      throw new UnsupportedOperationException("please use deleteContainer");
-   }
-
-   /**
-    * This implementation invokes {@link AzureBlobAsyncClient#getBlob}
-    * 
-    * @param container
-    *           container name
-    * @param key
-    *           blob key
-    */
-   @Override
-   public ListenableFuture<Blob> getBlob(String container, String key,
-            org.jclouds.blobstore.options.GetOptions options) {
-      GetOptions azureOptions = blob2ObjectGetOptions.apply(options);
-      ListenableFuture<AzureBlob> returnVal = async.getBlob(container, key, azureOptions);
-      return compose(convertExceptionToValue(returnVal, KeyNotFoundException.class, null),
-               blob2Blob, service);
-   }
-
-   /**
-    * This implementation invokes {@link AzureBlobAsyncClient#putBlob}
-    * 
-    * @param container
-    *           container name
-    * @param blob
-    *           blob
-    */
-   @Override
-   public ListenableFuture<String> putBlob(String container, Blob blob) {
-      return async.putBlob(container, blob2Object.apply(blob));
-   }
-
-   /**
-    * This implementation invokes {@link AzureBlobAsyncClient#deleteObject}
-    * 
-    * @param container
-    *           container name
-    * @param key
-    *           blob key
-    */
-   @Override
-   public ListenableFuture<Void> removeBlob(String container, String key) {
-      return async.deleteBlob(container, key);
    }
 
 }

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/AzureBlobStore.java

@@ -18,18 +18,12 @@
  */
 package org.jclouds.azure.storage.blob.blobstore;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static org.jclouds.azure.storage.options.ListOptions.Builder.includeMetadata;
-import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
-import static org.jclouds.blobstore.util.BlobStoreUtils.keyNotFoundToNullOrPropagate;
-
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
-import javax.inject.Named;
+import javax.inject.Singleton;
 
-import org.jclouds.Constants;
-import org.jclouds.azure.storage.blob.AzureBlobAsyncClient;
 import org.jclouds.azure.storage.blob.AzureBlobClient;
 import org.jclouds.azure.storage.blob.blobstore.functions.AzureBlobToBlob;
 import org.jclouds.azure.storage.blob.blobstore.functions.BlobPropertiesToBlobMetadata;
@@ -37,58 +31,69 @@ import org.jclouds.azure.storage.blob.blobstore.functions.BlobToAzureBlob;
 import org.jclouds.azure.storage.blob.blobstore.functions.ContainerToResourceMetadata;
 import org.jclouds.azure.storage.blob.blobstore.functions.ListBlobsResponseToResourceList;
 import org.jclouds.azure.storage.blob.blobstore.functions.ListOptionsToListBlobsOptions;
-import org.jclouds.azure.storage.blob.blobstore.internal.BaseAzureBlobStore;
 import org.jclouds.azure.storage.blob.domain.ContainerProperties;
 import org.jclouds.azure.storage.blob.options.ListBlobsOptions;
-import org.jclouds.blobstore.BlobStore;
+import org.jclouds.azure.storage.domain.BoundedSet;
-import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
-import org.jclouds.blobstore.domain.ListResponse;
 import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.Blob.Factory;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
-import org.jclouds.blobstore.domain.internal.ListResponseImpl;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
+import org.jclouds.blobstore.internal.BaseBlobStore;
 import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
+import org.jclouds.blobstore.util.BlobStoreUtils;
-import org.jclouds.blobstore.strategy.GetDirectoryStrategy;
-import org.jclouds.blobstore.strategy.MkdirStrategy;
 import org.jclouds.http.options.GetOptions;
-import org.jclouds.logging.Logger.LoggerFactory;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 
-public class AzureBlobStore extends BaseAzureBlobStore implements BlobStore {
+/**
+ * @author Adrian Cole
+ */
+@Singleton
+public class AzureBlobStore extends BaseBlobStore {
+   private final AzureBlobClient sync;
+   private final ContainerToResourceMetadata container2ResourceMd;
+   private final ListOptionsToListBlobsOptions blobStore2AzureContainerListOptions;
+   private final ListBlobsResponseToResourceList azure2BlobStoreResourceList;
+   private final AzureBlobToBlob azureBlob2Blob;
+   private final BlobToAzureBlob blob2AzureBlob;
+   private final BlobPropertiesToBlobMetadata blob2BlobMd;
+   private final BlobToHttpGetOptions blob2ObjectGetOptions;
 
    @Inject
-   public AzureBlobStore(AzureBlobAsyncClient async, AzureBlobClient sync, Factory blobFactory,
+   AzureBlobStore(BlobStoreUtils blobUtils, AzureBlobClient sync,
-            LoggerFactory logFactory, ClearListStrategy clearContainerStrategy,
+            ContainerToResourceMetadata container2ResourceMd,
-            BlobPropertiesToBlobMetadata object2BlobMd, AzureBlobToBlob object2Blob,
+            ListOptionsToListBlobsOptions blobStore2AzureContainerListOptions,
-            BlobToAzureBlob blob2Object,
+            ListBlobsResponseToResourceList azure2BlobStoreResourceList,
-            ListOptionsToListBlobsOptions container2ContainerListOptions,
+            AzureBlobToBlob azureBlob2Blob, BlobToAzureBlob blob2AzureBlob,
-            BlobToHttpGetOptions blob2ObjectGetOptions, GetDirectoryStrategy getDirectoryStrategy,
+            BlobPropertiesToBlobMetadata blob2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions) {
-            MkdirStrategy mkdirStrategy, ContainerToResourceMetadata container2ResourceMd,
+      super(blobUtils);
-            ListBlobsResponseToResourceList container2ResourceList,
+      this.sync = checkNotNull(sync, "sync");
-            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service) {
+      this.container2ResourceMd = checkNotNull(container2ResourceMd, "container2ResourceMd");
-      super(async, sync, blobFactory, logFactory, clearContainerStrategy, object2BlobMd,
+      this.blobStore2AzureContainerListOptions = checkNotNull(blobStore2AzureContainerListOptions,
-               object2Blob, blob2Object, container2ContainerListOptions, blob2ObjectGetOptions,
+               "blobStore2AzureContainerListOptions");
-               getDirectoryStrategy, mkdirStrategy, container2ResourceMd, container2ResourceList,
+      this.azure2BlobStoreResourceList = checkNotNull(azure2BlobStoreResourceList,
-               service);
+               "azure2BlobStoreResourceList");
+      this.azureBlob2Blob = checkNotNull(azureBlob2Blob, "azureBlob2Blob");
+      this.blob2AzureBlob = checkNotNull(blob2AzureBlob, "blob2AzureBlob");
+      this.blob2BlobMd = checkNotNull(blob2BlobMd, "blob2BlobMd");
+      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
    }
 
    /**
     * This implementation invokes {@link AzureBlobClient#listContainers}
     */
    @Override
-   public ListResponse<? extends StorageMetadata> list() {
+   public PageSet<? extends StorageMetadata> list() {
-      return new Function<Set<ContainerProperties>, org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata>>() {
+      return new Function<BoundedSet<ContainerProperties>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
-         public org.jclouds.blobstore.domain.ListResponse<? extends StorageMetadata> apply(
+         public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(
-                  Set<ContainerProperties> from) {
+                  BoundedSet<ContainerProperties> from) {
-            return new ListResponseImpl<StorageMetadata>(Iterables.transform(from,
+            return new PageSetImpl<StorageMetadata>(
-                     container2ResourceMd), null, null, false);
+                     Iterables.transform(from, container2ResourceMd), from.getNextMarker());
          }
+         // TODO this may be a list that isn't complete due to 1000 container limit
       }.apply(sync.listContainers(includeMetadata()));
    }
 
@@ -116,18 +121,6 @@ public class AzureBlobStore extends BaseAzureBlobStore implements BlobStore {
       return sync.createContainer(container);
    }
 
-   /**
-    * This implementation invokes
-    * {@link #list(String,org.jclouds.blobstore.options.ListContainerOptions)}
-    * 
-    * @param container
-    *           container name
-    */
-   @Override
-   public ListContainerResponse<? extends StorageMetadata> list(String container) {
-      return this.list(container, org.jclouds.blobstore.options.ListContainerOptions.NONE);
-   }
-
    /**
     * This implementation invokes {@link AzureBlobClient#listBlobs}
     * 
@@ -135,23 +128,10 @@ public class AzureBlobStore extends BaseAzureBlobStore implements BlobStore {
     *           container name
     */
    @Override
-   public ListContainerResponse<? extends StorageMetadata> list(String container,
+   public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
-            ListContainerOptions optionsList) {
+      ListBlobsOptions azureOptions = blobStore2AzureContainerListOptions.apply(options);
-      ListBlobsOptions azureOptions = container2ContainerListOptions.apply(optionsList);
+      return azure2BlobStoreResourceList.apply(sync.listBlobs(container, azureOptions
-      return container2ResourceList
+               .includeMetadata()));
-               .apply(sync.listBlobs(container, azureOptions.includeMetadata()));
-   }
-
-   /**
-    * This implementation invokes {@link ClearListStrategy#clearContainerStrategy} with the
-    * {@link ListContainerOptions#recursive} option.
-    * 
-    * @param container
-    *           container name
-    */
-   @Override
-   public void clearContainer(final String container) {
-      clearContainerStrategy.execute(container, recursive());
    }
 
    /**
@@ -165,37 +145,6 @@ public class AzureBlobStore extends BaseAzureBlobStore implements BlobStore {
       sync.deleteContainer(container);
    }
 
-   /**
-    * This implementation invokes {@link GetDirectoryStrategy#execute}
-    * 
-    * @param container
-    *           container name
-    * @param directory
-    *           virtual path
-    */
-   @Override
-   public boolean directoryExists(String containerName, String directory) {
-      try {
-         getDirectoryStrategy.execute(containerName, directory);
-         return true;
-      } catch (KeyNotFoundException e) {
-         return false;
-      }
-   }
-
-   /**
-    * This implementation invokes {@link MkdirStrategy#execute}
-    * 
-    * @param container
-    *           container name
-    * @param directory
-    *           virtual path
-    */
-   @Override
-   public void createDirectory(String containerName, String directory) {
-      mkdirStrategy.execute(containerName, directory);
-   }
-
    /**
     * This implementation invokes {@link AzureBlobClient#blobExists}
     * 
@@ -209,37 +158,6 @@ public class AzureBlobStore extends BaseAzureBlobStore implements BlobStore {
       return sync.blobExists(container, key);
    }
 
-   /**
-    * This implementation invokes {@link AzureBlobClient#getBlobProperties}
-    * 
-    * @param container
-    *           container name
-    * @param key
-    *           blob key
-    */
-   @Override
-   public BlobMetadata blobMetadata(String container, String key) {
-      try {
-         return blob2BlobMd.apply(sync.getBlobProperties(container, key));
-      } catch (Exception e) {
-         return keyNotFoundToNullOrPropagate(e);
-      }
-   }
-
-   /**
-    * This implementation invokes
-    * {@link #getBlob(String,String,org.jclouds.blobstore.options.GetOptions)}
-    * 
-    * @param container
-    *           container name
-    * @param key
-    *           blob key
-    */
-   @Override
-   public Blob getBlob(String container, String key) {
-      return getBlob(container, key, org.jclouds.blobstore.options.GetOptions.NONE);
-   }
-
    /**
     * This implementation invokes {@link AzureBlobClient#getBlob}
     * 
@@ -250,13 +168,10 @@ public class AzureBlobStore extends BaseAzureBlobStore implements BlobStore {
     */
    @Override
    public Blob getBlob(String container, String key,
-            org.jclouds.blobstore.options.GetOptions optionsList) {
+            org.jclouds.blobstore.options.GetOptions options) {
-      GetOptions azureOptions = blob2ObjectGetOptions.apply(optionsList);
+      GetOptions azureOptions = blob2ObjectGetOptions.apply(options);
-      try {
+      return azureBlob2Blob.apply(sync.getBlob(container, key, azureOptions));
-         return blob2Blob.apply(sync.getBlob(container, key, azureOptions));
+
-      } catch (Exception e) {
-         return keyNotFoundToNullOrPropagate(e);
-      }
    }
 
    /**
@@ -269,7 +184,7 @@ public class AzureBlobStore extends BaseAzureBlobStore implements BlobStore {
     */
    @Override
    public String putBlob(String container, Blob blob) {
-      return sync.putBlob(container, blob2Object.apply(blob));
+      return sync.putBlob(container, blob2AzureBlob.apply(blob));
    }
 
    /**
@@ -285,4 +200,22 @@ public class AzureBlobStore extends BaseAzureBlobStore implements BlobStore {
       sync.deleteBlob(container, key);
    }
 
+   /**
+    * This implementation invokes {@link AzureBlobClient#getBlobProperties}
+    * 
+    * @param container
+    *           container name
+    * @param key
+    *           blob key
+    */
+   @Override
+   public BlobMetadata blobMetadata(String container, String key) {
+      return blob2BlobMd.apply(sync.getBlobProperties(container, key));
+   }
+
+   @Override
+   protected boolean deleteAndVerifyContainerGone(String container) {
+      throw new UnsupportedOperationException("please use deleteContainer");
+   }
+
 }

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/AzureBlobToBlob.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.azure.storage.blob.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -47,8 +49,7 @@ public class AzureBlobToBlob implements Function<AzureBlob, Blob> {
       Blob blob = blobFactory.create(blobPr2BlobMd.apply(from.getProperties()));
       if (from.getContentLength() != null)
          blob.setContentLength(from.getContentLength());
-      if (from.getPayload() != null)
+      blob.setPayload(checkNotNull(from.getPayload(), "payload: " + from));
-         blob.setPayload(from.getPayload());
       blob.setAllHeaders(from.getAllHeaders());
       return blob;
    }

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/BlobPropertiesToBlobMetadata.java

@@ -25,7 +25,7 @@ import org.jclouds.azure.storage.blob.domain.BlobProperties;
 import org.jclouds.blobstore.domain.MutableBlobMetadata;
 import org.jclouds.blobstore.domain.StorageType;
 import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
-import org.jclouds.blobstore.strategy.IsDirectoryStrategy;
+import org.jclouds.blobstore.strategy.IfDirectoryReturnNameStrategy;
 
 import com.google.common.base.Function;
 
@@ -34,11 +34,11 @@ import com.google.common.base.Function;
  */
 @Singleton
 public class BlobPropertiesToBlobMetadata implements Function<BlobProperties, MutableBlobMetadata> {
-   private final IsDirectoryStrategy isDirectoryStrategy;
+   private final IfDirectoryReturnNameStrategy ifDirectoryReturnName;
 
    @Inject
-   public BlobPropertiesToBlobMetadata(IsDirectoryStrategy isDirectoryStrategy) {
+   public BlobPropertiesToBlobMetadata(IfDirectoryReturnNameStrategy ifDirectoryReturnName) {
-      this.isDirectoryStrategy = isDirectoryStrategy;
+      this.ifDirectoryReturnName = ifDirectoryReturnName;
    }
 
    public MutableBlobMetadata apply(BlobProperties from) {
@@ -54,9 +54,12 @@ public class BlobPropertiesToBlobMetadata implements Function<BlobProperties, Mu
       to.setLastModified(from.getLastModified());
       to.setName(from.getName());
       to.setSize(from.getContentLength());
-      to.setType(StorageType.BLOB);
+      String directoryName = ifDirectoryReturnName.execute(to);
-      if (isDirectoryStrategy.execute(to)) {
+      if (directoryName != null) {
+         to.setName(directoryName);
          to.setType(StorageType.RELATIVE_PATH);
+      } else {
+         to.setType(StorageType.BLOB);
       }
       return to;
    }

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/BlobToAzureBlob.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.azure.storage.blob.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -46,7 +48,7 @@ public class BlobToAzureBlob implements Function<Blob, AzureBlob> {
       AzureBlob object = objectProvider.create(blob2ObjectMd.apply(from.getMetadata()));
       if (from.getContentLength() != null)
          object.setContentLength(from.getContentLength());
-      object.setPayload(from.getPayload());
+      object.setPayload(checkNotNull(from.getPayload(), "payload: " + from));
       object.setAllHeaders(from.getAllHeaders());
       return object;
    }

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/ListBlobsResponseToResourceList.java

@@ -18,18 +18,22 @@
  */
 package org.jclouds.azure.storage.blob.blobstore.functions;
 
-import java.util.SortedSet;
+import java.util.Map;
+import java.util.Set;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
 import org.jclouds.azure.storage.blob.domain.ListBlobsResponse;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.blobstore.domain.internal.ListContainerResponseImpl;
+import org.jclouds.blobstore.domain.StorageType;
+import org.jclouds.blobstore.domain.internal.PageSetImpl;
+import org.jclouds.blobstore.functions.PrefixToResourceMetadata;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
 /**
@@ -37,22 +41,35 @@ import com.google.common.collect.Sets;
  */
 @Singleton
 public class ListBlobsResponseToResourceList implements
-         Function<ListBlobsResponse, ListContainerResponse<? extends StorageMetadata>> {
+         Function<ListBlobsResponse, PageSet<? extends StorageMetadata>> {
    private final BlobPropertiesToBlobMetadata object2blobMd;
-   private final CommonPrefixesToResourceMetadata prefix2ResourceMd;
+   private final PrefixToResourceMetadata prefix2ResourceMd;
+
+   protected final Function<StorageMetadata, String> indexer = new Function<StorageMetadata, String>() {
+      @Override
+      public String apply(StorageMetadata from) {
+         return from.getName();
+      }
+   };
 
    @Inject
    public ListBlobsResponseToResourceList(BlobPropertiesToBlobMetadata object2blobMd,
-            CommonPrefixesToResourceMetadata prefix2ResourceMd) {
+            PrefixToResourceMetadata prefix2ResourceMd) {
       this.object2blobMd = object2blobMd;
       this.prefix2ResourceMd = prefix2ResourceMd;
    }
 
-   public ListContainerResponse<? extends StorageMetadata> apply(ListBlobsResponse from) {
+   public PageSet<? extends StorageMetadata> apply(ListBlobsResponse from) {
-      SortedSet<StorageMetadata> contents = Sets.newTreeSet(Iterables.concat(Iterables.transform(
+      Set<StorageMetadata> contents = Sets.<StorageMetadata> newHashSet(Iterables.transform(from,
-               from, object2blobMd), prefix2ResourceMd.apply(from.getBlobPrefixes())));
+               object2blobMd));
-      return new ListContainerResponseImpl<StorageMetadata>(contents, from.getPrefix(), from
-               .getMarker(), from.getMaxResults(), from.size() == from.getMaxResults());
 
+      Map<String, StorageMetadata> nameToMd = Maps.uniqueIndex(contents, indexer);
+      for (String prefix : from.getBlobPrefixes()) {
+         prefix = prefix.endsWith("/") ? prefix.substring(0, prefix.lastIndexOf('/')) : prefix;
+         if (!nameToMd.containsKey(prefix)
+                  || nameToMd.get(prefix).getType() != StorageType.RELATIVE_PATH)
+            contents.add(prefix2ResourceMd.apply(prefix));
+      }
+      return new PageSetImpl<StorageMetadata>(contents, from.getNextMarker());
    }
 }

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/ListOptionsToListBlobsOptions.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.azure.storage.blob.blobstore.functions;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import javax.inject.Singleton;
 
 import org.jclouds.azure.storage.blob.options.ListBlobsOptions;
@@ -32,20 +34,19 @@ import com.google.common.base.Function;
 public class ListOptionsToListBlobsOptions implements
          Function<ListContainerOptions, ListBlobsOptions> {
    public ListBlobsOptions apply(ListContainerOptions from) {
+      checkNotNull(from, "set options to instance NONE instead of passing null");
       ListBlobsOptions httpOptions = new ListBlobsOptions();
-      if (from != null && from != ListContainerOptions.NONE) {
+      if (!from.isRecursive()) {
-         if (!from.isRecursive()) {
+         httpOptions.delimiter("/");
-            httpOptions.delimiter("/");
+      }
-         }
+      if (from.getDir() != null) {
-         if (from.getDir() != null) {
+         httpOptions.prefix(from.getDir().endsWith("/") ? from.getDir() : from.getDir() + "/");
-            httpOptions.prefix(from.getDir());
+      }
-         }
+      if (from.getMarker() != null) {
-         if (from.getMarker() != null) {
+         httpOptions.marker(from.getMarker());
-            httpOptions.marker(from.getMarker());
+      }
-         }
+      if (from.getMaxResults() != null) {
-         if (from.getMaxResults() != null) {
+         httpOptions.maxResults(from.getMaxResults());
-            httpOptions.maxResults(from.getMaxResults());
-         }
       }
       return httpOptions;
    }

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/internal/BaseAzureBlobStore.java

@@ -1,92 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
- *
- * ====================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ====================================================================
- */
-package org.jclouds.azure.storage.blob.blobstore.internal;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.concurrent.ExecutorService;
-
-import javax.inject.Inject;
-
-import org.jclouds.azure.storage.blob.AzureBlobAsyncClient;
-import org.jclouds.azure.storage.blob.AzureBlobClient;
-import org.jclouds.azure.storage.blob.blobstore.functions.AzureBlobToBlob;
-import org.jclouds.azure.storage.blob.blobstore.functions.BlobPropertiesToBlobMetadata;
-import org.jclouds.azure.storage.blob.blobstore.functions.BlobToAzureBlob;
-import org.jclouds.azure.storage.blob.blobstore.functions.ContainerToResourceMetadata;
-import org.jclouds.azure.storage.blob.blobstore.functions.ListBlobsResponseToResourceList;
-import org.jclouds.azure.storage.blob.blobstore.functions.ListOptionsToListBlobsOptions;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
-import org.jclouds.blobstore.strategy.GetDirectoryStrategy;
-import org.jclouds.blobstore.strategy.MkdirStrategy;
-import org.jclouds.logging.Logger.LoggerFactory;
-
-public class BaseAzureBlobStore {
-   protected final AzureBlobAsyncClient async;
-   protected final AzureBlobClient sync;
-   protected final Blob.Factory blobFactory;
-   protected final LoggerFactory logFactory;
-   protected final ClearListStrategy clearContainerStrategy;
-   protected final BlobPropertiesToBlobMetadata blob2BlobMd;
-   protected final AzureBlobToBlob blob2Blob;
-   protected final BlobToAzureBlob blob2Object;
-   protected final ListOptionsToListBlobsOptions container2ContainerListOptions;
-   protected final BlobToHttpGetOptions blob2ObjectGetOptions;
-   protected final ContainerToResourceMetadata container2ResourceMd;
-   protected final ListBlobsResponseToResourceList container2ResourceList;
-   protected final ExecutorService service;
-   protected final GetDirectoryStrategy getDirectoryStrategy;
-   protected final MkdirStrategy mkdirStrategy;
-
-   @Inject
-   protected BaseAzureBlobStore(AzureBlobAsyncClient async, AzureBlobClient sync,
-            Blob.Factory blobFactory, LoggerFactory logFactory,
-            ClearListStrategy clearContainerStrategy, BlobPropertiesToBlobMetadata blob2BlobMd,
-            AzureBlobToBlob blob2Blob, BlobToAzureBlob blob2Object,
-            ListOptionsToListBlobsOptions container2ContainerListOptions,
-            BlobToHttpGetOptions blob2ObjectGetOptions, GetDirectoryStrategy getDirectoryStrategy,
-            MkdirStrategy mkdirStrategy, ContainerToResourceMetadata container2ResourceMd,
-            ListBlobsResponseToResourceList container2ResourceList, ExecutorService service) {
-      this.async = checkNotNull(async, "async");
-      this.sync = checkNotNull(sync, "sync");
-      this.blobFactory = checkNotNull(blobFactory, "blobFactory");
-      this.logFactory = checkNotNull(logFactory, "logFactory");
-      this.clearContainerStrategy = checkNotNull(clearContainerStrategy, "clearContainerStrategy");
-      this.blob2BlobMd = checkNotNull(blob2BlobMd, "blob2BlobMd");
-      this.blob2Blob = checkNotNull(blob2Blob, "blob2Blob");
-      this.blob2Object = checkNotNull(blob2Object, "blob2Object");
-      this.container2ContainerListOptions = checkNotNull(container2ContainerListOptions,
-               "container2ContainerListOptions");
-      this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
-      this.getDirectoryStrategy = checkNotNull(getDirectoryStrategy, "getDirectoryStrategy");
-      this.mkdirStrategy = checkNotNull(mkdirStrategy, "mkdirStrategy");
-      this.container2ResourceMd = checkNotNull(container2ResourceMd, "container2ResourceMd");
-      this.container2ResourceList = checkNotNull(container2ResourceList, "container2ResourceList");
-      this.service = checkNotNull(service, "service");
-   }
-
-   public Blob newBlob(String name) {
-      Blob blob = blobFactory.create(null);
-      blob.getMetadata().setName(name);
-      return blob;
-   }
-
-}

azure/src/main/java/org/jclouds/azure/storage/blob/domain/BlobProperties.java

@@ -36,6 +36,8 @@ public interface BlobProperties extends Comparable<BlobProperties> {
     */
    BlobType getType();
 
+   LeaseStatus getLeaseStatus();
+
    URI getUrl();
 
    String getName();

azure/src/main/java/org/jclouds/azure/storage/blob/domain/LeaseStatus.java

@@ -16,24 +16,29 @@
  * limitations under the License.
  * ====================================================================
  */
-package org.jclouds.blobstore.domain.internal;
+package org.jclouds.azure.storage.blob.domain;
 
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import static com.google.common.base.Preconditions.checkNotNull;
 
-public class ListContainerResponseImpl<T> extends ListResponseImpl<T> implements ListContainerResponse<T> {
+import com.google.common.base.CaseFormat;
 
-   /** The serialVersionUID */
+/**
-   private static final long serialVersionUID = -7133632087734650835L;
+ * @author Adrian Cole
-   protected final String path;
+ */
+public enum LeaseStatus {
+   LOCKED, UNLOCKED;
 
-   public ListContainerResponseImpl(Iterable<T> contents, String path, String marker,
+   public String value() {
-            Integer maxResults, boolean isTruncated) {
+      return (CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, name()));
-      super(contents, marker, maxResults, isTruncated);
-      this.path = path;
    }
 
-   public String getPath() {
+   @Override
-      return path;
+   public String toString() {
+      return value();
    }
 
+   public static LeaseStatus fromValue(String type) {
+      return valueOf(CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, checkNotNull(type,
+               "type")));
+   }
 }

azure/src/main/java/org/jclouds/azure/storage/blob/domain/internal/AzureBlobImpl.java

@@ -124,4 +124,9 @@ public class AzureBlobImpl extends BasePayloadEnclosingImpl implements AzureBlob
       return true;
    }
 
+   @Override
+   public String toString() {
+      return "[properties=" + properties + "]";
+   }
+
 }

azure/src/main/java/org/jclouds/azure/storage/blob/domain/internal/BlobPropertiesImpl.java

@@ -28,6 +28,7 @@ import java.util.Map;
 
 import org.jclouds.azure.storage.blob.domain.BlobProperties;
 import org.jclouds.azure.storage.blob.domain.BlobType;
+import org.jclouds.azure.storage.blob.domain.LeaseStatus;
 
 import com.google.common.collect.Maps;
 import com.google.inject.internal.Nullable;
@@ -52,12 +53,14 @@ public class BlobPropertiesImpl implements Serializable, BlobProperties {
    private final String contentEncoding;
    private final String contentLanguage;
    private final Map<String, String> metadata = Maps.newLinkedHashMap();
+   private final LeaseStatus leaseStatus;
 
    public BlobPropertiesImpl(BlobType type, String name, URI url, Date lastModified, String eTag,
             long size, String contentType, @Nullable byte[] contentMD5,
             @Nullable String contentEncoding, @Nullable String contentLanguage,
-            Map<String, String> metadata) {
+            LeaseStatus leaseStatus, Map<String, String> metadata) {
       this.type = checkNotNull(type, "type");
+      this.leaseStatus = checkNotNull(leaseStatus, "leaseStatus");
       this.name = checkNotNull(name, "name");
       this.url = checkNotNull(url, "url");
       this.lastModified = checkNotNull(lastModified, "lastModified");
@@ -166,6 +169,14 @@ public class BlobPropertiesImpl implements Serializable, BlobProperties {
       return url;
    }
 
+   /**
+    *{@inheritDoc}
+    */
+   @Override
+   public LeaseStatus getLeaseStatus() {
+      return leaseStatus;
+   }
+
    @Override
    public int hashCode() {
       final int prime = 31;
@@ -176,6 +187,7 @@ public class BlobPropertiesImpl implements Serializable, BlobProperties {
       result = prime * result + ((contentType == null) ? 0 : contentType.hashCode());
       result = prime * result + ((eTag == null) ? 0 : eTag.hashCode());
       result = prime * result + ((lastModified == null) ? 0 : lastModified.hashCode());
+      result = prime * result + ((leaseStatus == null) ? 0 : leaseStatus.hashCode());
       result = prime * result + ((metadata == null) ? 0 : metadata.hashCode());
       result = prime * result + ((name == null) ? 0 : name.hashCode());
       result = prime * result + (int) (size ^ (size >>> 32));
@@ -220,6 +232,11 @@ public class BlobPropertiesImpl implements Serializable, BlobProperties {
             return false;
       } else if (!lastModified.equals(other.lastModified))
          return false;
+      if (leaseStatus == null) {
+         if (other.leaseStatus != null)
+            return false;
+      } else if (!leaseStatus.equals(other.leaseStatus))
+         return false;
       if (metadata == null) {
          if (other.metadata != null)
             return false;
@@ -245,4 +262,10 @@ public class BlobPropertiesImpl implements Serializable, BlobProperties {
       return true;
    }
 
+   @Override
+   public String toString() {
+      return "[name=" + name + ", type=" + type + ", contentType=" + contentType
+               + ", eTag=" + eTag + ", lastModified=" + lastModified + ", size=" + size + "]";
+   }
+
 }

azure/src/main/java/org/jclouds/azure/storage/blob/domain/internal/HashSetListBlobsResponse.java

@@ -56,4 +56,41 @@ public class HashSetListBlobsResponse extends BoundedHashSet<BlobProperties> imp
    public Set<String> getBlobPrefixes() {
       return blobPrefixes;
    }
+
+   @Override
+   public int hashCode() {
+      final int prime = 31;
+      int result = super.hashCode();
+      result = prime * result + ((blobPrefixes == null) ? 0 : blobPrefixes.hashCode());
+      result = prime * result + ((delimiter == null) ? 0 : delimiter.hashCode());
+      return result;
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      if (this == obj)
+         return true;
+      if (!super.equals(obj))
+         return false;
+      if (getClass() != obj.getClass())
+         return false;
+      HashSetListBlobsResponse other = (HashSetListBlobsResponse) obj;
+      if (blobPrefixes == null) {
+         if (other.blobPrefixes != null)
+            return false;
+      } else if (!blobPrefixes.equals(other.blobPrefixes))
+         return false;
+      if (delimiter == null) {
+         if (other.delimiter != null)
+            return false;
+      } else if (!delimiter.equals(other.delimiter))
+         return false;
+      return true;
+   }
+
+   @Override
+   public String toString() {
+      return "[blobPrefixes=" + blobPrefixes + ", delimiter=" + delimiter
+               + "]";
+   }
 }

azure/src/main/java/org/jclouds/azure/storage/blob/domain/internal/MutableBlobPropertiesImpl.java

@@ -26,6 +26,7 @@ import java.util.Map;
 
 import org.jclouds.azure.storage.blob.domain.BlobProperties;
 import org.jclouds.azure.storage.blob.domain.BlobType;
+import org.jclouds.azure.storage.blob.domain.LeaseStatus;
 import org.jclouds.azure.storage.blob.domain.MutableBlobProperties;
 
 import com.google.common.collect.Maps;
@@ -41,6 +42,8 @@ public class MutableBlobPropertiesImpl implements Serializable, MutableBlobPrope
    private static final long serialVersionUID = -4648755473986695062L;
 
    private BlobType type = BlobType.BLOCK_BLOB;
+   private LeaseStatus leaseStatus = LeaseStatus.UNLOCKED;
+
    private String name;
    private URI url;
    private Date lastModified;
@@ -156,6 +159,14 @@ public class MutableBlobPropertiesImpl implements Serializable, MutableBlobPrope
       }
    }
 
+   /**
+    *{@inheritDoc}
+    */
+   @Override
+   public LeaseStatus getLeaseStatus() {
+      return leaseStatus;
+   }
+
    /**
     *{@inheritDoc}
     */
@@ -293,4 +304,9 @@ public class MutableBlobPropertiesImpl implements Serializable, MutableBlobPrope
       return true;
    }
 
+   @Override
+   public String toString() {
+      return "[name=" + name + ", type=" + type + ", lastModified=" + lastModified + "]";
+   }
+
 }

azure/src/main/java/org/jclouds/azure/storage/blob/functions/ParseBlobFromHeadersAndHttpContent.java

@@ -19,16 +19,15 @@
 package org.jclouds.azure.storage.blob.functions;
 
 import javax.inject.Inject;
+import javax.inject.Singleton;
 import javax.ws.rs.core.HttpHeaders;
 
 import org.jclouds.azure.storage.blob.domain.AzureBlob;
 import org.jclouds.blobstore.functions.ParseSystemAndUserMetadataFromHeaders;
-import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpResponse;
 import org.jclouds.rest.InvocationContext;
 import org.jclouds.rest.internal.GeneratedHttpRequest;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 
 /**
@@ -37,6 +36,7 @@ import com.google.common.base.Function;
  * @see ParseMetadataFromHeaders
  * @author Adrian Cole
  */
+@Singleton
 public class ParseBlobFromHeadersAndHttpContent implements Function<HttpResponse, AzureBlob>,
          InvocationContext {
 
@@ -60,16 +60,8 @@ public class ParseBlobFromHeadersAndHttpContent implements Function<HttpResponse
     */
    public AzureBlob apply(HttpResponse from) {
       AzureBlob object = objectProvider.create(metadataParser.apply(from));
-      addAllHeadersTo(from, object);
+      object.getAllHeaders().putAll(from.getHeaders());
-      if (from.getContent() != null)
-         object.setPayload(from.getContent());
-      attemptToParseSizeAndRangeFromHeaders(from, object);
-      return object;
-   }
 
-   @VisibleForTesting
-   void attemptToParseSizeAndRangeFromHeaders(HttpResponse from, AzureBlob object)
-            throws HttpException {
       String contentLength = from.getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH);
       String contentRange = from.getFirstHeaderOrNull("Content-Range");
 
@@ -77,17 +69,21 @@ public class ParseBlobFromHeadersAndHttpContent implements Function<HttpResponse
          object.setContentLength(Long.parseLong(contentLength));
       }
 
+      if (from.getContent() != null) {
+         object.setPayload(from.getContent());
+      } else if (object.getContentLength() != null && object.getContentLength() == 0) {
+         object.setPayload(new byte[0]);
+      } else {
+         assert false : "no content in " + from;
+      }
+
       if (contentRange == null && contentLength != null) {
          object.getProperties().setContentLength(object.getContentLength());
       } else if (contentRange != null) {
          object.getProperties().setContentLength(
                   Long.parseLong(contentRange.substring(contentRange.lastIndexOf('/') + 1)));
       }
-   }
+      return object;
-
-   @VisibleForTesting
-   void addAllHeadersTo(HttpResponse from, AzureBlob object) {
-      object.getAllHeaders().putAll(from.getHeaders());
    }
 
    public void setContext(GeneratedHttpRequest<?> request) {

azure/src/main/java/org/jclouds/azure/storage/blob/functions/ReturnFalseIfContainerAlreadyExists.java

@@ -28,13 +28,13 @@ import com.google.common.base.Function;
  * 
  * @author Adrian Cole
  */
-public class ReturnTrueIfContainerAlreadyExists implements Function<Exception, Boolean> {
+public class ReturnFalseIfContainerAlreadyExists implements Function<Exception, Boolean> {
 
    public Boolean apply(Exception from) {
       if (from instanceof AzureStorageResponseException) {
          AzureStorageResponseException responseException = (AzureStorageResponseException) from;
          if ("ContainerAlreadyExists".equals(responseException.getError().getCode())) {
-            return true;
+            return false;
          }
       }
       return Boolean.class.cast(propagateOrNull(from));

azure/src/main/java/org/jclouds/azure/storage/blob/xml/ContainerNameEnumerationResultsHandler.java

@@ -27,6 +27,7 @@ import javax.inject.Inject;
 
 import org.jclouds.azure.storage.blob.domain.BlobProperties;
 import org.jclouds.azure.storage.blob.domain.BlobType;
+import org.jclouds.azure.storage.blob.domain.LeaseStatus;
 import org.jclouds.azure.storage.blob.domain.ListBlobsResponse;
 import org.jclouds.azure.storage.blob.domain.internal.BlobPropertiesImpl;
 import org.jclouds.azure.storage.blob.domain.internal.HashSetListBlobsResponse;
@@ -77,6 +78,7 @@ public class ContainerNameEnumerationResultsHandler extends
    private Set<String> blobPrefixes = Sets.newHashSet();
    private byte[] currentContentMD5;
    private Map<String, String> currentMetadata = Maps.newHashMap();
+   private LeaseStatus currentLeaseStatus;
 
    @Inject
    public ContainerNameEnumerationResultsHandler(EncryptionService encryptionService,
@@ -129,11 +131,13 @@ public class ContainerNameEnumerationResultsHandler extends
          nextMarker = (nextMarker.equals("")) ? null : nextMarker;
       } else if (qName.equals("BlobType")) {
          currentBlobType = BlobType.fromValue(currentText.toString().trim());
+      } else if (qName.equals("LeaseStatus")) {
+         currentLeaseStatus = LeaseStatus.fromValue(currentText.toString().trim());
       } else if (qName.equals("Blob")) {
          BlobProperties md = new BlobPropertiesImpl(currentBlobType, currentName, currentUrl,
                   currentLastModified, currentETag, currentSize, currentContentType,
                   currentContentMD5, currentContentEncoding, currentContentLanguage,
-                  currentMetadata);
+                  currentLeaseStatus, currentMetadata);
          blobMetadata.add(md);
          currentBlobType = null;
          currentName = null;
@@ -145,6 +149,7 @@ public class ContainerNameEnumerationResultsHandler extends
          currentContentEncoding = null;
          currentContentLanguage = null;
          currentContentMD5 = null;
+         currentLeaseStatus = null;
          currentMetadata = Maps.newHashMap();
       } else if (qName.equals("Url")) {
          currentUrl = HttpUtils.createUri(currentText.toString().trim());
@@ -160,7 +165,8 @@ public class ContainerNameEnumerationResultsHandler extends
       } else if (qName.equals("Content-Length")) {
          currentSize = Long.parseLong(currentText.toString().trim());
       } else if (qName.equals("Content-MD5")) {
-         currentContentMD5 = encryptionService.fromBase64String(currentText.toString().trim());
+         if (!currentText.toString().trim().equals(""))
+            currentContentMD5 = encryptionService.fromBase64String(currentText.toString().trim());
       } else if (qName.equals("Content-Type")) {
          currentContentType = currentText.toString().trim();
       } else if (qName.equals("Content-Encoding")) {

azure/src/main/java/org/jclouds/azure/storage/handlers/ParseAzureStorageErrorFromXmlContent.java

@@ -58,50 +58,56 @@ public class ParseAzureStorageErrorFromXmlContent implements HttpErrorHandler {
       this.utils = utils;
    }
 
-   public static final Pattern CONTAINER_PATH = Pattern.compile("^[/]?([^/]+)");
+   public static final Pattern CONTAINER_PATH = Pattern.compile("^[/]?([^/]+)$");
-   public static final Pattern CONTAINER_KEY_PATH = Pattern.compile("^[/]?([^/]+)/(.*)");
+   public static final Pattern CONTAINER_KEY_PATH = Pattern.compile("^[/]?([^/]+)/(.*)$");
 
    public void handleError(HttpCommand command, HttpResponse response) {
       Exception exception = new HttpResponseException(command, response);
       try {
+         AzureStorageError error = parseErrorFromContentOrNull(command, response);
          switch (response.getStatusCode()) {
             case 401:
-               exception = new AuthorizationException(command.getRequest().getRequestLine());
+               exception = new AuthorizationException(command.getRequest(), error != null ? error
+                        .getMessage() : response.getStatusLine());
                break;
             case 404:
                if (!command.getRequest().getMethod().equals("DELETE")) {
+                  String message = error != null ? error.getMessage() : String.format("%s -> %s",
+                           command.getRequest().getRequestLine(), response.getStatusLine());
                   String path = command.getRequest().getEndpoint().getPath();
                   Matcher matcher = CONTAINER_PATH.matcher(path);
                   if (matcher.find()) {
-                     exception = new ContainerNotFoundException(matcher.group(1));
+                     exception = new ContainerNotFoundException(matcher.group(1), message);
                   } else {
                      matcher = CONTAINER_KEY_PATH.matcher(path);
                      if (matcher.find()) {
-                        exception = new KeyNotFoundException(matcher.group(1), matcher.group(2));
+                        exception = new KeyNotFoundException(matcher.group(1), matcher.group(2),
+                                 message);
                      }
                   }
                }
                break;
             default:
-               if (response.getContent() != null) {
+               exception = error != null ? new AzureStorageResponseException(command, response,
-                  try {
+                        error) : new HttpResponseException(command, response);
-                     String content = Utils.toStringAndClose(response.getContent());
-                     if (content.indexOf('<') >= 0) {
-                        AzureStorageError error = utils.parseAzureStorageErrorFromContent(command,
-                                 response, Utils.toInputStream(content));
-                        exception = new AzureStorageResponseException(command, response, error);
-
-                     } else {
-                        exception = new HttpResponseException(command, response, content);
-                     }
-                  } catch (IOException e) {
-                     logger.warn(e, "exception reading error from response", response);
-                  }
-               }
          }
       } finally {
          Closeables.closeQuietly(response.getContent());
          command.setException(exception);
       }
    }
+
+   AzureStorageError parseErrorFromContentOrNull(HttpCommand command, HttpResponse response) {
+      if (response.getContent() != null) {
+         try {
+            String content = Utils.toStringAndClose(response.getContent());
+            if (content != null && content.indexOf('<') >= 0)
+               return utils.parseAzureStorageErrorFromContent(command, response, Utils
+                        .toInputStream(content));
+         } catch (IOException e) {
+            logger.warn(e, "exception reading error from response", response);
+         }
+      }
+      return null;
+   }
 }

azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobAsyncClientTest.java

@@ -35,12 +35,13 @@ import javax.ws.rs.core.HttpHeaders;
 import org.jclouds.Constants;
 import org.jclouds.azure.storage.AzureBlob;
 import org.jclouds.azure.storage.blob.functions.ParseContainerPropertiesFromHeaders;
-import org.jclouds.azure.storage.blob.functions.ReturnTrueIfContainerAlreadyExists;
+import org.jclouds.azure.storage.blob.functions.ReturnFalseIfContainerAlreadyExists;
 import org.jclouds.azure.storage.blob.options.CreateContainerOptions;
 import org.jclouds.azure.storage.blob.options.ListBlobsOptions;
 import org.jclouds.azure.storage.config.AzureStorageRestClientModule;
 import org.jclouds.azure.storage.options.ListOptions;
 import org.jclouds.azure.storage.reference.AzureStorageConstants;
+import org.jclouds.blobstore.functions.ReturnNullOnContainerNotFound;
 import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
 import org.jclouds.blobstore.reference.BlobStoreConstants;
 import org.jclouds.concurrent.config.ExecutorServiceModule;
@@ -138,7 +139,7 @@ public class AzureBlobAsyncClientTest {
       // TODO check generic type of response parser
       assertEquals(processor
                .createExceptionParserOrThrowResourceNotFoundOn404IfNoAnnotation(method).getClass(),
-               ReturnTrueIfContainerAlreadyExists.class);
+               ReturnFalseIfContainerAlreadyExists.class);
    }
 
    public void testDeleteContainer() throws SecurityException, NoSuchMethodException {
@@ -184,7 +185,7 @@ public class AzureBlobAsyncClientTest {
       // TODO check generic type of response parser
       assertEquals(processor
                .createExceptionParserOrThrowResourceNotFoundOn404IfNoAnnotation(method).getClass(),
-               ReturnTrueIfContainerAlreadyExists.class);
+               ReturnFalseIfContainerAlreadyExists.class);
    }
 
    public void testCreateRootContainer() throws SecurityException, NoSuchMethodException {
@@ -206,7 +207,7 @@ public class AzureBlobAsyncClientTest {
       // TODO check generic type of response parser
       assertEquals(processor
                .createExceptionParserOrThrowResourceNotFoundOn404IfNoAnnotation(method).getClass(),
-               ReturnTrueIfContainerAlreadyExists.class);
+               ReturnFalseIfContainerAlreadyExists.class);
    }
 
    public void testDeleteRootContainer() throws SecurityException, NoSuchMethodException {
@@ -251,7 +252,7 @@ public class AzureBlobAsyncClientTest {
       // TODO check generic type of response parser
       assertEquals(processor
                .createExceptionParserOrThrowResourceNotFoundOn404IfNoAnnotation(method).getClass(),
-               ReturnTrueIfContainerAlreadyExists.class);
+               ReturnFalseIfContainerAlreadyExists.class);
    }
 
    public void testListBlobs() throws SecurityException, NoSuchMethodException {
@@ -310,7 +311,7 @@ public class AzureBlobAsyncClientTest {
                ParseContainerPropertiesFromHeaders.class);
       assertEquals(processor
                .createExceptionParserOrThrowResourceNotFoundOn404IfNoAnnotation(method).getClass(),
-               MapHttp4xxCodesToExceptions.class);
+               ReturnNullOnContainerNotFound.class);
    }
 
    public void testSetResourceMetadata() throws SecurityException, NoSuchMethodException {

azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobClientLiveTest.java

@@ -237,12 +237,7 @@ public class AzureBlobClientLiveTest {
                .getProperties().getContentMD5()));
 
       // Test HEAD of missing object
-      try {
+      assert connection.getBlobProperties(privateContainer, "non-existent-object") == null;
-         connection.getBlobProperties(privateContainer, "non-existent-object");
-         assert false;
-      } catch (Exception e) {
-         e.printStackTrace();
-      }
 
       // Test HEAD of object
       BlobProperties metadata = connection.getBlobProperties(privateContainer, object
@@ -270,12 +265,8 @@ public class AzureBlobClientLiveTest {
       // userMetadata));
 
       // Test GET of missing object
-      try {
+      assert connection.getBlob(privateContainer, "non-existent-object") == null;
-         connection.getBlob(privateContainer, "non-existent-object");
+
-         assert false;
-      } catch (Exception e) {
-         e.printStackTrace();
-      }
       // Test GET of object (including updated metadata)
       AzureBlob getBlob = connection.getBlob(privateContainer, object.getProperties().getName());
       assertEquals(Utils.toStringAndClose(getBlob.getContent()), data);

azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobContextBuilderTest.java

@@ -0,0 +1,109 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.azure.storage.blob;
+
+import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX;
+import static org.testng.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.jclouds.azure.storage.blob.AzureBlobAsyncClient;
+import org.jclouds.azure.storage.blob.AzureBlobClient;
+import org.jclouds.azure.storage.blob.AzureBlobPropertiesBuilder;
+import org.jclouds.azure.storage.blob.AzureBlobContextBuilder;
+import org.jclouds.azure.storage.blob.blobstore.AzureAsyncBlobStore;
+import org.jclouds.azure.storage.blob.blobstore.config.AzureBlobStoreContextModule;
+import org.jclouds.azure.storage.blob.config.AzureBlobRestClientModule;
+import org.jclouds.azure.storage.blob.config.AzureBlobStubClientModule;
+import org.jclouds.azure.storage.blob.domain.AzureBlob;
+import org.jclouds.azure.storage.blob.domain.internal.AzureBlobImpl;
+import org.jclouds.azure.storage.blob.internal.StubAzureBlobAsyncClient;
+import org.jclouds.azure.storage.reference.AzureStorageConstants;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.BlobStoreContextBuilder;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.internal.BlobImpl;
+import org.jclouds.blobstore.internal.BlobStoreContextImpl;
+import org.testng.annotations.Test;
+
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+/**
+ * Tests behavior of modules configured in AzureBlobContextBuilder
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "azureblob.AzureBlobContextBuilderTest")
+public class AzureBlobContextBuilderTest {
+
+   public void testNewBuilder() {
+      BlobStoreContextBuilder<AzureBlobAsyncClient, AzureBlobClient> builder = newBuilder();
+      assertEquals(builder.getProperties().getProperty(PROPERTY_USER_METADATA_PREFIX), "x-ms-meta-");
+      assertEquals(builder.getProperties().getProperty(
+               AzureStorageConstants.PROPERTY_AZURESTORAGE_ACCOUNT), "id");
+      assertEquals(builder.getProperties().getProperty(
+               AzureStorageConstants.PROPERTY_AZURESTORAGE_KEY), "secret");
+   }
+
+   private BlobStoreContextBuilder<AzureBlobAsyncClient, AzureBlobClient> newBuilder() {
+      return new AzureBlobContextBuilder(new AzureBlobPropertiesBuilder("id", "secret")
+               .build()).withModules(new AzureBlobStubClientModule());
+   }
+
+   public void testBuildContext() {
+      BlobStoreContext context = newBuilder().buildBlobStoreContext();
+      assertEquals(context.getClass(), BlobStoreContextImpl.class);
+      assertEquals(context.getProviderSpecificContext().getAsyncApi().getClass(),
+               StubAzureBlobAsyncClient.class);
+      assertEquals(context.getAsyncBlobStore().getClass(), AzureAsyncBlobStore.class);
+      assertEquals(((AzureBlobAsyncClient) context.getProviderSpecificContext().getAsyncApi())
+               .newBlob().getClass(), AzureBlobImpl.class);
+      assertEquals(context.getAsyncBlobStore().newBlob(null).getClass(), BlobImpl.class);
+      assertEquals(context.getProviderSpecificContext().getAccount(), "id");
+      assertEquals(context.getProviderSpecificContext().getEndPoint(), URI
+               .create("https://localhost/azurestub"));
+   }
+
+   public void testBuildInjector() {
+      Injector i = newBuilder().buildInjector();
+      assert i.getInstance(BlobStoreContext.class) != null;
+      assert i.getInstance(AzureBlob.class) != null;
+      assert i.getInstance(Blob.class) != null;
+   }
+
+   protected void testAddContextModule() {
+      List<Module> modules = new ArrayList<Module>();
+      AzureBlobContextBuilder builder = (AzureBlobContextBuilder) newBuilder();
+      builder.addContextModule(modules);
+      assertEquals(modules.size(), 1);
+      assertEquals(modules.get(0).getClass(), AzureBlobStoreContextModule.class);
+   }
+
+   protected void addClientModule() {
+      List<Module> modules = new ArrayList<Module>();
+      AzureBlobContextBuilder builder = (AzureBlobContextBuilder) newBuilder();
+      builder.addClientModule(modules);
+      assertEquals(modules.size(), 1);
+      assertEquals(modules.get(0).getClass(), AzureBlobRestClientModule.class);
+   }
+
+}

azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/functions/ResourceToListBlobsResponse.java

@@ -28,7 +28,7 @@ import org.jclouds.azure.storage.blob.domain.ListBlobsResponse;
 import org.jclouds.azure.storage.blob.domain.MutableBlobProperties;
 import org.jclouds.azure.storage.blob.domain.internal.HashSetListBlobsResponse;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.domain.StorageType;
 
@@ -42,7 +42,7 @@ import com.google.common.collect.Sets;
  */
 @Singleton
 public class ResourceToListBlobsResponse implements
-         Function<ListContainerResponse<? extends StorageMetadata>, ListBlobsResponse> {
+         Function<PageSet<? extends StorageMetadata>, ListBlobsResponse> {
    private final BlobMetadataToBlobProperties blob2ObjectMd;
 
    @Inject
@@ -50,7 +50,7 @@ public class ResourceToListBlobsResponse implements
       this.blob2ObjectMd = blob2ObjectMd;
    }
 
-   public ListBlobsResponse apply(ListContainerResponse<? extends StorageMetadata> list) {
+   public ListBlobsResponse apply(PageSet<? extends StorageMetadata> list) {
 
       Iterable<BlobProperties> contents = Iterables.transform(Iterables.filter(list,
                new Predicate<StorageMetadata>() {
@@ -81,7 +81,7 @@ public class ResourceToListBlobsResponse implements
          }
 
       }));
-      return new HashSetListBlobsResponse(contents, null, list.getPath(), null, list
+      return new HashSetListBlobsResponse(contents, null, null, null, null, list.getNextMarker(),
-               .getMaxResults(), list.getMarker(), "/", commonPrefixes);
+               "/", commonPrefixes);
    }
 }

azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobInputStreamMapIntegrationTest.java

@@ -27,4 +27,9 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "azureblob.AzureBlobInputStreamMapIntegrationTest")
 public class AzureBlobInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
 
+   @Override
+   protected int maxList() {
+      return 5000;
+   }
+
 }

azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobMapIntegrationTest.java

@@ -27,4 +27,9 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "azureblob.AzureBlobMapIntegrationTest")
 public class AzureBlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
 
+   @Override
+   protected int maxList() {
+      return 5000;
+   }
+
 }

azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobTestInitializer.java

@@ -40,15 +40,18 @@ public class AzureBlobTestInitializer extends BaseTestInitializer {
    @Override
    protected BlobStoreContext createLiveContext(Module configurationModule, String url, String app,
             String account, String key) throws IOException {
+      Properties properties = new Properties();
+//      properties.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, Integer.toString(0));
+//      properties.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, Integer.toString(0));
+//      properties.setProperty(PROPERTY_USER_THREADS, Integer.toString(0));
+//      properties.setProperty(PROPERTY_IO_WORKER_THREADS, Integer.toString(20));
       return (BlobStoreContext) new BlobStoreContextFactory().createContext("azureblob", account,
-               key, ImmutableSet.of(configurationModule, new Log4JLoggingModule()),
+               key, ImmutableSet.of(configurationModule, new Log4JLoggingModule()), properties);
-               new Properties());
    }
 
    @Override
    protected BlobStoreContext createStubContext() {
-      return AzureBlobContextFactory.createContext("user", "pass",
+      return AzureBlobContextFactory.createContext("user", "pass", new AzureBlobStubClientModule());
-               new AzureBlobStubClientModule());
    }
 
 }

azure/src/test/java/org/jclouds/azure/storage/blob/internal/StubAzureBlobAsyncClient.java

@@ -128,7 +128,6 @@ public class StubAzureBlobAsyncClient implements AzureBlobAsyncClient {
 
                   @Override
                   public BlobProperties apply(BlobMetadata from) {
-
                      return blob2ObjectInfo.apply(from);
                   }
 

azure/src/test/java/org/jclouds/azure/storage/blob/xml/ContainerNameEnumerationResultsHandlerTest.java

@@ -26,6 +26,7 @@ import java.util.Set;
 
 import org.jclouds.azure.storage.blob.domain.BlobProperties;
 import org.jclouds.azure.storage.blob.domain.BlobType;
+import org.jclouds.azure.storage.blob.domain.LeaseStatus;
 import org.jclouds.azure.storage.blob.domain.ListBlobsResponse;
 import org.jclouds.azure.storage.blob.domain.internal.BlobPropertiesImpl;
 import org.jclouds.azure.storage.blob.domain.internal.HashSetListBlobsResponse;
@@ -60,16 +61,18 @@ public class ContainerNameEnumerationResultsHandlerTest extends BaseHandlerTest
       contents.add(new BlobPropertiesImpl(BlobType.BLOCK_BLOB, "blob1.txt", URI
                .create("http://myaccount.blob.core.windows.net/mycontainer/blob1.txt"), dateService
                .rfc822DateParse("Thu, 18 Sep 2008 18:41:57 GMT"), "0x8CAE7D55D050B8B", 8,
-               "text/plain; charset=UTF-8", null, null, null, ImmutableMap.<String, String> of()));
+               "text/plain; charset=UTF-8", null, null, null, LeaseStatus.UNLOCKED, ImmutableMap
+                        .<String, String> of()));
       contents.add(new BlobPropertiesImpl(BlobType.BLOCK_BLOB, "blob2.txt", URI
                .create("http://myaccount.blob.core.windows.net/mycontainer/blob2.txt"), dateService
                .rfc822DateParse("Thu, 18 Sep 2008 18:41:57 GMT"), "0x8CAE7D55CF6C339", 14,
-               "text/plain; charset=UTF-8", null, null, null, ImmutableMap.<String, String> of()));
+               "text/plain; charset=UTF-8", null, null, null, LeaseStatus.UNLOCKED, ImmutableMap
+                        .<String, String> of()));
       contents.add(new BlobPropertiesImpl(BlobType.PAGE_BLOB, "newblob1.txt", URI
                .create("http://myaccount.blob.core.windows.net/mycontainer/newblob1.txt"),
                dateService.rfc822DateParse("Thu, 18 Sep 2008 18:41:57 GMT"), "0x8CAE7D55CF6C339",
-               25, "text/plain; charset=UTF-8", null, null, null, ImmutableMap
+               25, "text/plain; charset=UTF-8", null, null, null, LeaseStatus.UNLOCKED,
-                        .<String, String> of()));
+               ImmutableMap.<String, String> of()));
 
       ListBlobsResponse list = new HashSetListBlobsResponse(contents, URI
                .create("http://myaccount.blob.core.windows.net/mycontainer"),
@@ -81,4 +84,25 @@ public class ContainerNameEnumerationResultsHandlerTest extends BaseHandlerTest
 
       assertEquals(result, list);
    }
+
+   public void testOptions() {
+      InputStream is = getClass().getResourceAsStream("/blob/test_list_blobs_options.xml");
+      Set<BlobProperties> contents = Sets.newTreeSet();
+      contents.add(new BlobPropertiesImpl(BlobType.BLOCK_BLOB, "a", URI
+               .create("https://jclouds.blob.core.windows.net/adriancole-blobstore3/a"),
+               dateService.rfc822DateParse("Sat, 30 Jan 2010 17:46:15 GMT"), "0x8CC6FEB41736428",
+               8, "application/octet-stream", null, null, null, LeaseStatus.UNLOCKED, ImmutableMap
+                        .<String, String> of()));
+
+      ListBlobsResponse list = new HashSetListBlobsResponse(contents, URI
+               .create("https://jclouds.blob.core.windows.net/adriancole-blobstore3"),
+
+      null, null, 1, "2!68!MDAwMDA2IWFwcGxlcyEwMDAwMjghOTk5OS0xMi0zMVQyMzo1OTo1OS45OTk5OTk5WiE-",
+               "/", Sets.<String> newTreeSet());
+
+      ListBlobsResponse result = (ListBlobsResponse) factory.create(
+               injector.getInstance(ContainerNameEnumerationResultsHandler.class)).parse(is);
+
+      assertEquals(result, list);
+   }
 }

azure/src/test/resources/blob/test_list_blobs.xml

@@ -1,68 +1,49 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!--
-
-
-    Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
-
-    ====================================================================
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
-    ====================================================================
-
--->
 <EnumerationResults ContainerName="http://myaccount.blob.core.windows.net/mycontainer">
-  <MaxResults>4</MaxResults>
+    <MaxResults>4</MaxResults>
-  <Blobs>
+    <Blobs>
-    <Blob>
+        <Blob>
-      <Name>blob1.txt</Name>
+            <Name>blob1.txt</Name>
-      <Url>http://myaccount.blob.core.windows.net/mycontainer/blob1.txt</Url>
+            <Url>http://myaccount.blob.core.windows.net/mycontainer/blob1.txt
-      <Last-Modified>Thu, 18 Sep 2008 18:41:57 GMT</Last-Modified>
+            </Url>
-      <Etag>0x8CAE7D55D050B8B</Etag>
+            <Last-Modified>Thu, 18 Sep 2008 18:41:57 GMT</Last-Modified>
-      <Content-Length>8</Content-Length>
+            <Etag>0x8CAE7D55D050B8B</Etag>
-      <Content-Type>text/plain; charset=UTF-8</Content-Type>
+            <Content-Length>8</Content-Length>
-      <BlobType>BlockBlob</BlobType>
+            <Content-Type>text/plain; charset=UTF-8</Content-Type>
-      <Content-Encoding />
+            <BlobType>BlockBlob</BlobType>
-      <Content-Language />
+            <LeaseStatus>unlocked</LeaseStatus>
-    </Blob>
+            <Content-Encoding />
-    <Blob>
+            <Content-Language />
-      <Name>blob2.txt</Name>
+        </Blob>
-      <Url>http://myaccount.blob.core.windows.net/mycontainer/blob2.txt</Url>
+        <Blob>
-      <Last-Modified>Thu, 18 Sep 2008 18:41:57 GMT</Last-Modified>
+            <Name>blob2.txt</Name>
-      <Etag>0x8CAE7D55CF6C339</Etag>
+            <Url>http://myaccount.blob.core.windows.net/mycontainer/blob2.txt
-      <Content-Length>14</Content-Length>
+            </Url>
-      <Content-Type>text/plain; charset=UTF-8</Content-Type>
+            <Last-Modified>Thu, 18 Sep 2008 18:41:57 GMT</Last-Modified>
-      <BlobType>BlockBlob</BlobType>
+            <Etag>0x8CAE7D55CF6C339</Etag>
-      <Content-Encoding />
+            <Content-Length>14</Content-Length>
-      <Content-Language />
+            <Content-Type>text/plain; charset=UTF-8</Content-Type>
-    </Blob>
+            <BlobType>BlockBlob</BlobType>
-    <BlobPrefix>
+            <LeaseStatus>unlocked</LeaseStatus>
-      <Name>myfolder/</Name>
+            <Content-Encoding />
-    </BlobPrefix>
+            <Content-Language />
-    <Blob>
+        </Blob>
-      <Name>newblob1.txt</Name>
+        <BlobPrefix>
-      <Url>http://myaccount.blob.core.windows.net/mycontainer/newblob1.txt</Url>
+            <Name>myfolder/</Name>
-      <Last-Modified>Thu, 18 Sep 2008 18:41:57 GMT</Last-Modified>
+        </BlobPrefix>
-      <Etag>0x8CAE7D55CF6C339</Etag>
+        <Blob>
-      <Content-Length>25</Content-Length>
+            <Name>newblob1.txt</Name>
-      <Content-Type>text/plain; charset=UTF-8</Content-Type>
+            <Url>http://myaccount.blob.core.windows.net/mycontainer/newblob1.txt
-      <BlobType>PageBlob</BlobType>
+            </Url>
-      <Content-Encoding />
+            <Last-Modified>Thu, 18 Sep 2008 18:41:57 GMT</Last-Modified>
-      <Content-Language />
+            <Etag>0x8CAE7D55CF6C339</Etag>
-    </Blob>
+            <Content-Length>25</Content-Length>
-  </Blobs>
+            <Content-Type>text/plain; charset=UTF-8</Content-Type>
-  <NextMarker>newblob2.txt</NextMarker>
+            <BlobType>PageBlob</BlobType>
+            <LeaseStatus>unlocked</LeaseStatus>
+            <Content-Encoding />
+            <Content-Language />
+        </Blob>
+    </Blobs>
+    <NextMarker>newblob2.txt</NextMarker>
 </EnumerationResults>

azure/src/test/resources/blob/test_list_blobs_options.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EnumerationResults
+    ContainerName="https://jclouds.blob.core.windows.net/adriancole-blobstore3">
+    <MaxResults>1</MaxResults>
+    <Delimiter>/</Delimiter>
+    <Blobs>
+        <Blob>
+            <Name>a</Name>
+            <Url>https://jclouds.blob.core.windows.net/adriancole-blobstore3/a</Url>
+            <Properties>
+                <Last-Modified>Sat, 30 Jan 2010 17:46:15 GMT</Last-Modified>
+                <Etag>0x8CC6FEB41736428</Etag>
+                <Content-Length>8</Content-Length>
+                <Content-Type>application/octet-stream</Content-Type>
+                <Content-Encoding />
+                <Content-Language />
+                <Content-MD5 />
+                <Cache-Control />
+                <BlobType>BlockBlob</BlobType>
+                <LeaseStatus>unlocked</LeaseStatus>
+            </Properties>
+            <Metadata />
+        </Blob>
+    </Blobs>
+    <NextMarker>2!68!MDAwMDA2IWFwcGxlcyEwMDAwMjghOTk5OS0xMi0zMVQyMzo1OTo1OS45OTk5OTk5WiE-</NextMarker>
+</EnumerationResults>

azure/src/test/resources/log4j.xml

@@ -79,9 +79,38 @@
         <appender-ref ref="WIREFILE" />
     </appender>
 
+    <!-- A time/date based rolling appender -->
+    <appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="target/test-data/jclouds-blobstore.log" />
+        <param name="Append" value="true" />
+
+        <!-- Rollover at midnight each day -->
+        <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+        <param name="Threshold" value="TRACE" />
+
+        <layout class="org.apache.log4j.PatternLayout">
+            <!-- The default pattern: Date Priority [Category] Message\n -->
+            <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+            <!--
+                The full pattern: Date MS Priority [Category]
+                (Thread:NDC) Message\n <param name="ConversionPattern"
+                value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+            -->
+        </layout>
+    </appender>
+
+    <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+        <appender-ref ref="BLOBSTOREFILE" />
+    </appender>
     <!-- ================ -->
     <!-- Limit categories -->
     <!-- ================ -->
+    <category name="jclouds.blobstore">
+        <priority value="TRACE" />
+        <appender-ref ref="ASYNCBLOBSTORE" />
+    </category>
 
     <category name="org.jclouds">
         <priority value="DEBUG" />
@@ -92,13 +121,10 @@
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCWIRE" />
     </category>
-
+   
-    <category name="jclouds.wire">
+        <!--<category name="jclouds.wire"> <priority value="DEBUG" />
-        <priority value="DEBUG" />
+        <appender-ref ref="ASYNCWIRE" /> </category>
-        <appender-ref ref="ASYNCWIRE" />
+    --><!-- ======================= -->
-    </category>
-
-    <!-- ======================= -->
     <!-- Setup the Root category -->
     <!-- ======================= -->
 

blobstore/src/main/java/org/jclouds/blobstore/AsyncBlobStore.java

@@ -20,8 +20,7 @@ package org.jclouds.blobstore;
 
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
-import org.jclouds.blobstore.domain.ListResponse;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.options.GetOptions;
 import org.jclouds.blobstore.options.ListContainerOptions;
@@ -43,7 +42,7 @@ public interface AsyncBlobStore {
    /**
     * @see BlobStore#list
     */
-   ListenableFuture<? extends ListResponse<? extends StorageMetadata>> list();
+   ListenableFuture<? extends PageSet<? extends StorageMetadata>> list();
 
    /**
     * @see BlobStore#containerExists
@@ -58,20 +57,24 @@ public interface AsyncBlobStore {
    /**
     * @see BlobStore#list(String)
     */
-   ListenableFuture<? extends ListContainerResponse<? extends StorageMetadata>> list(
+   ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(String container);
-            String container);
 
    /**
     * @see BlobStore#list(String, ListContainerOptions)
     */
-   ListenableFuture<? extends ListContainerResponse<? extends StorageMetadata>> list(
+   ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(String container,
-            String container, ListContainerOptions options);
+            ListContainerOptions options);
 
    /**
-    * @see BlobStore#clearContainer
+    * @see BlobStore#clearContainer(String)
     */
    ListenableFuture<Void> clearContainer(String container);
 
+   /**
+    * @see BlobStore#clearDirectory(String, ListContainerOptions)
+    */
+   ListenableFuture<Void> clearContainer(String container, ListContainerOptions options);
+
    /**
     * @see BlobStore#deleteContainer
     */
@@ -87,6 +90,11 @@ public interface AsyncBlobStore {
     */
    ListenableFuture<Void> createDirectory(String container, String directory);
 
+   /**
+    * @see BlobStore#deleteDirectory
+    */
+   ListenableFuture<Void> deleteDirectory(String containerName, String name);
+
    /**
     * @see BlobStore#blobExists
     */
@@ -117,4 +125,14 @@ public interface AsyncBlobStore {
     */
    ListenableFuture<Void> removeBlob(String container, String key);
 
+   /**
+    * @see BlobStore#countBlobs(String)
+    */
+   ListenableFuture<Long> countBlobs(String container);
+
+   /**
+    * @see BlobStore#countBlobs(String,ListContainerOptions)
+    */
+   ListenableFuture<Long> countBlobs(String container, ListContainerOptions options);
+
 }

blobstore/src/main/java/org/jclouds/blobstore/BlobMap.java

@@ -18,9 +18,10 @@
  */
 package org.jclouds.blobstore;
 
+import javax.annotation.Nullable;
+
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.internal.BlobMapImpl;
-import org.jclouds.blobstore.options.ListContainerOptions;
 
 import com.google.inject.ImplementedBy;
 
@@ -33,11 +34,11 @@ import com.google.inject.ImplementedBy;
  */
 @ImplementedBy(BlobMapImpl.class)
 public interface BlobMap extends ListableMap<String, Blob> {
-   
+
    Blob newBlob(String name);
-   
+
    public static interface Factory {
-      BlobMap create(String containerName, ListContainerOptions listOptions);
+      BlobMap create(String containerName, @Nullable String dir);
    }
-   
+
 }

blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java

@@ -20,8 +20,7 @@ package org.jclouds.blobstore;
 
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.domain.BlobMetadata;
-import org.jclouds.blobstore.domain.ListContainerResponse;
+import org.jclouds.blobstore.domain.PageSet;
-import org.jclouds.blobstore.domain.ListResponse;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.options.GetOptions;
 import org.jclouds.blobstore.options.ListContainerOptions;
@@ -38,7 +37,7 @@ public interface BlobStore {
    /**
     * Lists all root-level resources available to the account.
     */
-   ListResponse<? extends StorageMetadata> list();
+   PageSet<? extends StorageMetadata> list();
 
    boolean containerExists(String container);
 
@@ -51,10 +50,9 @@ public interface BlobStore {
     * @param parent
     *           - base path to list; non-recursive
     */
-   ListContainerResponse<? extends StorageMetadata> list(String container);
+   PageSet<? extends StorageMetadata> list(String container);
 
-   ListContainerResponse<? extends StorageMetadata> list(String container,
+   PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options);
-            ListContainerOptions options);
 
    /**
     * This will delete the contents of a container without removing it
@@ -62,6 +60,7 @@ public interface BlobStore {
     * @param container
     */
    void clearContainer(String container);
+   void clearContainer(String container, ListContainerOptions options);
 
    /**
     * This will delete a container recursively.
@@ -74,6 +73,8 @@ public interface BlobStore {
 
    void createDirectory(String container, String directory);
 
+   void deleteDirectory(String containerName, String name);
+
    boolean blobExists(String container, String name);
 
    /**
@@ -90,7 +91,7 @@ public interface BlobStore {
     *            if the container doesn't exist
     */
    String putBlob(String container, Blob blob);
-   
+
    /**
     * Retrieves the metadata of a {@code Blob} at location {@code container/name}
     * 
@@ -121,7 +122,6 @@ public interface BlobStore {
 
    Blob getBlob(String container, String name, GetOptions options);
 
-
    /**
     * Deletes a {@code Blob} representing the data at location {@code container/name}
     * 
@@ -134,4 +134,8 @@ public interface BlobStore {
     */
    void removeBlob(String container, String name);
 
+   long countBlobs(String container);
+
+   long countBlobs(String container, ListContainerOptions options);
+
 }

blobstore/src/main/java/org/jclouds/blobstore/ContainerNotFoundException.java

@@ -31,8 +31,8 @@ public class ContainerNotFoundException extends RuntimeException {
       super();
    }
 
-   public ContainerNotFoundException(String container) {
+   public ContainerNotFoundException(String container, String message) {
-      super(String.format("%s not found", container));
+      super(String.format("%s not found: %s", container, message));
       this.container = container;
    }
 

blobstore/src/main/java/org/jclouds/blobstore/InputStreamMap.java

@@ -22,8 +22,9 @@ import java.io.File;
 import java.io.InputStream;
 import java.util.Map;
 
+import javax.annotation.Nullable;
+
 import org.jclouds.blobstore.internal.InputStreamMapImpl;
-import org.jclouds.blobstore.options.ListContainerOptions;
 
 import com.google.inject.ImplementedBy;
 
@@ -42,7 +43,7 @@ import com.google.inject.ImplementedBy;
 @ImplementedBy(InputStreamMapImpl.class)
 public interface InputStreamMap extends ListableMap<String, InputStream> {
    public static interface Factory {
-      InputStreamMap create(String containerName, ListContainerOptions listOptions);
+      InputStreamMap create(String containerName, @Nullable String dir);
    }
 
    InputStream putString(String key, String value);

blobstore/src/main/java/org/jclouds/blobstore/KeyNotFoundException.java

@@ -34,8 +34,8 @@ public class KeyNotFoundException extends ResourceNotFoundException {
       super();
    }
 
-   public KeyNotFoundException(String container, String key) {
+   public KeyNotFoundException(String container, String key, String message) {
-      super(String.format("%s not found in container %s", key, container));
+      super(String.format("%s not found in container %s: %s", key, container, message));
       this.container = container;
       this.key = key;
    }

blobstore/src/main/java/org/jclouds/blobstore/ListableMap.java

@@ -19,7 +19,6 @@
 package org.jclouds.blobstore;
 
 import java.util.Map;
-import java.util.SortedSet;
 
 import org.jclouds.blobstore.domain.StorageMetadata;
 
@@ -35,6 +34,6 @@ public interface ListableMap<K, V> extends Map<K, V> {
     * 
     * @return blob listing that this map represents
     */
-   SortedSet<? extends StorageMetadata> list();
+   Iterable<? extends StorageMetadata> list();
 
 }

blobstore/src/main/java/org/jclouds/blobstore/binders/BindBlobToPayload.java

@@ -42,8 +42,7 @@ public class BindBlobToPayload implements Binder {
 
    public void bindToRequest(HttpRequest request, Object payload) {
       Blob object = (Blob) payload;
-
+      request.setPayload(checkNotNull(object.getPayload(), "object.getPayload()"));
-      request.setPayload(checkNotNull(object.getContent(), "object.getContent()"));
       request.getHeaders()
                .put(
                         HttpHeaders.CONTENT_TYPE,

blobstore/src/main/java/org/jclouds/blobstore/config/BlobStoreMapModule.java

@@ -18,6 +18,7 @@
  */
 package org.jclouds.blobstore.config;
 
+import javax.annotation.Nullable;
 import javax.inject.Inject;
 
 import org.jclouds.blobstore.BlobMap;
@@ -26,13 +27,10 @@ import org.jclouds.blobstore.InputStreamMap;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.internal.BlobMapImpl;
 import org.jclouds.blobstore.internal.InputStreamMapImpl;
-import org.jclouds.blobstore.options.ListContainerOptions;
-import org.jclouds.blobstore.strategy.ClearListStrategy;
 import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
-import org.jclouds.blobstore.strategy.CountListStrategy;
 import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
-import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
 import org.jclouds.blobstore.strategy.PutBlobsStrategy;
+import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Scopes;
@@ -60,20 +58,15 @@ public class BlobStoreMapModule extends AbstractModule {
       @Inject
       GetBlobsInListStrategy getAllBlobs;
       @Inject
-      ListBlobMetadataStrategy getAllBlobMetadata;
-      @Inject
       ContainsValueInListStrategy containsValueStrategy;
       @Inject
-      ClearListStrategy clearContainerStrategy;
-      @Inject
-      CountListStrategy containerCountStrategy;
-      @Inject
       PutBlobsStrategy putBlobsStrategy;
+      @Inject
+      ListBlobMetadataInContainer listStrategy;
 
-      public BlobMap create(String containerName, ListContainerOptions listOptions) {
+      public BlobMap create(String containerName, @Nullable String dir) {
-         return new BlobMapImpl(connection, getAllBlobs, getAllBlobMetadata, containsValueStrategy,
+         return new BlobMapImpl(connection, getAllBlobs, containsValueStrategy, putBlobsStrategy,
-                  clearContainerStrategy, containerCountStrategy, putBlobsStrategy, containerName,
+                  listStrategy, containerName, dir);
-                  listOptions);
       }
 
    }
@@ -86,20 +79,15 @@ public class BlobStoreMapModule extends AbstractModule {
       @Inject
       GetBlobsInListStrategy getAllBlobs;
       @Inject
-      ListBlobMetadataStrategy getAllBlobMetadata;
-      @Inject
       ContainsValueInListStrategy containsValueStrategy;
       @Inject
-      ClearListStrategy clearContainerStrategy;
-      @Inject
-      CountListStrategy containerCountStrategy;
-      @Inject
       PutBlobsStrategy putBlobsStrategy;
+      @Inject
+      ListBlobMetadataInContainer listStrategy;
 
-      public InputStreamMap create(String containerName, ListContainerOptions listOptions) {
+      public InputStreamMap create(String containerName, @Nullable String dir) {
-         return new InputStreamMapImpl(connection, blobFactory, getAllBlobs, getAllBlobMetadata,
+         return new InputStreamMapImpl(connection, blobFactory, getAllBlobs, listStrategy,
-                  containsValueStrategy, clearContainerStrategy, containerCountStrategy,
+                  containsValueStrategy, putBlobsStrategy, containerName, dir);
-                  putBlobsStrategy, containerName, listOptions);
       }
 
    }

blobstore/src/main/java/org/jclouds/blobstore/domain/PageSet.java

@@ -18,19 +18,21 @@
  */
 package org.jclouds.blobstore.domain;
 
-import java.util.SortedSet;
+import java.util.Set;
 
 /**
  * 
  * @author Adrian Cole
  * 
  */
-public interface ListResponse<T> extends SortedSet<T> {
+public interface PageSet<T> extends Set<T> {
 
-   String getMarker();
+   /**
-
+    * If there is a next marker, then the set is incomplete and you should issue another command to
-   int getMaxResults();
+    * retrieve the rest, setting the option {@code marker} to this value 
-
+    * 
-   boolean isTruncated();
+    * @return next marker, or null if list is complete
+    */
+   String getNextMarker();
 
 }

blobstore/src/main/java/org/jclouds/blobstore/domain/internal/BlobImpl.java

@@ -49,76 +49,67 @@ public class BlobImpl extends BasePayloadEnclosingImpl implements Blob, Comparab
       this.metadata = metadata;
    }
 
+   /**
+    * {@inheritDoc}
+    */
    @Override
    protected void setContentMD5(byte[] md5) {
       getMetadata().setContentMD5(checkNotNull(md5, "md5"));
    }
 
    /**
-    * @return System and User metadata relevant to this object.
+    * {@inheritDoc}
     */
+   @Override
    public MutableBlobMetadata getMetadata() {
       return metadata;
    }
 
    /**
-    * @return all http response headers associated with this Value
+    * {@inheritDoc}
     */
+   @Override
    public Multimap<String, String> getAllHeaders() {
       return allHeaders;
    }
 
+   /**
+    * {@inheritDoc}
+    */
+   @Override
    public void setAllHeaders(Multimap<String, String> allHeaders) {
       this.allHeaders = checkNotNull(allHeaders, "allHeaders");
    }
 
+   /**
+    * {@inheritDoc}
+    */
+   @Override
    public int compareTo(Blob o) {
       if (getMetadata().getName() == null)
          return -1;
       return (this == o) ? 0 : getMetadata().getName().compareTo(o.getMetadata().getName());
    }
 
+   /**
+    * {@inheritDoc}
+    */
    @Override
    public int hashCode() {
-      final int prime = 31;
+      return metadata.hashCode();
-      int result = 1;
+   }
-      result = prime * result + ((allHeaders == null) ? 0 : allHeaders.hashCode());
+
-      result = prime * result + ((contentLength == null) ? 0 : contentLength.hashCode());
+   /**
-      result = prime * result + ((payload == null) ? 0 : payload.hashCode());
+    * {@inheritDoc}
-      result = prime * result + ((metadata == null) ? 0 : metadata.hashCode());
+    */
-      return result;
+   @Override
+   public boolean equals(Object obj) {
+      return metadata.equals(obj);
    }
 
    @Override
-   public boolean equals(Object obj) {
+   public String toString() {
-      if (this == obj)
+      return "[metadata=" + metadata + "]";
-         return true;
-      if (obj == null)
-         return false;
-      if (getClass() != obj.getClass())
-         return false;
-      BlobImpl other = (BlobImpl) obj;
-      if (allHeaders == null) {
-         if (other.allHeaders != null)
-            return false;
-      } else if (!allHeaders.equals(other.allHeaders))
-         return false;
-      if (contentLength == null) {
-         if (other.contentLength != null)
-            return false;
-      } else if (!contentLength.equals(other.contentLength))
-         return false;
-      if (payload == null) {
-         if (other.payload != null)
-            return false;
-      } else if (!payload.equals(other.payload))
-         return false;
-      if (metadata == null) {
-         if (other.metadata != null)
-            return false;
-      } else if (!metadata.equals(other.metadata))
-         return false;
-      return true;
    }
 
 }

blobstore/src/main/java/org/jclouds/blobstore/domain/internal/MutableBlobMetadataImpl.java

@@ -18,8 +18,6 @@
  */
 package org.jclouds.blobstore.domain.internal;
 
-import java.util.Arrays;
-
 import javax.ws.rs.core.MediaType;
 
 import org.jclouds.blobstore.domain.Blob;
@@ -52,10 +50,18 @@ public class MutableBlobMetadataImpl extends MutableStorageMetadataImpl implemen
       this.contentMD5 = from.getContentMD5();
    }
 
+   /**
+    * {@inheritDoc}
+    */
+   @Override
    public String getContentType() {
       return contentType;
    }
 
+   /**
+    * {@inheritDoc}
+    */
+   @Override
    public byte[] getContentMD5() {
       if (contentMD5 != null) {
          byte[] retval = new byte[contentMD5.length];
@@ -66,12 +72,10 @@ public class MutableBlobMetadataImpl extends MutableStorageMetadataImpl implemen
       }
    }
 
-   public int compareTo(BlobMetadata o) {
+   /**
-      if (getName() == null)
+    * {@inheritDoc}
-         return -1;
+    */
-      return (this == o) ? 0 : getName().compareTo(o.getName());
+   @Override
-   }
-
    public void setContentMD5(byte[] md5) {
       if (md5 != null) {
          byte[] retval = new byte[md5.length];
@@ -80,35 +84,12 @@ public class MutableBlobMetadataImpl extends MutableStorageMetadataImpl implemen
       }
    }
 
+   /**
+    * {@inheritDoc}
+    */
+   @Override
    public void setContentType(String type) {
       this.contentType = type;
    }
 
-   @Override
-   public int hashCode() {
-      final int prime = 31;
-      int result = super.hashCode();
-      result = prime * result + Arrays.hashCode(contentMD5);
-      result = prime * result + ((contentType == null) ? 0 : contentType.hashCode());
-      return result;
-   }
-
-   @Override
-   public boolean equals(Object obj) {
-      if (this == obj)
-         return true;
-      if (!super.equals(obj))
-         return false;
-      if (getClass() != obj.getClass())
-         return false;
-      MutableBlobMetadataImpl other = (MutableBlobMetadataImpl) obj;
-      if (!Arrays.equals(contentMD5, other.contentMD5))
-         return false;
-      if (contentType == null) {
-         if (other.contentType != null)
-            return false;
-      } else if (!contentType.equals(other.contentType))
-         return false;
-      return true;
-   }
 }

blobstore/src/main/java/org/jclouds/blobstore/domain/internal/MutableStorageMetadataImpl.java

@@ -100,41 +100,4 @@ public class MutableStorageMetadataImpl extends MutableResourceMetadataImpl<Stor
       this.eTag = eTag;
    }
 
-   @Override
-   public int hashCode() {
-      final int prime = 31;
-      int result = super.hashCode();
-      result = prime * result + ((eTag == null) ? 0 : eTag.hashCode());
-      result = prime * result + ((lastModified == null) ? 0 : lastModified.hashCode());
-      result = prime * result + ((size == null) ? 0 : size.hashCode());
-      return result;
-   }
-
-   @Override
-   public boolean equals(Object obj) {
-      if (this == obj)
-         return true;
-      if (!super.equals(obj))
-         return false;
-      if (getClass() != obj.getClass())
-         return false;
-      MutableStorageMetadataImpl other = (MutableStorageMetadataImpl) obj;
-      if (eTag == null) {
-         if (other.eTag != null)
-            return false;
-      } else if (!eTag.equals(other.eTag))
-         return false;
-      if (lastModified == null) {
-         if (other.lastModified != null)
-            return false;
-      } else if (!lastModified.equals(other.lastModified))
-         return false;
-      if (size == null) {
-         if (other.size != null)
-            return false;
-      } else if (!size.equals(other.size))
-         return false;
-      return true;
-   }
-
 }