Issue 202: withDetails support for BlobStore.list(String,ListContainerOptions)

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java

@@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
 import javax.inject.Named;
+import javax.inject.Provider;
 import javax.inject.Singleton;
 
 import org.jclouds.Constants;
@@ -38,6 +39,8 @@ import org.jclouds.atmosonline.saas.blobstore.functions.DirectoryEntryListToReso
 import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlob;
 import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlobMetadata;
 import org.jclouds.atmosonline.saas.domain.AtmosObject;
+import org.jclouds.atmosonline.saas.domain.BoundedSet;
+import org.jclouds.atmosonline.saas.domain.DirectoryEntry;
 import org.jclouds.atmosonline.saas.options.ListOptions;
 import org.jclouds.atmosonline.saas.util.AtmosStorageUtils;
 import org.jclouds.blobstore.BlobStoreContext;
@@ -47,6 +50,7 @@ import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
 import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
+import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
 import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.concurrent.ConcurrentUtils;
 import org.jclouds.encryption.EncryptionService;
@@ -69,6 +73,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
    private final DirectoryEntryListToResourceMetadataList container2ResourceList;
    private final EncryptionService encryptionService;
    private final BlobToHttpGetOptions blob2ObjectGetOptions;
+   private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
 
    @Inject
    AtmosAsyncBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils,
@@ -77,7 +82,8 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
             ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object,
             BlobStoreListOptionsToListOptions container2ContainerListOptions,
             DirectoryEntryListToResourceMetadataList container2ResourceList,
-            EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions) {
+            EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions,
+            Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
       super(context, blobUtils, service);
       this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
       this.sync = checkNotNull(sync, "sync");
@@ -89,6 +95,8 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
       this.blob2Object = checkNotNull(blob2Object, "blob2Object");
       this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
       this.encryptionService = checkNotNull(encryptionService, "encryptionService");
+      this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
+               "fetchBlobMetadataProvider");
    }
 
    /**
@@ -208,7 +216,12 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
             org.jclouds.blobstore.options.ListContainerOptions options) {
       container = AtmosStorageUtils.adjustContainerIfDirOptionPresent(container, options);
       ListOptions nativeOptions = container2ContainerListOptions.apply(options);
-      return compose(async.listDirectory(container, nativeOptions), container2ResourceList, service);
+      ListenableFuture<? extends BoundedSet<? extends DirectoryEntry>> returnVal = async
+               .listDirectory(container, nativeOptions);
+      ListenableFuture<PageSet<? extends StorageMetadata>> list = compose(returnVal,
+               container2ResourceList, service);
+      return options.isDetailed() ? compose(list, fetchBlobMetadataProvider.get().setContainerName(
+               container), service) : list;
    }
 
    /**

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java

@@ -21,6 +21,7 @@ package org.jclouds.atmosonline.saas.blobstore;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import javax.inject.Inject;
+import javax.inject.Provider;
 import javax.inject.Singleton;
 
 import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient;
@@ -39,6 +40,7 @@ import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
 import org.jclouds.blobstore.internal.BaseBlobStore;
+import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
 import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.encryption.EncryptionService;
 import org.jclouds.http.options.GetOptions;
@@ -56,13 +58,15 @@ public class AtmosBlobStore extends BaseBlobStore {
    private final DirectoryEntryListToResourceMetadataList container2ResourceList;
    private final EncryptionService encryptionService;
    private final BlobToHttpGetOptions blob2ObjectGetOptions;
+   private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
 
    @Inject
    AtmosBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils, AtmosStorageClient sync,
             ObjectToBlob object2Blob, ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object,
             BlobStoreListOptionsToListOptions container2ContainerListOptions,
             DirectoryEntryListToResourceMetadataList container2ResourceList,
-            EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions) {
+            EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions,
+            Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
       super(context, blobUtils);
       this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
       this.sync = checkNotNull(sync, "sync");
@@ -73,6 +77,8 @@ public class AtmosBlobStore extends BaseBlobStore {
       this.blob2Object = checkNotNull(blob2Object, "blob2Object");
       this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
       this.encryptionService = checkNotNull(encryptionService, "encryptionService");
+      this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
+               "fetchBlobMetadataProvider");
    }
 
    /**
@@ -180,7 +186,11 @@ public class AtmosBlobStore extends BaseBlobStore {
             org.jclouds.blobstore.options.ListContainerOptions options) {
       container = AtmosStorageUtils.adjustContainerIfDirOptionPresent(container, options);
       ListOptions nativeOptions = container2ContainerListOptions.apply(options);
-      return container2ResourceList.apply(sync.listDirectory(container, nativeOptions));
+      // until includeMeta() option works for namespace interface
+      PageSet<? extends StorageMetadata> list = container2ResourceList.apply(sync.listDirectory(
+               container, nativeOptions));
+      return options.isDetailed() ? fetchBlobMetadataProvider.get().setContainerName(container)
+               .apply(list) : list;
    }
 
    /**

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobMetadataToObject.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.atmosonline.saas.blobstore.functions;
 
+import java.util.Map.Entry;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -48,8 +50,10 @@ public class BlobMetadataToObject implements Function<BlobMetadata, AtmosObject>
       if (from == null)
          return null;
       UserMetadata userMd = new UserMetadata();
-      if (from.getUserMetadata() != null)
+      if (from.getUserMetadata() != null) {
-         userMd.getMetadata().putAll(from.getUserMetadata());
+         for (Entry<String, String> entry : from.getUserMetadata().entrySet())
+            userMd.getMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
+      }
       return factory.create(blob2ContentMd.apply(from), blob2SysMd.apply(from), userMd);
    }
 

atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobStoreListOptionsToListOptions.java

@@ -42,6 +42,9 @@ public class BlobStoreListOptionsToListOptions implements
       if (from.getMaxResults() != null) {
          httpOptions.limit(from.getMaxResults());
       }
+      if (from.isDetailed()) {
+         httpOptions.includeMeta();
+      }
       return httpOptions;
    }
 }

atmos/src/main/java/org/jclouds/atmosonline/saas/options/ListOptions.java

@@ -56,6 +56,20 @@ public class ListOptions extends BaseHttpRequestOptions {
       return this;
    }
 
+   /**
+    * the maximum number of items that should be returned. If this is not specified, there is no
+    * limit.
+    */
+   public ListOptions includeMeta() {
+      headers.put("x-emc-include-meta", Integer.toString(1));
+      return this;
+   }
+
+   public boolean metaIncluded() {
+      String meta = getFirstHeaderOrNull("x-emc-include-meta");
+      return (meta != null) ? meta.equals("1") : false;
+   }
+
    public Integer getLimit() {
       String maxresults = getFirstHeaderOrNull("x-emc-limit");
       return (maxresults != null) ? new Integer(maxresults) : null;
@@ -71,6 +85,14 @@ public class ListOptions extends BaseHttpRequestOptions {
          return options.token(token);
       }
 
+      /**
+       * @see ListOptions#includeMeta()
+       */
+      public static ListOptions includeMeta() {
+         ListOptions options = new ListOptions();
+         return options.includeMeta();
+      }
+
       /**
        * @see ListOptions#limit(int)
        */

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/functions/ListOptionsToBlobStoreListOptions.java

@@ -39,6 +39,9 @@ public class ListOptionsToBlobStoreListOptions implements
          if (optionsList[0].getLimit() != null) {
             options.maxResults(optionsList[0].getLimit());
          }
+         if (optionsList[0].metaIncluded()) {
+            options.withDetails();
+         }
       }
       return options;
    }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageContainerIntegrationTest.java

@@ -35,4 +35,8 @@ public class AtmosStorageContainerIntegrationTest extends BaseContainerIntegrati
       // Not currently working
    }
 
+   @Override
+   public void testListContainerMarker() throws InterruptedException, UnsupportedEncodingException {
+      // Not currently working https://community.emc.com/thread/100545
+   }
 }

atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageTestInitializer.java

@@ -40,8 +40,8 @@ public class AtmosStorageTestInitializer extends BaseTestInitializer {
    @Override
    protected BlobStoreContext createLiveContext(Module configurationModule, String url, String app,
             String account, String key) throws IOException {
-      return new BlobStoreContextFactory().createContext("atmos", account, key, ImmutableSet.of(
+      return new BlobStoreContextFactory().createContext("atmosonline", account, key, ImmutableSet
-               configurationModule, new Log4JLoggingModule()), new Properties());
+               .of(configurationModule, new Log4JLoggingModule()), new Properties());
    }
 
    @Override

atmos/src/test/java/org/jclouds/atmosonline/saas/options/ListOptionsTest.java

@@ -37,20 +37,37 @@ public class ListOptionsTest {
       assertEquals(ImmutableList.of("a"), options.buildRequestHeaders().get("x-emc-token"));
    }
 
+   public void testTokenStatic() {
+      ListOptions options = ListOptions.Builder.token("a");
+      assertEquals(ImmutableList.of("a"), options.buildRequestHeaders().get("x-emc-token"));
+   }
+
    public void testLimit() {
       int limit = 1;
       ListOptions options = new ListOptions().limit(limit);
       assertEquals(ImmutableList.of("1"), options.buildRequestHeaders().get("x-emc-limit"));
    }
 
-   public void testTokenStatic() {
-      ListOptions options = ListOptions.Builder.token("a");
-      assertEquals(ImmutableList.of("a"), options.buildRequestHeaders().get("x-emc-token"));
-   }
-
    public void testLimitStatic() {
       int limit = 1;
       ListOptions options = ListOptions.Builder.limit(limit);
       assertEquals(ImmutableList.of("1"), options.buildRequestHeaders().get("x-emc-limit"));
    }
+
+   public void testNoMeta() {
+      ListOptions options = new ListOptions();
+      assert !options.metaIncluded();
+   }
+
+   public void testMeta() {
+      ListOptions options = new ListOptions().includeMeta();
+      assertEquals(ImmutableList.of("1"), options.buildRequestHeaders().get("x-emc-include-meta"));
+      assert options.metaIncluded();
+   }
+
+   public void testMetaStatic() {
+      ListOptions options = ListOptions.Builder.includeMeta();
+      assertEquals(ImmutableList.of("1"), options.buildRequestHeaders().get("x-emc-include-meta"));
+      assert options.metaIncluded();
+   }
 }

aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java

@@ -321,19 +321,23 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule {
                   holder.logger.debug("<< didn't match os(%s)", matcher.group(1));
                }
             }
-            images
+            try {
-                     .add(new ImageImpl(
+               images
-                              from.getId(),
+                        .add(new ImageImpl(
-                              name,
+                                 from.getId(),
-                              region.toString(),
+                                 name,
-                              null,
+                                 region.toString(),
-                              ImmutableMap.<String, String> of("owner", from.getImageOwnerId()),
+                                 null,
-                              from.getDescription(),
+                                 ImmutableMap.<String, String> of("owner", from.getImageOwnerId()),
-                              version,
+                                 from.getDescription(),
-                              os,
+                                 version,
-                              osDescription,
+                                 os,
-                              from.getArchitecture() == org.jclouds.aws.ec2.domain.Image.Architecture.I386 ? Architecture.X86_32
+                                 osDescription,
-                                       : Architecture.X86_64));
+                                 from.getArchitecture() == org.jclouds.aws.ec2.domain.Image.Architecture.I386 ? Architecture.X86_32
+                                          : Architecture.X86_64));
+            } catch (NullPointerException e) {
+               holder.logger.debug("<< image (%s) missing (%s)", from.getId(), e.getMessage());
+            }
          }
       }
       holder.logger.debug("<< images(%d)", images.size());

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3AsyncBlobStore.java

@@ -26,6 +26,7 @@ import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
 import javax.inject.Named;
+import javax.inject.Provider;
 import javax.inject.Singleton;
 
 import org.jclouds.Constants;
@@ -52,6 +53,7 @@ import org.jclouds.blobstore.domain.internal.PageSetImpl;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
 import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
 import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
 import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.http.options.GetOptions;
 
@@ -75,6 +77,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
    private final ObjectToBlob object2Blob;
    private final BlobToObject blob2Object;
    private final ObjectToBlobMetadata object2BlobMd;
+   private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
 
    @Inject
    S3AsyncBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils,
@@ -83,7 +86,8 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
             ContainerToBucketListOptions container2BucketListOptions,
             BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
             BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
-            ObjectToBlobMetadata object2BlobMd) {
+            ObjectToBlobMetadata object2BlobMd,
+            Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
       super(context, blobUtils, service);
       this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
       this.async = checkNotNull(async, "async");
@@ -95,6 +99,8 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
       this.object2Blob = checkNotNull(object2Blob, "object2Blob");
       this.blob2Object = checkNotNull(blob2Object, "blob2Object");
       this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
+      this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
+               "fetchBlobMetadataProvider");
    }
 
    /**
@@ -148,7 +154,10 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
             ListContainerOptions options) {
       ListBucketOptions httpOptions = container2BucketListOptions.apply(options);
       ListenableFuture<ListBucketResponse> returnVal = async.listBucket(container, httpOptions);
-      return compose(returnVal, bucket2ResourceList, service);
+      ListenableFuture<PageSet<? extends StorageMetadata>> list = compose(returnVal,
+               bucket2ResourceList, service);
+      return options.isDetailed() ? compose(list, fetchBlobMetadataProvider.get().setContainerName(
+               container), service) : list;
    }
 
    /**

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java

@@ -23,6 +23,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import java.util.SortedSet;
 
 import javax.inject.Inject;
+import javax.inject.Provider;
 import javax.inject.Singleton;
 
 import org.jclouds.aws.domain.Region;
@@ -45,6 +46,7 @@ import org.jclouds.blobstore.domain.internal.PageSetImpl;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
 import org.jclouds.blobstore.internal.BaseBlobStore;
 import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
 import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.http.options.GetOptions;
 import org.jclouds.util.Utils;
@@ -67,6 +69,7 @@ public class S3BlobStore extends BaseBlobStore {
    private final BlobToObject blob2Object;
    private final ObjectToBlobMetadata object2BlobMd;
    private final BlobToHttpGetOptions blob2ObjectGetOptions;
+   private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
 
    @Inject
    S3BlobStore(BlobStoreContext context, BlobStoreUtils blobUtils, S3Client sync,
@@ -74,7 +77,8 @@ public class S3BlobStore extends BaseBlobStore {
             ContainerToBucketListOptions container2BucketListOptions,
             BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
             BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
-            ObjectToBlobMetadata object2BlobMd) {
+            ObjectToBlobMetadata object2BlobMd,
+            Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
       super(context, blobUtils);
       this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
       this.sync = checkNotNull(sync, "sync");
@@ -85,6 +89,8 @@ public class S3BlobStore extends BaseBlobStore {
       this.object2Blob = checkNotNull(object2Blob, "object2Blob");
       this.blob2Object = checkNotNull(blob2Object, "blob2Object");
       this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
+      this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
+               "fetchBlobMetadataProvider");
    }
 
    /**
@@ -132,9 +138,12 @@ public class S3BlobStore extends BaseBlobStore {
     *           bucket name
     */
    @Override
-   public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions optionsList) {
+   public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
-      ListBucketOptions httpOptions = container2BucketListOptions.apply(optionsList);
+      ListBucketOptions httpOptions = container2BucketListOptions.apply(options);
-      return bucket2ResourceList.apply(sync.listBucket(container, httpOptions));
+      PageSet<? extends StorageMetadata> list = bucket2ResourceList.apply(sync.listBucket(
+               container, httpOptions));
+      return options.isDetailed() ? fetchBlobMetadataProvider.get().setContainerName(container)
+               .apply(list) : list;
    }
 
    /**
@@ -191,7 +200,6 @@ public class S3BlobStore extends BaseBlobStore {
    @Override
    public BlobMetadata blobMetadata(String container, String key) {
       return object2BlobMd.apply(sync.headObject(container, key));
-
    }
 
    /**

aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/BlobToObjectMetadata.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.aws.s3.blobstore.functions;
 
+import java.util.Map.Entry;
+
 import javax.inject.Singleton;
 
 import org.jclouds.aws.s3.domain.MutableObjectMetadata;
@@ -42,8 +44,10 @@ public class BlobToObjectMetadata implements Function<BlobMetadata, MutableObjec
       to.setLastModified(from.getLastModified());
       if (from.getSize() != null)
          to.setSize(from.getSize());
-      if (from.getUserMetadata() != null)
+      if (from.getUserMetadata() != null) {
-         to.setUserMetadata(from.getUserMetadata());
+         for (Entry<String, String> entry : from.getUserMetadata().entrySet())
+            to.getUserMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
+      }
       return to;
    }
 

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/BlobMetadataToBlobProperties.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.azure.storage.blob.blobstore.functions;
 
+import java.util.Map.Entry;
+
 import javax.inject.Singleton;
 
 import org.jclouds.azure.storage.blob.domain.MutableBlobProperties;
@@ -42,8 +44,10 @@ public class BlobMetadataToBlobProperties implements Function<BlobMetadata, Muta
       to.setLastModified(from.getLastModified());
       if (from.getSize() != null)
          to.setContentLength(from.getSize());
-      if (from.getUserMetadata() != null)
+      if (from.getUserMetadata() != null) {
-         to.setMetadata(from.getUserMetadata());
+         for (Entry<String, String> entry : from.getUserMetadata().entrySet())
+            to.getMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
+      }
       return to;
    }
 

azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/ListOptionsToListBlobsOptions.java

@@ -48,6 +48,9 @@ public class ListOptionsToListBlobsOptions implements
       if (from.getMaxResults() != null) {
          httpOptions.maxResults(from.getMaxResults());
       }
+      if (from.isDetailed()) {
+         httpOptions.includeMetadata();
+      }
       return httpOptions;
    }
 }

azure/src/main/java/org/jclouds/azure/storage/options/ListOptions.java

@@ -47,6 +47,10 @@ public class ListOptions extends BaseHttpRequestOptions {
       return this;
    }
 
+   public boolean getIncludeMetadata() {
+      return getFirstQueryOrNull("include").equals("metadata");
+   }
+
    /**
     * Filters the results to return only objects whose name begins with the specified prefix.
     */

azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/functions/ListBlobsOptionsToListOptions.java

@@ -29,7 +29,8 @@ import com.google.common.base.Function;
  * @author Adrian Cole
  */
 @Singleton
-public class ListBlobsOptionsToListOptions implements Function<ListBlobsOptions[], ListContainerOptions> {
+public class ListBlobsOptionsToListOptions implements
+         Function<ListBlobsOptions[], ListContainerOptions> {
    public ListContainerOptions apply(ListBlobsOptions[] optionsList) {
       ListContainerOptions options = new ListContainerOptions();
       if (optionsList.length != 0) {
@@ -47,6 +48,9 @@ public class ListBlobsOptionsToListOptions implements Function<ListBlobsOptions[
          if (optionsList[0].getPrefix() != null) {
             options.inDirectory(optionsList[0].getPrefix());
          }
+         if (optionsList[0].getIncludeMetadata()) {
+            options.withDetails();
+         }
       }
       return options;
    }

azure/src/test/resources/log4j.xml

@@ -122,9 +122,11 @@
         <appender-ref ref="ASYNCWIRE" />
     </category>
 
-        <!--<category name="jclouds.wire"> <priority value="DEBUG" />
+    <category name="jclouds.wire">
-        <appender-ref ref="ASYNCWIRE" /> </category>
+        <priority value="DEBUG" />
-    --><!-- ======================= -->
+        <appender-ref ref="ASYNCWIRE" />
+    </category>
+    <!-- ======================= -->
     <!-- Setup the Root category -->
     <!-- ======================= -->
 

blobstore/src/main/java/org/jclouds/blobstore/TransientAsyncBlobStore.java

@@ -144,20 +144,20 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
     * default maxResults is 1000
     */
    @Override
-   public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(final String name,
+   public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(
-            ListContainerOptions options) {
+            final String container, ListContainerOptions options) {
-      final Map<String, Blob> realContents = getContainerToBlobs().get(name);
+      final Map<String, Blob> realContents = getContainerToBlobs().get(container);
 
       if (realContents == null)
-         return immediateFailedFuture(cnfe(name));
+         return immediateFailedFuture(cnfe(container));
 
       SortedSet<StorageMetadata> contents = Sets.newTreeSet(Iterables.transform(realContents
                .keySet(), new Function<String, StorageMetadata>() {
          public StorageMetadata apply(String key) {
             Blob oldBlob = realContents.get(key);
             checkState(oldBlob != null, "blob " + key
-                     + " is not present although it was in the list of " + name);
+                     + " is not present although it was in the list of " + container);
-            checkState(oldBlob.getMetadata() != null, "blob " + name + "/" + key
+            checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key
                      + " has no metadata");
             MutableBlobMetadata md = copy(oldBlob.getMetadata());
             String directoryName = ifDirectoryReturnName.execute(md);
@@ -224,6 +224,14 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
                      }
                   }));
       }
+
+      // trim metadata, if the response isn't supposed to be detailed.
+      if (!options.isDetailed()) {
+         for (StorageMetadata md : contents) {
+            md.getUserMetadata().clear();
+         }
+      }
+
       return immediateFuture(new PageSetImpl<StorageMetadata>(contents, marker));
 
    }

blobstore/src/main/java/org/jclouds/blobstore/domain/internal/PageSetImpl.java

@@ -32,7 +32,7 @@ public class PageSetImpl<T> extends HashSet<T> implements PageSet<T> {
    private static final long serialVersionUID = -7133632087734650835L;
    protected final String marker;
 
-   public PageSetImpl(Iterable<T> contents, @Nullable String nextMarker) {
+   public PageSetImpl(Iterable<? extends T> contents, @Nullable String nextMarker) {
       Iterables.addAll(this, contents);
       this.marker = nextMarker;
    }

blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java

@@ -37,13 +37,23 @@ import static com.google.common.base.Preconditions.checkNotNull;
  * @author Adrian Cole
  */
 public class ListContainerOptions extends ListOptions implements Cloneable {
+
+   public static final ImmutableListContainerOptions NONE = new ImmutableListContainerOptions(
+            new ListContainerOptions());
+
+   private String dir;
+   private boolean recursive;
+   private boolean detailed;
+
    public ListContainerOptions() {
    }
 
-   ListContainerOptions(Integer maxKeys, String marker, String dir, boolean recursive) {
+   ListContainerOptions(Integer maxKeys, String marker, String dir, boolean recursive,
+            boolean detailed) {
       super(maxKeys, marker);
       this.dir = dir;
       this.recursive = recursive;
+      this.detailed = detailed;
    }
 
    public static class ImmutableListContainerOptions extends ListContainerOptions {
@@ -68,6 +78,11 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
          throw new UnsupportedOperationException();
       }
 
+      @Override
+      public boolean isDetailed() {
+         return delegate.isDetailed();
+      }
+
       @Override
       public boolean isRecursive() {
          return delegate.isRecursive();
@@ -106,13 +121,6 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
 
    }
 
-   public static final ImmutableListContainerOptions NONE = new ImmutableListContainerOptions(
-            new ListContainerOptions());
-
-   private String dir;
-
-   private boolean recursive;
-
    public String getDir() {
       return dir;
    }
@@ -121,6 +129,10 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
       return recursive;
    }
 
+   public boolean isDetailed() {
+      return detailed;
+   }
+
    /**
     * This will list the contents of a virtual or real directory path.
     * 
@@ -154,6 +166,15 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
       return this;
    }
 
+   /**
+    * populate each result with detailed such as metadata even if it incurs extra requests to the
+    * service.
+    */
+   public ListContainerOptions withDetails() {
+      this.detailed = true;
+      return this;
+   }
+
    public static class Builder {
 
       /**
@@ -188,15 +209,23 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
          return options.recursive();
       }
 
+      /**
+       * @see ListContainerOptions#withDetails()
+       */
+      public static ListContainerOptions withDetails() {
+         ListContainerOptions options = new ListContainerOptions();
+         return options.withDetails();
+      }
    }
 
    @Override
    public ListContainerOptions clone() {
-      return new ListContainerOptions(getMaxResults(), getMarker(), dir, recursive);
+      return new ListContainerOptions(getMaxResults(), getMarker(), dir, recursive, detailed);
    }
 
    @Override
    public String toString() {
-      return "[dir=" + dir + ", recursive=" + recursive + ", maxResults=" + getMaxResults() + "]";
+      return "[dir=" + dir + ", recursive=" + recursive + ", detailed=" + detailed
+               + ", maxResults=" + getMaxResults() + "]";
    }
 }

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java

@@ -103,7 +103,7 @@ public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrateg
                   }
                }
             }, Executors.sameThreadExecutor());
-            responses.put(md, ablobstore.getBlob(container, md.getName()));
+            responses.put(md, future);
          }
          exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, String.format(
                   "getting from containerName: %s", container));

blobstore/src/test/java/org/jclouds/blobstore/integration/TransientContainerIntegrationTest.java

@@ -18,9 +18,20 @@
  */
 package org.jclouds.blobstore.integration;
 
+import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults;
+import static org.testng.Assert.assertEquals;
+
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobMetadata;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.integration.internal.BaseContainerIntegrationTest;
 import org.testng.annotations.Test;
 
+import com.google.common.collect.Iterables;
+
 /**
  * @author James Murty
  * @author Adrian Cole
@@ -28,4 +39,33 @@ import org.testng.annotations.Test;
 @Test(groups = { "integration", "live" }, testName = "blobstore.TransientContainerIntegrationTest")
 public class TransientContainerIntegrationTest extends BaseContainerIntegrationTest {
 
+   @Test(groups = { "integration", "live" })
+   public void testNotWithDetails() throws InterruptedException {
+
+      String key = "hello";
+
+      Blob object = context.getBlobStore().newBlob(key);
+      object.setPayload(TEST_STRING);
+      object.getMetadata().setContentType(MediaType.TEXT_PLAIN);
+      object.getMetadata().setSize(new Long(TEST_STRING.length()));
+      // NOTE all metadata in jclouds comes out as lowercase, in an effort to normalize the
+      // providers.
+      object.getMetadata().getUserMetadata().put("Adrian", "powderpuff");
+      String containerName = getContainerName();
+      try {
+         addBlobToContainer(containerName, object);
+         validateContent(containerName, key);
+
+         PageSet<? extends StorageMetadata> container = context.getBlobStore().list(containerName,
+                  maxResults(1));
+
+         BlobMetadata metadata = (BlobMetadata) Iterables.getOnlyElement(container);
+         // transient container should be lenient and not return metadata on undetailed listing.
+
+         assertEquals(metadata.getUserMetadata().size(), 0);
+
+      } finally {
+         returnContainer(containerName);
+      }
+   }
 }

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java

@@ -28,12 +28,17 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
 
+import javax.ws.rs.core.MediaType;
+
 import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobMetadata;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
+import org.jclouds.encryption.internal.JCEEncryptionService;
 import org.testng.annotations.Test;
 
 import com.google.common.base.Throwables;
+import com.google.common.collect.Iterables;
 
 /**
  * @author Adrian Cole
@@ -58,6 +63,38 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
       }
    }
 
+   @Test(groups = { "integration", "live" })
+   public void testWithDetails() throws InterruptedException {
+      String key = "hello";
+
+      Blob object = context.getBlobStore().newBlob(key);
+      object.setPayload(TEST_STRING);
+      object.getMetadata().setContentType(MediaType.TEXT_PLAIN);
+      object.getMetadata().setSize(new Long(TEST_STRING.length()));
+      // NOTE all metadata in jclouds comes out as lowercase, in an effort to normalize the
+      // providers.
+      object.getMetadata().getUserMetadata().put("Adrian", "powderpuff");
+      object.getMetadata().setContentMD5(new JCEEncryptionService().md5(TEST_STRING.getBytes()));
+      String containerName = getContainerName();
+      try {
+         addBlobToContainer(containerName, object);
+         validateContent(containerName, key);
+
+         PageSet<? extends StorageMetadata> container = context.getBlobStore().list(containerName,
+                  maxResults(1).withDetails());
+
+         BlobMetadata metadata = (BlobMetadata) Iterables.getOnlyElement(container);
+         
+         assert metadata.getContentType().startsWith("text/plain") : metadata.getContentType();
+         assertEquals(metadata.getSize(), new Long(TEST_STRING.length()));
+         assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff");
+         assertEquals(metadata.getContentMD5(), new JCEEncryptionService().md5(TEST_STRING
+                  .getBytes()));
+      } finally {
+         returnContainer(containerName);
+      }
+   }
+
    @Test(groups = { "integration", "live" })
    public void testClearWhenContentsUnderPath() throws InterruptedException {
       String containerName = getContainerName();
@@ -151,7 +188,7 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
 
          context.getBlobStore().clearContainer(containerName, inDirectory(directory));
          assert context.getBlobStore().directoryExists(containerName, directory);
-         assert context.getBlobStore().directoryExists(containerName,  directory + "/" + directory);
+         assert context.getBlobStore().directoryExists(containerName, directory + "/" + directory);
 
          // should have only the 2 level-deep directory above
          container = context.getBlobStore().list(containerName, inDirectory(directory));

rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java

@@ -18,6 +18,7 @@
  */
 package org.jclouds.rackspace.cloudfiles.blobstore;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.util.concurrent.Futures.compose;
 
 import java.util.Set;
@@ -25,6 +26,7 @@ import java.util.concurrent.ExecutorService;
 
 import javax.inject.Inject;
 import javax.inject.Named;
+import javax.inject.Provider;
 import javax.inject.Singleton;
 
 import org.jclouds.Constants;
@@ -37,6 +39,7 @@ import org.jclouds.blobstore.domain.internal.PageSetImpl;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
 import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
 import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
 import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.http.options.GetOptions;
 import org.jclouds.rackspace.cloudfiles.CloudFilesAsyncClient;
@@ -71,6 +74,7 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore {
    private final BlobToObject blob2Object;
    private final ObjectToBlobMetadata object2BlobMd;
    private final BlobToHttpGetOptions blob2ObjectGetOptions;
+   private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
 
    @Inject
    CloudFilesAsyncBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils,
@@ -79,7 +83,8 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore {
             BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
             ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob,
             BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd,
-            BlobToHttpGetOptions blob2ObjectGetOptions) {
+            BlobToHttpGetOptions blob2ObjectGetOptions,
+            Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
       super(context, blobUtils, service);
       this.sync = sync;
       this.async = async;
@@ -90,6 +95,8 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore {
       this.blob2Object = blob2Object;
       this.object2BlobMd = object2BlobMd;
       this.blob2ObjectGetOptions = blob2ObjectGetOptions;
+      this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
+               "fetchBlobMetadataProvider");
    }
 
    /**
@@ -139,7 +146,10 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore {
       org.jclouds.rackspace.cloudfiles.options.ListContainerOptions httpOptions = container2ContainerListOptions
                .apply(options);
       ListenableFuture<PageSet<ObjectInfo>> returnVal = async.listObjects(container, httpOptions);
-      return compose(returnVal, container2ResourceList, service);
+      ListenableFuture<PageSet<? extends StorageMetadata>> list = compose(returnVal,
+               container2ResourceList, service);
+      return options.isDetailed() ? compose(list, fetchBlobMetadataProvider.get().setContainerName(
+               container), service) : list;
    }
 
    /**

rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesBlobStore.java

@@ -18,9 +18,12 @@
  */
 package org.jclouds.rackspace.cloudfiles.blobstore;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import java.util.Set;
 
 import javax.inject.Inject;
+import javax.inject.Provider;
 import javax.inject.Singleton;
 
 import org.jclouds.blobstore.BlobStoreContext;
@@ -32,6 +35,7 @@ import org.jclouds.blobstore.domain.internal.PageSetImpl;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
 import org.jclouds.blobstore.internal.BaseBlobStore;
 import org.jclouds.blobstore.options.ListContainerOptions;
+import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
 import org.jclouds.blobstore.util.BlobStoreUtils;
 import org.jclouds.http.options.GetOptions;
 import org.jclouds.rackspace.cloudfiles.CloudFilesClient;
@@ -60,6 +64,7 @@ public class CloudFilesBlobStore extends BaseBlobStore {
    private final BlobToObject blob2Object;
    private final ObjectToBlobMetadata object2BlobMd;
    private final BlobToHttpGetOptions blob2ObjectGetOptions;
+   private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
 
    @Inject
    CloudFilesBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils, CloudFilesClient sync,
@@ -67,7 +72,8 @@ public class CloudFilesBlobStore extends BaseBlobStore {
             BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
             ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob,
             BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd,
-            BlobToHttpGetOptions blob2ObjectGetOptions) {
+            BlobToHttpGetOptions blob2ObjectGetOptions,
+            Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
       super(context, blobUtils);
       this.sync = sync;
       this.container2ResourceMd = container2ResourceMd;
@@ -77,6 +83,8 @@ public class CloudFilesBlobStore extends BaseBlobStore {
       this.blob2Object = blob2Object;
       this.object2BlobMd = object2BlobMd;
       this.blob2ObjectGetOptions = blob2ObjectGetOptions;
+      this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
+               "fetchBlobMetadataProvider");
    }
 
    /**
@@ -124,10 +132,13 @@ public class CloudFilesBlobStore extends BaseBlobStore {
     *           container name
     */
    @Override
-   public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions optionsList) {
+   public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
       org.jclouds.rackspace.cloudfiles.options.ListContainerOptions httpOptions = container2ContainerListOptions
-               .apply(optionsList);
+               .apply(options);
-      return container2ResourceList.apply(sync.listObjects(container, httpOptions));
+      PageSet<? extends StorageMetadata> list = container2ResourceList.apply(sync.listObjects(
+               container, httpOptions));
+      return options.isDetailed() ? fetchBlobMetadataProvider.get().setContainerName(container)
+               .apply(list) : list;
    }
 
    /**

rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/ResourceToObjectInfo.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.rackspace.cloudfiles.blobstore.functions;
 
+import java.util.Map.Entry;
+
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -59,8 +61,10 @@ public class ResourceToObjectInfo implements
       to.setLastModified(from.getLastModified());
       if (from.getSize() != null)
          to.setBytes(from.getSize());
-      if (from.getUserMetadata() != null)
+      if (from.getUserMetadata() != null) {
-         to.getMetadata().putAll(from.getUserMetadata());
+         for (Entry<String, String> entry : from.getUserMetadata().entrySet())
+            to.getMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
+      }
       return to;
    }