Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 202: withDetails support for BlobStore.list(String,ListContainerOptions)

This commit is contained in:
Adrian Cole 2010-03-14 23:34:47 -07:00
parent b221f8006f
commit dc512e8bae
27 changed files with 329 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java
View File

@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.Constants;
@ -38,6 +39,8 @@ import org.jclouds.atmosonline.saas.blobstore.functions.DirectoryEntryListToReso
import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlob;
import org.jclouds.atmosonline.saas.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.atmosonline.saas.domain.AtmosObject;
import org.jclouds.atmosonline.saas.domain.BoundedSet;
import org.jclouds.atmosonline.saas.domain.DirectoryEntry;
import org.jclouds.atmosonline.saas.options.ListOptions;
import org.jclouds.atmosonline.saas.util.AtmosStorageUtils;
import org.jclouds.blobstore.BlobStoreContext;
@ -47,6 +50,7 @@ import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobStoreUtils;
import org.jclouds.concurrent.ConcurrentUtils;
import org.jclouds.encryption.EncryptionService;
@ -69,6 +73,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
private final DirectoryEntryListToResourceMetadataList container2ResourceList;
private final EncryptionService encryptionService;
private final BlobToHttpGetOptions blob2ObjectGetOptions;
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
@Inject
AtmosAsyncBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils,
@ -77,7 +82,8 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object,
BlobStoreListOptionsToListOptions container2ContainerListOptions,
DirectoryEntryListToResourceMetadataList container2ResourceList,
EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions) {
EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
super(context, blobUtils, service);
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
this.sync = checkNotNull(sync, "sync");
@ -89,6 +95,8 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
this.blob2Object = checkNotNull(blob2Object, "blob2Object");
this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
this.encryptionService = checkNotNull(encryptionService, "encryptionService");
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
"fetchBlobMetadataProvider");
}
/**
@ -208,7 +216,12 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
org.jclouds.blobstore.options.ListContainerOptions options) {
container = AtmosStorageUtils.adjustContainerIfDirOptionPresent(container, options);
ListOptions nativeOptions = container2ContainerListOptions.apply(options);
return compose(async.listDirectory(container, nativeOptions), container2ResourceList, service);
ListenableFuture<? extends BoundedSet<? extends DirectoryEntry>> returnVal = async
.listDirectory(container, nativeOptions);
ListenableFuture<PageSet<? extends StorageMetadata>> list = compose(returnVal,
container2ResourceList, service);
return options.isDetailed() ? compose(list, fetchBlobMetadataProvider.get().setContainerName(
container), service) : list;
}
/**

14
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java
View File

@ -21,6 +21,7 @@ package org.jclouds.atmosonline.saas.blobstore;
import static com.google.common.base.Preconditions.checkNotNull;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient;
@ -39,6 +40,7 @@ import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseBlobStore;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobStoreUtils;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.options.GetOptions;
@ -56,13 +58,15 @@ public class AtmosBlobStore extends BaseBlobStore {
private final DirectoryEntryListToResourceMetadataList container2ResourceList;
private final EncryptionService encryptionService;
private final BlobToHttpGetOptions blob2ObjectGetOptions;
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
@Inject
AtmosBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils, AtmosStorageClient sync,
ObjectToBlob object2Blob, ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object,
BlobStoreListOptionsToListOptions container2ContainerListOptions,
DirectoryEntryListToResourceMetadataList container2ResourceList,
EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions) {
EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
super(context, blobUtils);
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
this.sync = checkNotNull(sync, "sync");
@ -73,6 +77,8 @@ public class AtmosBlobStore extends BaseBlobStore {
this.blob2Object = checkNotNull(blob2Object, "blob2Object");
this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
this.encryptionService = checkNotNull(encryptionService, "encryptionService");
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
"fetchBlobMetadataProvider");
}
/**
@ -180,7 +186,11 @@ public class AtmosBlobStore extends BaseBlobStore {
org.jclouds.blobstore.options.ListContainerOptions options) {
container = AtmosStorageUtils.adjustContainerIfDirOptionPresent(container, options);
ListOptions nativeOptions = container2ContainerListOptions.apply(options);
return container2ResourceList.apply(sync.listDirectory(container, nativeOptions));
// until includeMeta() option works for namespace interface
PageSet<? extends StorageMetadata> list = container2ResourceList.apply(sync.listDirectory(
container, nativeOptions));
return options.isDetailed() ? fetchBlobMetadataProvider.get().setContainerName(container)
.apply(list) : list;
}
/**

8
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobMetadataToObject.java
View File

@ -18,6 +18,8 @@
*/
package org.jclouds.atmosonline.saas.blobstore.functions;
import java.util.Map.Entry;
import javax.inject.Inject;
import javax.inject.Singleton;
@ -48,8 +50,10 @@ public class BlobMetadataToObject implements Function<BlobMetadata, AtmosObject>
if (from == null)
return null;
UserMetadata userMd = new UserMetadata();
if (from.getUserMetadata() != null)
userMd.getMetadata().putAll(from.getUserMetadata());
if (from.getUserMetadata() != null) {
for (Entry<String, String> entry : from.getUserMetadata().entrySet())
userMd.getMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
}
return factory.create(blob2ContentMd.apply(from), blob2SysMd.apply(from), userMd);
}

3
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobStoreListOptionsToListOptions.java
View File

@ -42,6 +42,9 @@ public class BlobStoreListOptionsToListOptions implements
if (from.getMaxResults() != null) {
httpOptions.limit(from.getMaxResults());
}
if (from.isDetailed()) {
httpOptions.includeMeta();
}
return httpOptions;
}
}

22
atmos/src/main/java/org/jclouds/atmosonline/saas/options/ListOptions.java
View File

@ -56,6 +56,20 @@ public class ListOptions extends BaseHttpRequestOptions {
return this;
}
/**
* the maximum number of items that should be returned. If this is not specified, there is no
* limit.
*/
public ListOptions includeMeta() {
headers.put("x-emc-include-meta", Integer.toString(1));
return this;
}
public boolean metaIncluded() {
String meta = getFirstHeaderOrNull("x-emc-include-meta");
return (meta != null) ? meta.equals("1") : false;
}
public Integer getLimit() {
String maxresults = getFirstHeaderOrNull("x-emc-limit");
return (maxresults != null) ? new Integer(maxresults) : null;
@ -71,6 +85,14 @@ public class ListOptions extends BaseHttpRequestOptions {
return options.token(token);
}
/**
* @see ListOptions#includeMeta()
*/
public static ListOptions includeMeta() {
ListOptions options = new ListOptions();
return options.includeMeta();
}
/**
* @see ListOptions#limit(int)
*/

3
atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/functions/ListOptionsToBlobStoreListOptions.java
View File

@ -39,6 +39,9 @@ public class ListOptionsToBlobStoreListOptions implements
if (optionsList[0].getLimit() != null) {
options.maxResults(optionsList[0].getLimit());
}
if (optionsList[0].metaIncluded()) {
options.withDetails();
}
}
return options;
}

4
atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageContainerIntegrationTest.java
View File

@ -35,4 +35,8 @@ public class AtmosStorageContainerIntegrationTest extends BaseContainerIntegrati
// Not currently working
}
@Override
public void testListContainerMarker() throws InterruptedException, UnsupportedEncodingException {
// Not currently working https://community.emc.com/thread/100545
}
}

4
atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageTestInitializer.java
View File

@ -40,8 +40,8 @@ public class AtmosStorageTestInitializer extends BaseTestInitializer {
@Override
protected BlobStoreContext createLiveContext(Module configurationModule, String url, String app,
String account, String key) throws IOException {
return new BlobStoreContextFactory().createContext("atmos", account, key, ImmutableSet.of(
configurationModule, new Log4JLoggingModule()), new Properties());
return new BlobStoreContextFactory().createContext("atmosonline", account, key, ImmutableSet
.of(configurationModule, new Log4JLoggingModule()), new Properties());
}
@Override

27
atmos/src/test/java/org/jclouds/atmosonline/saas/options/ListOptionsTest.java
View File

@ -37,20 +37,37 @@ public class ListOptionsTest {
assertEquals(ImmutableList.of("a"), options.buildRequestHeaders().get("x-emc-token"));
}
public void testTokenStatic() {
ListOptions options = ListOptions.Builder.token("a");
assertEquals(ImmutableList.of("a"), options.buildRequestHeaders().get("x-emc-token"));
}
public void testLimit() {
int limit = 1;
ListOptions options = new ListOptions().limit(limit);
assertEquals(ImmutableList.of("1"), options.buildRequestHeaders().get("x-emc-limit"));
}
public void testTokenStatic() {
ListOptions options = ListOptions.Builder.token("a");
assertEquals(ImmutableList.of("a"), options.buildRequestHeaders().get("x-emc-token"));
}
public void testLimitStatic() {
int limit = 1;
ListOptions options = ListOptions.Builder.limit(limit);
assertEquals(ImmutableList.of("1"), options.buildRequestHeaders().get("x-emc-limit"));
}
public void testNoMeta() {
ListOptions options = new ListOptions();
assert !options.metaIncluded();
}
public void testMeta() {
ListOptions options = new ListOptions().includeMeta();
assertEquals(ImmutableList.of("1"), options.buildRequestHeaders().get("x-emc-include-meta"));
assert options.metaIncluded();
}
public void testMetaStatic() {
ListOptions options = ListOptions.Builder.includeMeta();
assertEquals(ImmutableList.of("1"), options.buildRequestHeaders().get("x-emc-include-meta"));
assert options.metaIncluded();
}
}

4
aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java
View File

@ -321,6 +321,7 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule {
holder.logger.debug("<< didn't match os(%s)", matcher.group(1));
}
}
try {
images
.add(new ImageImpl(
from.getId(),
@ -334,6 +335,9 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule {
osDescription,
from.getArchitecture() == org.jclouds.aws.ec2.domain.Image.Architecture.I386 ? Architecture.X86_32
: Architecture.X86_64));
} catch (NullPointerException e) {
holder.logger.debug("<< image (%s) missing (%s)", from.getId(), e.getMessage());
}
}
}
holder.logger.debug("<< images(%d)", images.size());

13
aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3AsyncBlobStore.java
View File

@ -26,6 +26,7 @@ import java.util.concurrent.ExecutorService;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.Constants;
@ -52,6 +53,7 @@ import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobStoreUtils;
import org.jclouds.http.options.GetOptions;
@ -75,6 +77,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
private final ObjectToBlob object2Blob;
private final BlobToObject blob2Object;
private final ObjectToBlobMetadata object2BlobMd;
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
@Inject
S3AsyncBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils,
@ -83,7 +86,8 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
ContainerToBucketListOptions container2BucketListOptions,
BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
ObjectToBlobMetadata object2BlobMd) {
ObjectToBlobMetadata object2BlobMd,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
super(context, blobUtils, service);
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
this.async = checkNotNull(async, "async");
@ -95,6 +99,8 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
this.object2Blob = checkNotNull(object2Blob, "object2Blob");
this.blob2Object = checkNotNull(blob2Object, "blob2Object");
this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
"fetchBlobMetadataProvider");
}
/**
@ -148,7 +154,10 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
ListContainerOptions options) {
ListBucketOptions httpOptions = container2BucketListOptions.apply(options);
ListenableFuture<ListBucketResponse> returnVal = async.listBucket(container, httpOptions);
return compose(returnVal, bucket2ResourceList, service);
ListenableFuture<PageSet<? extends StorageMetadata>> list = compose(returnVal,
bucket2ResourceList, service);
return options.isDetailed() ? compose(list, fetchBlobMetadataProvider.get().setContainerName(
container), service) : list;
}
/**

18
aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java
View File

@ -23,6 +23,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.util.SortedSet;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.aws.domain.Region;
@ -45,6 +46,7 @@ import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseBlobStore;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobStoreUtils;
import org.jclouds.http.options.GetOptions;
import org.jclouds.util.Utils;
@ -67,6 +69,7 @@ public class S3BlobStore extends BaseBlobStore {
private final BlobToObject blob2Object;
private final ObjectToBlobMetadata object2BlobMd;
private final BlobToHttpGetOptions blob2ObjectGetOptions;
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
@Inject
S3BlobStore(BlobStoreContext context, BlobStoreUtils blobUtils, S3Client sync,
@ -74,7 +77,8 @@ public class S3BlobStore extends BaseBlobStore {
ContainerToBucketListOptions container2BucketListOptions,
BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob,
BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object,
ObjectToBlobMetadata object2BlobMd) {
ObjectToBlobMetadata object2BlobMd,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
super(context, blobUtils);
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
this.sync = checkNotNull(sync, "sync");
@ -85,6 +89,8 @@ public class S3BlobStore extends BaseBlobStore {
this.object2Blob = checkNotNull(object2Blob, "object2Blob");
this.blob2Object = checkNotNull(blob2Object, "blob2Object");
this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
"fetchBlobMetadataProvider");
}
/**
@ -132,9 +138,12 @@ public class S3BlobStore extends BaseBlobStore {
* bucket name
*/
@Override
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions optionsList) {
ListBucketOptions httpOptions = container2BucketListOptions.apply(optionsList);
return bucket2ResourceList.apply(sync.listBucket(container, httpOptions));
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
ListBucketOptions httpOptions = container2BucketListOptions.apply(options);
PageSet<? extends StorageMetadata> list = bucket2ResourceList.apply(sync.listBucket(
container, httpOptions));
return options.isDetailed() ? fetchBlobMetadataProvider.get().setContainerName(container)
.apply(list) : list;
}
/**
@ -191,7 +200,6 @@ public class S3BlobStore extends BaseBlobStore {
@Override
public BlobMetadata blobMetadata(String container, String key) {
return object2BlobMd.apply(sync.headObject(container, key));
}
/**

8
aws/core/src/main/java/org/jclouds/aws/s3/blobstore/functions/BlobToObjectMetadata.java
View File

@ -18,6 +18,8 @@
*/
package org.jclouds.aws.s3.blobstore.functions;
import java.util.Map.Entry;
import javax.inject.Singleton;
import org.jclouds.aws.s3.domain.MutableObjectMetadata;
@ -42,8 +44,10 @@ public class BlobToObjectMetadata implements Function<BlobMetadata, MutableObjec
to.setLastModified(from.getLastModified());
if (from.getSize() != null)
to.setSize(from.getSize());
if (from.getUserMetadata() != null)
to.setUserMetadata(from.getUserMetadata());
if (from.getUserMetadata() != null) {
for (Entry<String, String> entry : from.getUserMetadata().entrySet())
to.getUserMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
}
return to;
}

8
azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/BlobMetadataToBlobProperties.java
View File

@ -18,6 +18,8 @@
*/
package org.jclouds.azure.storage.blob.blobstore.functions;
import java.util.Map.Entry;
import javax.inject.Singleton;
import org.jclouds.azure.storage.blob.domain.MutableBlobProperties;
@ -42,8 +44,10 @@ public class BlobMetadataToBlobProperties implements Function<BlobMetadata, Muta
to.setLastModified(from.getLastModified());
if (from.getSize() != null)
to.setContentLength(from.getSize());
if (from.getUserMetadata() != null)
to.setMetadata(from.getUserMetadata());
if (from.getUserMetadata() != null) {
for (Entry<String, String> entry : from.getUserMetadata().entrySet())
to.getMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
}
return to;
}

3
azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/functions/ListOptionsToListBlobsOptions.java
View File

@ -48,6 +48,9 @@ public class ListOptionsToListBlobsOptions implements
if (from.getMaxResults() != null) {
httpOptions.maxResults(from.getMaxResults());
}
if (from.isDetailed()) {
httpOptions.includeMetadata();
}
return httpOptions;
}
}

4
azure/src/main/java/org/jclouds/azure/storage/options/ListOptions.java
View File

@ -47,6 +47,10 @@ public class ListOptions extends BaseHttpRequestOptions {
return this;
}
public boolean getIncludeMetadata() {
return getFirstQueryOrNull("include").equals("metadata");
}
/**
* Filters the results to return only objects whose name begins with the specified prefix.
*/

6
azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/functions/ListBlobsOptionsToListOptions.java
View File

@ -29,7 +29,8 @@ import com.google.common.base.Function;
* @author Adrian Cole
*/
@Singleton
public class ListBlobsOptionsToListOptions implements Function<ListBlobsOptions[], ListContainerOptions> {
public class ListBlobsOptionsToListOptions implements
Function<ListBlobsOptions[], ListContainerOptions> {
public ListContainerOptions apply(ListBlobsOptions[] optionsList) {
ListContainerOptions options = new ListContainerOptions();
if (optionsList.length != 0) {
@ -47,6 +48,9 @@ public class ListBlobsOptionsToListOptions implements Function<ListBlobsOptions[
if (optionsList[0].getPrefix() != null) {
options.inDirectory(optionsList[0].getPrefix());
}
if (optionsList[0].getIncludeMetadata()) {
options.withDetails();
}
}
return options;
}

8
azure/src/test/resources/log4j.xml
View File

@ -122,9 +122,11 @@
<appender-ref ref="ASYNCWIRE" />
</category>
<!--<category name="jclouds.wire"> <priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" /> </category>
--><!-- ======================= -->
<category name="jclouds.wire">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
</category>
<!-- ======================= -->
<!-- Setup the Root category -->
<!-- ======================= -->

20
blobstore/src/main/java/org/jclouds/blobstore/TransientAsyncBlobStore.java
View File

@ -144,20 +144,20 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
* default maxResults is 1000
*/
@Override
public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(final String name,
ListContainerOptions options) {
final Map<String, Blob> realContents = getContainerToBlobs().get(name);
public ListenableFuture<? extends PageSet<? extends StorageMetadata>> list(
final String container, ListContainerOptions options) {
final Map<String, Blob> realContents = getContainerToBlobs().get(container);
if (realContents == null)
return immediateFailedFuture(cnfe(name));
return immediateFailedFuture(cnfe(container));
SortedSet<StorageMetadata> contents = Sets.newTreeSet(Iterables.transform(realContents
.keySet(), new Function<String, StorageMetadata>() {
public StorageMetadata apply(String key) {
Blob oldBlob = realContents.get(key);
checkState(oldBlob != null, "blob " + key
+ " is not present although it was in the list of " + name);
checkState(oldBlob.getMetadata() != null, "blob " + name + "/" + key
+ " is not present although it was in the list of " + container);
checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key
+ " has no metadata");
MutableBlobMetadata md = copy(oldBlob.getMetadata());
String directoryName = ifDirectoryReturnName.execute(md);
@ -224,6 +224,14 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
}
}));
}
// trim metadata, if the response isn't supposed to be detailed.
if (!options.isDetailed()) {
for (StorageMetadata md : contents) {
md.getUserMetadata().clear();
}
}
return immediateFuture(new PageSetImpl<StorageMetadata>(contents, marker));
}

2
blobstore/src/main/java/org/jclouds/blobstore/domain/internal/PageSetImpl.java
View File

@ -32,7 +32,7 @@ public class PageSetImpl<T> extends HashSet<T> implements PageSet<T> {
private static final long serialVersionUID = -7133632087734650835L;
protected final String marker;
public PageSetImpl(Iterable<T> contents, @Nullable String nextMarker) {
public PageSetImpl(Iterable<? extends T> contents, @Nullable String nextMarker) {
Iterables.addAll(this, contents);
this.marker = nextMarker;
}

49
blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java
View File

@ -37,13 +37,23 @@ import static com.google.common.base.Preconditions.checkNotNull;
* @author Adrian Cole
*/
public class ListContainerOptions extends ListOptions implements Cloneable {
public static final ImmutableListContainerOptions NONE = new ImmutableListContainerOptions(
new ListContainerOptions());
private String dir;
private boolean recursive;
private boolean detailed;
public ListContainerOptions() {
}
ListContainerOptions(Integer maxKeys, String marker, String dir, boolean recursive) {
ListContainerOptions(Integer maxKeys, String marker, String dir, boolean recursive,
boolean detailed) {
super(maxKeys, marker);
this.dir = dir;
this.recursive = recursive;
this.detailed = detailed;
}
public static class ImmutableListContainerOptions extends ListContainerOptions {
@ -68,6 +78,11 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
throw new UnsupportedOperationException();
}
@Override
public boolean isDetailed() {
return delegate.isDetailed();
}
@Override
public boolean isRecursive() {
return delegate.isRecursive();
@ -106,13 +121,6 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
}
public static final ImmutableListContainerOptions NONE = new ImmutableListContainerOptions(
new ListContainerOptions());
private String dir;
private boolean recursive;
public String getDir() {
return dir;
}
@ -121,6 +129,10 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
return recursive;
}
public boolean isDetailed() {
return detailed;
}
/**
* This will list the contents of a virtual or real directory path.
*
@ -154,6 +166,15 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
return this;
}
/**
* populate each result with detailed such as metadata even if it incurs extra requests to the
* service.
*/
public ListContainerOptions withDetails() {
this.detailed = true;
return this;
}
public static class Builder {
/**
@ -188,15 +209,23 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
return options.recursive();
}
/**
* @see ListContainerOptions#withDetails()
*/
public static ListContainerOptions withDetails() {
ListContainerOptions options = new ListContainerOptions();
return options.withDetails();
}
}
@Override
public ListContainerOptions clone() {
return new ListContainerOptions(getMaxResults(), getMarker(), dir, recursive);
return new ListContainerOptions(getMaxResults(), getMarker(), dir, recursive, detailed);
}
@Override
public String toString() {
return "[dir=" + dir + ", recursive=" + recursive + ", maxResults=" + getMaxResults() + "]";
return "[dir=" + dir + ", recursive=" + recursive + ", detailed=" + detailed
+ ", maxResults=" + getMaxResults() + "]";
}
}

2
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java
View File

@ -103,7 +103,7 @@ public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrateg
}
}
}, Executors.sameThreadExecutor());
responses.put(md, ablobstore.getBlob(container, md.getName()));
responses.put(md, future);
}
exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, String.format(
"getting from containerName: %s", container));

40
blobstore/src/test/java/org/jclouds/blobstore/integration/TransientContainerIntegrationTest.java
View File

@ -18,9 +18,20 @@
*/
package org.jclouds.blobstore.integration;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults;
import static org.testng.Assert.assertEquals;
import javax.ws.rs.core.MediaType;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.integration.internal.BaseContainerIntegrationTest;
import org.testng.annotations.Test;
import com.google.common.collect.Iterables;
/**
* @author James Murty
* @author Adrian Cole
@ -28,4 +39,33 @@ import org.testng.annotations.Test;
@Test(groups = { "integration", "live" }, testName = "blobstore.TransientContainerIntegrationTest")
public class TransientContainerIntegrationTest extends BaseContainerIntegrationTest {
@Test(groups = { "integration", "live" })
public void testNotWithDetails() throws InterruptedException {
String key = "hello";
Blob object = context.getBlobStore().newBlob(key);
object.setPayload(TEST_STRING);
object.getMetadata().setContentType(MediaType.TEXT_PLAIN);
object.getMetadata().setSize(new Long(TEST_STRING.length()));
// NOTE all metadata in jclouds comes out as lowercase, in an effort to normalize the
// providers.
object.getMetadata().getUserMetadata().put("Adrian", "powderpuff");
String containerName = getContainerName();
try {
addBlobToContainer(containerName, object);
validateContent(containerName, key);
PageSet<? extends StorageMetadata> container = context.getBlobStore().list(containerName,
maxResults(1));
BlobMetadata metadata = (BlobMetadata) Iterables.getOnlyElement(container);
// transient container should be lenient and not return metadata on undetailed listing.
assertEquals(metadata.getUserMetadata().size(), 0);
} finally {
returnContainer(containerName);
}
}
}

37
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java
View File

@ -28,12 +28,17 @@ import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import javax.ws.rs.core.MediaType;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.encryption.internal.JCEEncryptionService;
import org.testng.annotations.Test;
import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
/**
* @author Adrian Cole
@ -58,6 +63,38 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
}
}
@Test(groups = { "integration", "live" })
public void testWithDetails() throws InterruptedException {
String key = "hello";
Blob object = context.getBlobStore().newBlob(key);
object.setPayload(TEST_STRING);
object.getMetadata().setContentType(MediaType.TEXT_PLAIN);
object.getMetadata().setSize(new Long(TEST_STRING.length()));
// NOTE all metadata in jclouds comes out as lowercase, in an effort to normalize the
// providers.
object.getMetadata().getUserMetadata().put("Adrian", "powderpuff");
object.getMetadata().setContentMD5(new JCEEncryptionService().md5(TEST_STRING.getBytes()));
String containerName = getContainerName();
try {
addBlobToContainer(containerName, object);
validateContent(containerName, key);
PageSet<? extends StorageMetadata> container = context.getBlobStore().list(containerName,
maxResults(1).withDetails());
BlobMetadata metadata = (BlobMetadata) Iterables.getOnlyElement(container);
assert metadata.getContentType().startsWith("text/plain") : metadata.getContentType();
assertEquals(metadata.getSize(), new Long(TEST_STRING.length()));
assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff");
assertEquals(metadata.getContentMD5(), new JCEEncryptionService().md5(TEST_STRING
.getBytes()));
} finally {
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testClearWhenContentsUnderPath() throws InterruptedException {
String containerName = getContainerName();

14
rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java
View File

@ -18,6 +18,7 @@
*/
package org.jclouds.rackspace.cloudfiles.blobstore;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.util.concurrent.Futures.compose;
import java.util.Set;
@ -25,6 +26,7 @@ import java.util.concurrent.ExecutorService;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.Constants;
@ -37,6 +39,7 @@ import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobStoreUtils;
import org.jclouds.http.options.GetOptions;
import org.jclouds.rackspace.cloudfiles.CloudFilesAsyncClient;
@ -71,6 +74,7 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore {
private final BlobToObject blob2Object;
private final ObjectToBlobMetadata object2BlobMd;
private final BlobToHttpGetOptions blob2ObjectGetOptions;
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
@Inject
CloudFilesAsyncBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils,
@ -79,7 +83,8 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore {
BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob,
BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd,
BlobToHttpGetOptions blob2ObjectGetOptions) {
BlobToHttpGetOptions blob2ObjectGetOptions,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
super(context, blobUtils, service);
this.sync = sync;
this.async = async;
@ -90,6 +95,8 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore {
this.blob2Object = blob2Object;
this.object2BlobMd = object2BlobMd;
this.blob2ObjectGetOptions = blob2ObjectGetOptions;
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
"fetchBlobMetadataProvider");
}
/**
@ -139,7 +146,10 @@ public class CloudFilesAsyncBlobStore extends BaseAsyncBlobStore {
org.jclouds.rackspace.cloudfiles.options.ListContainerOptions httpOptions = container2ContainerListOptions
.apply(options);
ListenableFuture<PageSet<ObjectInfo>> returnVal = async.listObjects(container, httpOptions);
return compose(returnVal, container2ResourceList, service);
ListenableFuture<PageSet<? extends StorageMetadata>> list = compose(returnVal,
container2ResourceList, service);
return options.isDetailed() ? compose(list, fetchBlobMetadataProvider.get().setContainerName(
container), service) : list;
}
/**

19
rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/CloudFilesBlobStore.java
View File

@ -18,9 +18,12 @@
*/
package org.jclouds.rackspace.cloudfiles.blobstore;
import static com.google.common.base.Preconditions.checkNotNull;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.blobstore.BlobStoreContext;
@ -32,6 +35,7 @@ import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseBlobStore;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobStoreUtils;
import org.jclouds.http.options.GetOptions;
import org.jclouds.rackspace.cloudfiles.CloudFilesClient;
@ -60,6 +64,7 @@ public class CloudFilesBlobStore extends BaseBlobStore {
private final BlobToObject blob2Object;
private final ObjectToBlobMetadata object2BlobMd;
private final BlobToHttpGetOptions blob2ObjectGetOptions;
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
@Inject
CloudFilesBlobStore(BlobStoreContext context, BlobStoreUtils blobUtils, CloudFilesClient sync,
@ -67,7 +72,8 @@ public class CloudFilesBlobStore extends BaseBlobStore {
BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob,
BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd,
BlobToHttpGetOptions blob2ObjectGetOptions) {
BlobToHttpGetOptions blob2ObjectGetOptions,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
super(context, blobUtils);
this.sync = sync;
this.container2ResourceMd = container2ResourceMd;
@ -77,6 +83,8 @@ public class CloudFilesBlobStore extends BaseBlobStore {
this.blob2Object = blob2Object;
this.object2BlobMd = object2BlobMd;
this.blob2ObjectGetOptions = blob2ObjectGetOptions;
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
"fetchBlobMetadataProvider");
}
/**
@ -124,10 +132,13 @@ public class CloudFilesBlobStore extends BaseBlobStore {
* container name
*/
@Override
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions optionsList) {
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
org.jclouds.rackspace.cloudfiles.options.ListContainerOptions httpOptions = container2ContainerListOptions
.apply(optionsList);
return container2ResourceList.apply(sync.listObjects(container, httpOptions));
.apply(options);
PageSet<? extends StorageMetadata> list = container2ResourceList.apply(sync.listObjects(
container, httpOptions));
return options.isDetailed() ? fetchBlobMetadataProvider.get().setContainerName(container)
.apply(list) : list;
}
/**

8
rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/ResourceToObjectInfo.java
View File

@ -18,6 +18,8 @@
*/
package org.jclouds.rackspace.cloudfiles.blobstore.functions;
import java.util.Map.Entry;
import javax.inject.Inject;
import javax.inject.Singleton;
@ -59,8 +61,10 @@ public class ResourceToObjectInfo implements
to.setLastModified(from.getLastModified());
if (from.getSize() != null)
to.setBytes(from.getSize());
if (from.getUserMetadata() != null)
to.getMetadata().putAll(from.getUserMetadata());
if (from.getUserMetadata() != null) {
for (Entry<String, String> entry : from.getUserMetadata().entrySet())
to.getMetadata().put(entry.getKey().toLowerCase(), entry.getValue());
}
return to;
}