Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

corrected CNFE, as it should have been a subclass of ResourceNotFoundException; stabalized the directory support for rackspace and atmos 

git-svn-id: http://jclouds.googlecode.com/svn/trunk@2753 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2010-02-02 04:07:49 +00:00
parent a268309c94
commit 7120f6e536
74 changed files with 788 additions and 688 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java
View File

@ -45,8 +45,6 @@ import org.jclouds.atmosonline.saas.functions.ReturnEndpointIfAlreadyExists;
import org.jclouds.atmosonline.saas.options.ListOptions;
import org.jclouds.blobstore.attr.ConsistencyModel;
import org.jclouds.blobstore.attr.ConsistencyModels;
import org.jclouds.blobstore.functions.ReturnFalseOnKeyNotFound;
import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404;
import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
@ -59,6 +57,8 @@ import org.jclouds.rest.annotations.QueryParams;
import org.jclouds.rest.annotations.RequestFilters;
import org.jclouds.rest.annotations.ResponseParser;
import org.jclouds.rest.annotations.SkipEncoding;
import org.jclouds.rest.functions.ReturnFalseOnResourceNotFound;
import org.jclouds.rest.functions.ReturnNullOnResourceNotFound;
import com.google.common.util.concurrent.ListenableFuture;
@ -75,7 +75,9 @@ import com.google.common.util.concurrent.ListenableFuture;
@SkipEncoding( { '/' })
@ConsistencyModel(ConsistencyModels.EVENTUAL)
public interface AtmosStorageAsyncClient {
/**
* Creates a default implementation of AtmosObject
*/
AtmosObject newObject();
/**
@ -134,7 +136,7 @@ public interface AtmosStorageAsyncClient {
*/
@GET
@ResponseParser(ParseObjectFromHeadersAndHttpContent.class)
@ExceptionParser(ReturnNullOnKeyNotFound.class)
@ExceptionParser(ReturnNullOnResourceNotFound.class)
@Path("/rest/namespace/{path}")
@Consumes(MediaType.WILDCARD)
ListenableFuture<AtmosObject> readFile(@PathParam("path") String path, GetOptions... options);
@ -144,7 +146,7 @@ public interface AtmosStorageAsyncClient {
*/
@HEAD
@ResponseParser(ParseObjectFromHeadersAndHttpContent.class)
@ExceptionParser(ReturnNullOnKeyNotFound.class)
@ExceptionParser(ReturnNullOnResourceNotFound.class)
@Path("/rest/namespace/{path}")
@Consumes(MediaType.WILDCARD)
ListenableFuture<AtmosObject> headFile(@PathParam("path") String path);
@ -154,7 +156,7 @@ public interface AtmosStorageAsyncClient {
*/
@HEAD
@ResponseParser(ParseSystemMetadataFromHeaders.class)
@ExceptionParser(ReturnNullOnKeyNotFound.class)
@ExceptionParser(ReturnNullOnResourceNotFound.class)
// currently throws 403 errors @QueryParams(keys = "metadata/system")
@Path("/rest/namespace/{path}")
@Consumes(MediaType.WILDCARD)
@ -165,7 +167,7 @@ public interface AtmosStorageAsyncClient {
*/
@HEAD
@ResponseParser(ParseSystemMetadataFromHeaders.class)
@ExceptionParser(ReturnNullOnKeyNotFound.class)
@ExceptionParser(ReturnNullOnResourceNotFound.class)
@Path("/rest/namespace/{path}")
@QueryParams(keys = "metadata/user")
@Consumes(MediaType.WILDCARD)
@ -184,7 +186,7 @@ public interface AtmosStorageAsyncClient {
* @see AtmosStorageClient#pathExists
*/
@HEAD
@ExceptionParser(ReturnFalseOnKeyNotFound.class)
@ExceptionParser(ReturnFalseOnResourceNotFound.class)
@Path("/rest/namespace/{path}")
@Consumes(MediaType.WILDCARD)
ListenableFuture<Boolean> pathExists(@PathParam("path") String path);

22
atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageClient.java
View File

@ -40,13 +40,14 @@ import org.jclouds.http.options.GetOptions;
*/
@Timeout(duration = 120, timeUnit = TimeUnit.SECONDS)
public interface AtmosStorageClient {
/**
* Creates a default implementation of AtmosObject
*/
AtmosObject newObject();
BoundedSet<? extends DirectoryEntry> listDirectories(ListOptions... options);
BoundedSet<? extends DirectoryEntry> listDirectory(String directoryName,
ListOptions... options);
BoundedSet<? extends DirectoryEntry> listDirectory(String directoryName, ListOptions... options);
URI createDirectory(String directoryName);
@ -65,23 +66,8 @@ public interface AtmosStorageClient {
UserMetadata getUserMetadata(String path);
/**
*
* @param path
* @return
* @throws AtmosStorageResponseException
* , if the path is a directory and not empty
*/
void deletePath(String path);
boolean pathExists(String path);
// signature currently doesn't work
// @POST
// @QueryParams(keys = "acl")
// @Headers(keys = { "x-emc-useracl", "x-emc-groupacl" }, values = { "root=FULL_CONTROL",
// "other=READ" })
// @Consumes(MediaType.WILDCARD)
// void makePublic(@Endpoint URI url);
}

47
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java
View File

@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.util.concurrent.Futures.compose;
import java.net.URI;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import javax.inject.Inject;
@ -46,13 +47,11 @@ import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
import org.jclouds.blobstore.util.BlobStoreUtils;
import org.jclouds.concurrent.ConcurrentUtils;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.options.GetOptions;
import org.jclouds.util.Utils;
import com.google.common.base.Function;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.util.concurrent.ListenableFuture;
/**
@ -140,8 +139,8 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
* {@link AtmosStorageAsyncClient#pathExists} until it is true.
*/
protected boolean deleteAndVerifyContainerGone(final String container) {
sync.deletePath(container);
return !sync.pathExists(container);
sync.deletePath(container + "/");
return !sync.pathExists(container + "/");
}
/**
@ -149,7 +148,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<Boolean> containerExists(String container) {
return async.pathExists(container);
return async.pathExists(container + "/");
}
/**
@ -160,6 +159,14 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
return async.pathExists(container + "/" + directory + "/");
}
/**
* This implementation invokes {@link #removeBlob}
*/
@Override
public ListenableFuture<Void> deleteDirectory(String containerName, String directory) {
return removeBlob(containerName, directory + "/");
}
/**
* This implementation invokes {@link AtmosStorageAsyncClient#pathExists}
*
@ -210,31 +217,15 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<String> putBlob(final String container, final Blob blob) {
final String path = container + "/" + blob.getMetadata().getName();
return compose(async.deletePath(path), new Function<Void, String>() {
return ConcurrentUtils.makeListenable(service.submit(new Callable<String>() {
@Override
public String call() throws Exception {
return AtmosStorageUtils.putBlob(sync, encryptionService, blob2Object, container, blob);
public String apply(Void from) {
try {
if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
public Boolean get() {
return !sync.pathExists(path);
}
}, 300)) {
throw new IllegalStateException(path + " still exists after deleting!");
}
if (blob.getMetadata().getContentMD5() != null)
blob.getMetadata().getUserMetadata().put("content-md5",
encryptionService.toHexString(blob.getMetadata().getContentMD5()));
sync.createFile(container, blob2Object.apply(blob));
return path;
} catch (InterruptedException e) {
Throwables.propagate(e);
}
assert false : " should have propagated error";
return null;
}
}, service);
}), service);
}

22
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java
View File

@ -87,8 +87,8 @@ public class AtmosBlobStore extends BaseBlobStore {
* {@link AtmosStorageAsyncClient#pathExists} until it is true.
*/
protected boolean deleteAndVerifyContainerGone(final String container) {
sync.deletePath(container);
return !sync.pathExists(container);
sync.deletePath(container + "/");
return !sync.pathExists(container + "/");
}
/**
@ -116,12 +116,20 @@ public class AtmosBlobStore extends BaseBlobStore {
sync.createDirectory(container + "/" + directory);
}
/**
* This implementation invokes {@link #removeBlob}
*/
@Override
public void deleteDirectory(String containerName, String directory) {
removeBlob(containerName, directory + "/");
}
/**
* This implementation invokes {@link AtmosStorageClient#pathExists}
*/
@Override
public boolean containerExists(String container) {
return sync.pathExists(container);
return sync.pathExists(container + "/");
}
/**
@ -181,13 +189,7 @@ public class AtmosBlobStore extends BaseBlobStore {
*/
@Override
public String putBlob(final String container, final Blob blob) {
final String path = container + "/" + blob.getMetadata().getName();
deleteAndEnsurePathGone(path);
if (blob.getMetadata().getContentMD5() != null)
blob.getMetadata().getUserMetadata().put("content-md5",
encryptionService.toHexString(blob.getMetadata().getContentMD5()));
sync.createFile(container, blob2Object.apply(blob));
return path;
return AtmosStorageUtils.putBlob(sync, encryptionService, blob2Object, container, blob);
}
/**

5
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/config/AtmosBlobStoreContextModule.java
View File

@ -25,7 +25,6 @@ import org.jclouds.atmosonline.saas.AtmosStorageClient;
import org.jclouds.atmosonline.saas.blobstore.AtmosAsyncBlobStore;
import org.jclouds.atmosonline.saas.blobstore.AtmosBlobStore;
import org.jclouds.atmosonline.saas.blobstore.strategy.FindMD5InUserMetadata;
import org.jclouds.atmosonline.saas.blobstore.strategy.RecursiveRemove;
import org.jclouds.atmosonline.saas.config.AtmosStorageContextModule;
import org.jclouds.blobstore.AsyncBlobStore;
import org.jclouds.blobstore.BlobMap;
@ -34,8 +33,6 @@ import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.InputStreamMap;
import org.jclouds.blobstore.config.BlobStoreMapModule;
import org.jclouds.blobstore.internal.BlobStoreContextImpl;
import org.jclouds.blobstore.strategy.ClearContainerStrategy;
import org.jclouds.blobstore.strategy.ClearListStrategy;
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.lifecycle.Closer;
import org.jclouds.rest.RestContext;
@ -56,8 +53,6 @@ public class AtmosBlobStoreContextModule extends AtmosStorageContextModule {
bind(AsyncBlobStore.class).to(AtmosAsyncBlobStore.class).asEagerSingleton();
bind(BlobStore.class).to(AtmosBlobStore.class).asEagerSingleton();
bind(ContainsValueInListStrategy.class).to(FindMD5InUserMetadata.class);
bind(ClearListStrategy.class).to(RecursiveRemove.class);
bind(ClearContainerStrategy.class).to(RecursiveRemove.class);
}
@Provides

6
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/BlobToSystemMetadata.java
View File

@ -32,9 +32,9 @@ import com.google.common.base.Function;
@Singleton
public class BlobToSystemMetadata implements Function<BlobMetadata, SystemMetadata> {
public SystemMetadata apply(BlobMetadata base) {
return new SystemMetadata(null, base.getLastModified(), null, null, null, 1, null, base
.getName(), null, (base.getSize() != null) ? base.getSize() : 0, FileType.REGULAR,
"root");
return new SystemMetadata(base.getContentMD5(), null, base.getLastModified(), null, null,
null, 1, null, base.getName(), null, (base.getSize() != null) ? base.getSize() : 0,
FileType.REGULAR, "root");
}
}

1
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlob.java
View File

@ -49,6 +49,7 @@ public class ObjectToBlob implements Function<AtmosObject, Blob> {
Blob blob = blobFactory.create(object2BlobMd.apply(from));
if (from.getContentMetadata().getContentLength() != null)
blob.setContentLength(from.getContentMetadata().getContentLength());
blob.getMetadata().setContentMD5(from.getSystemMetadata().getContentMD5());
blob.setPayload(checkNotNull(from.getPayload(), "payload: " + from));
blob.setAllHeaders(from.getAllHeaders());
return blob;

9
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/functions/ObjectToBlobMetadata.java
View File

@ -31,7 +31,6 @@ import org.jclouds.atmosonline.saas.functions.AtmosObjectName;
import org.jclouds.blobstore.domain.MutableBlobMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
import org.jclouds.encryption.EncryptionService;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableSet;
@ -46,12 +45,10 @@ public class ObjectToBlobMetadata implements Function<AtmosObject, MutableBlobMe
private static final Set<String> systemMetadata = ImmutableSet.of("atime", "mtime", "ctime",
"itime", "type", "uid", "gid", "objectid", "objname", "size", "nlink", "policyname",
"content-md5");
private final EncryptionService encryptionService;
@Inject
protected ObjectToBlobMetadata(AtmosObjectName objectName, EncryptionService encryptionService) {
protected ObjectToBlobMetadata(AtmosObjectName objectName) {
this.objectName = objectName;
this.encryptionService = encryptionService;
}
public MutableBlobMetadata apply(AtmosObject from) {
@ -60,9 +57,7 @@ public class ObjectToBlobMetadata implements Function<AtmosObject, MutableBlobMe
MutableBlobMetadata to = new MutableBlobMetadataImpl();
to.setId(from.getSystemMetadata().getObjectID());
to.setLastModified(from.getSystemMetadata().getLastUserDataModification());
String md5hex = from.getUserMetadata().getMetadata().get("content-md5");
if (md5hex != null)
to.setContentMD5(encryptionService.fromHexString(md5hex));
to.setContentMD5(from.getSystemMetadata().getContentMD5());
if (from.getContentMetadata().getContentType() != null)
to.setContentType(from.getContentMetadata().getContentType());
to.setName(objectName.apply(from));

18
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java
View File

@ -18,6 +18,7 @@
*/
package org.jclouds.atmosonline.saas.blobstore.strategy;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import java.util.Arrays;
@ -40,7 +41,7 @@ import org.jclouds.blobstore.functions.ObjectMD5;
import org.jclouds.blobstore.internal.BlobRuntimeException;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
import org.jclouds.blobstore.strategy.ListBlobsInContainer;
import org.jclouds.logging.Logger;
import com.google.common.base.Throwables;
@ -58,7 +59,7 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
@Resource
protected Logger logger = Logger.NULL;
protected final ObjectMD5 objectMD5;
protected final ListBlobMetadataStrategy getAllBlobMetadata;
protected final ListBlobsInContainer getAllBlobMetadata;
private final AtmosStorageAsyncClient client;
private final ExecutorService userExecutor;
/**
@ -71,7 +72,7 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
@Inject
private FindMD5InUserMetadata(
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
ObjectMD5 objectMD5, ListBlobMetadataStrategy getAllBlobMetadata,
ObjectMD5 objectMD5, ListBlobsInContainer getAllBlobMetadata,
AtmosStorageAsyncClient client) {
this.objectMD5 = objectMD5;
this.getAllBlobMetadata = getAllBlobMetadata;
@ -89,9 +90,16 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
future.addListener(new Runnable() {
public void run() {
try {
if (Arrays.equals(toSearch, future.get().getContentMetadata().getContentMD5())) {
AtmosObject object = future.get();
checkNotNull(object.getSystemMetadata(), object + " has no content metadata");
if (object.getSystemMetadata().getContentMD5() != null) {
if (Arrays.equals(toSearch, object.getSystemMetadata().getContentMD5())) {
queue.put(true);
}
} else {
logger.debug("object %s has no content md5", object.getSystemMetadata()
.getObjectID());
}
} catch (InterruptedException e) {
Throwables.propagate(e);
} catch (ExecutionException e) {
@ -107,7 +115,7 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
throw new BlobRuntimeException(String.format("searching for md5 in container %s: %s",
containerName, exceptions));
try {
return queue.poll(1, TimeUnit.MICROSECONDS);
return queue.poll(1, TimeUnit.MICROSECONDS) != null;
} catch (InterruptedException e) {
Throwables.propagate(e);
return false;

132
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/RecursiveRemove.java
View File

@ -1,132 +0,0 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.atmosonline.saas.blobstore.strategy;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient;
import org.jclouds.atmosonline.saas.AtmosStorageClient;
import org.jclouds.atmosonline.saas.domain.DirectoryEntry;
import org.jclouds.atmosonline.saas.domain.FileType;
import org.jclouds.blobstore.internal.BlobRuntimeException;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ClearContainerStrategy;
import org.jclouds.blobstore.strategy.ClearListStrategy;
import org.jclouds.logging.Logger;
import org.jclouds.util.Utils;
import com.google.common.base.Function;
import com.google.common.base.Supplier;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.inject.Inject;
/**
* Recursively remove a path.
*
* @author Adrian Cole
*/
@Singleton
public class RecursiveRemove implements ClearListStrategy, ClearContainerStrategy {
/**
* maximum duration of an blob Request
*/
@Inject(optional = true)
@Named(Constants.PROPERTY_HTTP_REQUEST_TIMEOUT)
protected Long maxTime;
protected final AtmosStorageAsyncClient async;
protected final AtmosStorageClient sync;
private final ExecutorService userExecutor;
@Resource
protected Logger logger = Logger.NULL;
@Inject
public RecursiveRemove(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
AtmosStorageAsyncClient connection, AtmosStorageClient sync) {
this.async = connection;
this.sync = sync;
this.userExecutor = userExecutor;
}
public void execute(String containerName) {
logger.debug("clearing container ", containerName);
execute(containerName, new ListContainerOptions().recursive());
logger.trace("cleared container " + containerName);
}
private ListenableFuture<Void> rm(final String fullPath, FileType type, boolean recursive) {
Map<String, ListenableFuture<?>> responses = Maps.newHashMap();
if ((type == FileType.DIRECTORY) && recursive) {
for (DirectoryEntry child : sync.listDirectory(fullPath)) {
responses.put(fullPath + "/" + child.getObjectName(), rm(fullPath + "/"
+ child.getObjectName(), child.getType(), true));
}
}
Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
String.format("deleting from path: %s", fullPath));
if (exceptions.size() > 0)
throw new BlobRuntimeException(String.format("deleting from path %s: %s", fullPath,
exceptions));
return Futures.compose(async.deletePath(fullPath), new Function<Void, Void>() {
public Void apply(Void from) {
try {
if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
public Boolean get() {
return !sync.pathExists(fullPath);
}
}, maxTime != null ? maxTime : 1000)) {
throw new IllegalStateException(fullPath + " still exists after deleting!");
}
return null;
} catch (InterruptedException e) {
throw new IllegalStateException(fullPath + " still exists after deleting!", e);
}
}
});
}
public void execute(String path, ListContainerOptions options) {
if (options.getDir() != null)
path += "/" + options.getDir();
Map<String, ListenableFuture<?>> responses = Maps.newHashMap();
for (DirectoryEntry md : sync.listDirectory(path)) {
responses.put(path + "/" + md.getObjectName(), rm(path + "/" + md.getObjectName(), md
.getType(), options.isRecursive()));
}
Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
String.format("deleting from path: %s", path));
if (exceptions.size() > 0)
throw new BlobRuntimeException(String
.format("deleting from path %s: %s", path, exceptions));
}
}

6
atmos/src/main/java/org/jclouds/atmosonline/saas/domain/DirectoryEntry.java
View File

@ -88,4 +88,10 @@ public class DirectoryEntry implements Comparable<DirectoryEntry> {
return false;
return true;
}
@Override
public String toString() {
return "DirectoryEntry [type=" + type + ", objectid=" + objectid + ", objname=" + objname
+ "]";
}
}

15
atmos/src/main/java/org/jclouds/atmosonline/saas/domain/SystemMetadata.java
View File

@ -20,6 +20,8 @@ package org.jclouds.atmosonline.saas.domain;
import java.util.Date;
import javax.annotation.Nullable;
/**
* Metadata of a Atmos Online object
*
@ -36,10 +38,12 @@ public class SystemMetadata extends DirectoryEntry {
private final String policyname;
private final long size;
private final String uid;
private final byte[] contentmd5;
public SystemMetadata(Date atime, Date ctime, String gid, Date itime, Date mtime, int nlink,
public SystemMetadata(@Nullable byte [] contentmd5, Date atime, Date ctime, String gid, Date itime, Date mtime, int nlink,
String objectid, String objname, String policyname, long size, FileType type, String uid) {
super(objectid, type, objname);
this.contentmd5 = contentmd5;
this.atime = atime;
this.ctime = ctime;
this.gid = gid;
@ -87,6 +91,10 @@ public class SystemMetadata extends DirectoryEntry {
return uid;
}
public byte[] getContentMD5() {
return contentmd5;
}
@Override
public int hashCode() {
final int prime = 31;
@ -154,4 +162,9 @@ public class SystemMetadata extends DirectoryEntry {
return true;
}
@Override
public String toString() {
return "[type=" + getType() + ", id=" + getObjectID() + ", name=" + getObjectName() + "]";
}
}

21
atmos/src/main/java/org/jclouds/atmosonline/saas/domain/internal/AtmosObjectImpl.java
View File

@ -78,6 +78,27 @@ public class AtmosObjectImpl extends BasePayloadEnclosingImpl implements AtmosOb
this.allHeaders = checkNotNull(allHeaders, "allHeaders");
}
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
return systemMetadata != null ? systemMetadata.hashCode() : super.hashCode();
}
/**
* {@inheritDoc}
*/
@Override
public boolean equals(Object obj) {
return systemMetadata != null ? systemMetadata.equals(obj) : super.equals(obj);
}
@Override
public String toString() {
return "[metadata=" + systemMetadata + "]";
}
/**
* {@inheritDoc}
*/

11
atmos/src/main/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeaders.java
View File

@ -29,6 +29,7 @@ import org.jclouds.atmosonline.saas.domain.FileType;
import org.jclouds.atmosonline.saas.domain.SystemMetadata;
import org.jclouds.atmosonline.saas.reference.AtmosStorageHeaders;
import org.jclouds.date.DateService;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpResponse;
import com.google.common.base.Function;
@ -40,10 +41,13 @@ import com.google.common.collect.Maps;
@Singleton
public class ParseSystemMetadataFromHeaders implements Function<HttpResponse, SystemMetadata> {
private final DateService dateService;
private final EncryptionService encryptionService;
@Inject
public ParseSystemMetadataFromHeaders(DateService dateService) {
public ParseSystemMetadataFromHeaders(DateService dateService,
EncryptionService encryptionService) {
this.dateService = dateService;
this.encryptionService = encryptionService;
}
public SystemMetadata apply(HttpResponse from) {
@ -56,8 +60,9 @@ public class ParseSystemMetadataFromHeaders implements Function<HttpResponse, Sy
metaMap.put(entrySplit[0], entrySplit[1]);
}
assert metaMap.size() >= 12 : String.format("Should be 12 entries in %s", metaMap);
return new SystemMetadata(dateService.iso8601SecondsDateParse(checkNotNull(metaMap
byte[] md5 = metaMap.containsKey("content-md5") ? encryptionService.fromHexString(metaMap
.get("content-md5")) : null;
return new SystemMetadata(md5, dateService.iso8601SecondsDateParse(checkNotNull(metaMap
.get("atime"), "atime")), dateService.iso8601SecondsDateParse(checkNotNull(metaMap
.get("ctime"), "ctime")), checkNotNull(metaMap.get("gid"), "gid"), dateService
.iso8601SecondsDateParse(checkNotNull(metaMap.get("itime"), "itime")), dateService

8
atmos/src/main/java/org/jclouds/atmosonline/saas/handlers/ParseAtmosStorageErrorFromXmlContent.java
View File

@ -60,8 +60,8 @@ public class ParseAtmosStorageErrorFromXmlContent implements HttpErrorHandler {
this.utils = utils;
}
public static final Pattern CONTAINER_PATH = Pattern.compile("^/rest/namespace/?([^/]+)[/]?$");
public static final Pattern CONTAINER_KEY_PATH = Pattern
public static final Pattern DIRECTORY_PATH = Pattern.compile("^/rest/namespace/?([^/]+)/$");
public static final Pattern DIRECTORY_KEY_PATH = Pattern
.compile("^/rest/namespace/?([^/]+)/(.*)");
public void handleError(HttpCommand command, HttpResponse response) {
@ -84,11 +84,11 @@ public class ParseAtmosStorageErrorFromXmlContent implements HttpErrorHandler {
"%s -> %s", command.getRequest().getRequestLine(), response
.getStatusLine());
String path = command.getRequest().getEndpoint().getPath();
Matcher matcher = CONTAINER_PATH.matcher(path);
Matcher matcher = DIRECTORY_PATH.matcher(path);
if (matcher.find()) {
exception = new ContainerNotFoundException(matcher.group(1), message);
} else {
matcher = CONTAINER_KEY_PATH.matcher(path);
matcher = DIRECTORY_KEY_PATH.matcher(path);
if (matcher.find()) {
exception = new KeyNotFoundException(matcher.group(1), matcher.group(2),
message);

2
atmos/src/main/java/org/jclouds/atmosonline/saas/options/ListOptions.java
View File

@ -51,7 +51,7 @@ public class ListOptions extends BaseHttpRequestOptions {
*/
public ListOptions limit(int maxresults) {
checkState(maxresults >= 0, "maxresults must be >= 0");
checkState(maxresults <= 10000, "maxresults must be <= 5000");
checkState(maxresults <= 10000, "maxresults must be <= 10000");
headers.put("x-emc-limit", Integer.toString(maxresults));
return this;
}

39
atmos/src/main/java/org/jclouds/atmosonline/saas/util/AtmosStorageUtils.java
View File

@ -24,13 +24,20 @@ import java.io.InputStream;
import javax.inject.Inject;
import javax.inject.Provider;
import org.jclouds.atmosonline.saas.AtmosStorageClient;
import org.jclouds.atmosonline.saas.blobstore.functions.BlobToObject;
import org.jclouds.atmosonline.saas.domain.AtmosStorageError;
import org.jclouds.atmosonline.saas.filters.SignRequest;
import org.jclouds.atmosonline.saas.xml.ErrorHandler;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpCommand;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ParseSax;
import org.jclouds.util.Utils;
import com.google.common.base.Supplier;
/**
* Encryption, Hashing, and IO Utilities needed to sign and verify Atmos Storage requests and
@ -60,6 +67,32 @@ public class AtmosStorageUtils {
}
public static String putBlob(final AtmosStorageClient sync, EncryptionService encryptionService,
BlobToObject blob2Object, String container, Blob blob) {
final String path = container + "/" + blob.getMetadata().getName();
deleteAndEnsureGone(sync, path);
if (blob.getMetadata().getContentMD5() != null)
blob.getMetadata().getUserMetadata().put("content-md5",
encryptionService.toHexString(blob.getMetadata().getContentMD5()));
sync.createFile(container, blob2Object.apply(blob));
return path;
}
public static void deleteAndEnsureGone(final AtmosStorageClient sync, final String path) {
try {
if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
public Boolean get() {
sync.deletePath(path);
return !sync.pathExists(path);
}
}, 3000)) {
throw new IllegalStateException(path + " still exists after deleting!");
}
} catch (InterruptedException e) {
new IllegalStateException(path + " interrupted during deletion!", e);
}
}
public AtmosStorageError parseAtmosStorageErrorFromContent(HttpCommand command,
HttpResponse response, String content) throws HttpException {
return parseAtmosStorageErrorFromContent(command, response, new ByteArrayInputStream(content
@ -69,9 +102,9 @@ public class AtmosStorageUtils {
public static String adjustContainerIfDirOptionPresent(String container,
org.jclouds.blobstore.options.ListContainerOptions options) {
if (options != org.jclouds.blobstore.options.ListContainerOptions.NONE) {
if (options.isRecursive()) {
throw new UnsupportedOperationException("recursive not currently supported in emcsaas");
}
// if (options.isRecursive()) {
// throw new UnsupportedOperationException("recursive not currently supported in emcsaas");
// }
if (options.getDir() != null) {
container = container + "/" + options.getDir();
}

26
atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java
View File

@ -26,22 +26,18 @@ import java.lang.reflect.UndeclaredThrowableException;
import java.net.URI;
import java.security.SecureRandom;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeoutException;
import org.jclouds.atmosonline.saas.blobstore.strategy.RecursiveRemove;
import org.jclouds.atmosonline.saas.domain.AtmosObject;
import org.jclouds.atmosonline.saas.domain.BoundedSet;
import org.jclouds.atmosonline.saas.domain.DirectoryEntry;
import org.jclouds.atmosonline.saas.domain.FileType;
import org.jclouds.atmosonline.saas.domain.SystemMetadata;
import org.jclouds.atmosonline.saas.options.ListOptions;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.KeyAlreadyExistsException;
import org.jclouds.blobstore.KeyNotFoundException;
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
import org.jclouds.blobstore.strategy.ClearContainerStrategy;
import org.jclouds.http.HttpResponseException;
import org.jclouds.http.Payloads;
import org.jclouds.http.payloads.InputStreamPayload;
@ -107,7 +103,7 @@ public class AtmosStorageClientLiveTest {
private static final int INCONSISTENCY_WINDOW = 5000;
protected AtmosStorageClient connection;
private String containerPrefix = BaseBlobStoreIntegrationTest.CONTAINER_PREFIX+"live";
private String containerPrefix = BaseBlobStoreIntegrationTest.CONTAINER_PREFIX + "live";
URI container1;
URI container2;
@ -116,23 +112,15 @@ public class AtmosStorageClientLiveTest {
public void setupClient() throws InterruptedException, ExecutionException, TimeoutException {
String uid = checkNotNull(System.getProperty("jclouds.test.user"), "jclouds.test.user");
String key = checkNotNull(System.getProperty("jclouds.test.key"), "jclouds.test.key");
RestContext<AtmosStorageAsyncClient, AtmosStorageClient> context = new AtmosStorageContextBuilder(
BlobStoreContext blobStoreContext = new AtmosStorageContextBuilder(
new AtmosStoragePropertiesBuilder(uid, key).build()).withModules(
new Log4JLoggingModule()).buildContext();
ExecutorService service = Executors.newCachedThreadPool();
new Log4JLoggingModule()).buildBlobStoreContext();
RestContext<AtmosStorageAsyncClient, AtmosStorageClient> context = blobStoreContext
.getProviderSpecificContext();
connection = context.getApi();
ClearContainerStrategy clearer = new RecursiveRemove(service, context.getAsyncApi(),
connection);
for (DirectoryEntry entry : connection.listDirectories()) {
try {
if (entry.getObjectName().startsWith(containerPrefix)) {
clearer.execute(entry.getObjectName());
deleteConfirmed(entry.getObjectName());
}
} catch (ContainerNotFoundException e) {
if (entry.getType() != FileType.DIRECTORY)
throw e;
blobStoreContext.getBlobStore().deleteContainer(entry.getObjectName());
}
}
}

6
atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientTest.java
View File

@ -39,7 +39,6 @@ import org.jclouds.atmosonline.saas.options.ListOptions;
import org.jclouds.atmosonline.saas.reference.AtmosStorageConstants;
import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
import org.jclouds.blobstore.config.BlobStoreObjectModule;
import org.jclouds.blobstore.functions.ReturnNullOnKeyNotFound;
import org.jclouds.blobstore.functions.ReturnVoidOnNotFoundOr404;
import org.jclouds.blobstore.functions.ThrowContainerNotFoundOn404;
import org.jclouds.blobstore.functions.ThrowKeyNotFoundOn404;
@ -51,6 +50,7 @@ import org.jclouds.http.options.GetOptions;
import org.jclouds.logging.Logger;
import org.jclouds.logging.Logger.LoggerFactory;
import org.jclouds.rest.RestClientTest;
import org.jclouds.rest.functions.ReturnNullOnResourceNotFound;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.util.Jsr330;
@ -221,7 +221,7 @@ public class AtmosStorageClientTest extends RestClientTest<AtmosStorageAsyncClie
assertResponseParserClassEquals(method, httpMethod,
ParseObjectFromHeadersAndHttpContent.class);
assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, ReturnNullOnKeyNotFound.class);
assertExceptionParserClassEquals(method, ReturnNullOnResourceNotFound.class);
checkFilters(httpMethod);
}
@ -238,7 +238,7 @@ public class AtmosStorageClientTest extends RestClientTest<AtmosStorageAsyncClie
assertResponseParserClassEquals(method, httpMethod, ParseSystemMetadataFromHeaders.class);
assertSaxResponseParserClassEquals(method, null);
assertExceptionParserClassEquals(method, ReturnNullOnKeyNotFound.class);
assertExceptionParserClassEquals(method, ReturnNullOnResourceNotFound.class);
checkFilters(httpMethod);
}

13
atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageContainerIntegrationTest.java
View File

@ -30,16 +30,9 @@ import org.testng.annotations.Test;
public class AtmosStorageContainerIntegrationTest extends BaseContainerIntegrationTest {
@Override
@Test(enabled = false)
// some reason this fails on the stub.
public void testClearWhenContentsUnderPath() throws InterruptedException {
super.testClearWhenContentsUnderPath();
}
@Override
@Test(enabled = false)
public void testDirectory() throws InterruptedException, UnsupportedEncodingException {
// TODO
public void testListContainerMaxResults() throws InterruptedException,
UnsupportedEncodingException {
// Not currently working
}
}

5
atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageInputStreamMapIntegrationTest.java
View File

@ -82,9 +82,4 @@ public class AtmosStorageInputStreamMapIntegrationTest extends BaseInputStreamMa
// TODO not reliable NPE
}
@Override
protected int maxList() {
return 0;
}
}

20
atmos/src/test/java/org/jclouds/atmosonline/saas/blobstore/integration/AtmosStorageMapIntegrationTest.java
View File

@ -18,9 +18,6 @@
*/
package org.jclouds.atmosonline.saas.blobstore.integration;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import org.jclouds.blobstore.integration.internal.BaseBlobMapIntegrationTest;
import org.testng.annotations.Test;
@ -30,21 +27,4 @@ import org.testng.annotations.Test;
@Test(groups = { "integration", "live" }, testName = "emcsaas.AtmosStorageMapIntegrationTest")
public class AtmosStorageMapIntegrationTest extends BaseBlobMapIntegrationTest {
@Override
@Test(enabled = false)
public void testContains() throws InterruptedException, ExecutionException, TimeoutException {
// TODO not reliable
}
@Override
@Test(enabled = false)
public void testValues() {
// TODO not reliable KeyAlreadyExistsException@AtmosBlobStore.java:213
}
@Override
protected int maxList() {
return 0;
}
}

7
atmos/src/test/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeadersTest.java
View File

@ -23,6 +23,7 @@ import static org.testng.Assert.assertEquals;
import org.jclouds.atmosonline.saas.domain.FileType;
import org.jclouds.atmosonline.saas.domain.SystemMetadata;
import org.jclouds.date.DateService;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpResponse;
import org.testng.annotations.Test;
@ -42,17 +43,19 @@ public class ParseSystemMetadataFromHeadersTest {
ParseSystemMetadataFromHeaders parser = injector
.getInstance(ParseSystemMetadataFromHeaders.class);
DateService dateService = injector.getInstance(DateService.class);
EncryptionService encryptionService = injector.getInstance(EncryptionService.class);
HttpResponse response = new HttpResponse();
response
.getHeaders()
.put(
"x-emc-meta",
"atime=2009-10-12T16:09:42Z, mtime=2009-10-19T04:37:00Z,"
"content-md5=1f3870be274f6c49b3e31a0c6728957f, atime=2009-10-12T16:09:42Z, mtime=2009-10-19T04:37:00Z,"
+ " ctime=2009-10-19T04:37:00Z, itime=2009-10-12T16:09:42Z, type=directory, uid=root, "
+ "gid=rootr, objectid=4980cdb2b010109b04a44f7bb83f5f04ad354c638ae5, "
+ "objname=e913e09366364e9ba384b8fead643d43, size=4096, nlink=1, policyname=default");
SystemMetadata expected = new SystemMetadata(
SystemMetadata expected = new SystemMetadata(encryptionService
.fromHexString("1f3870be274f6c49b3e31a0c6728957f"),
dateService.iso8601SecondsDateParse("2009-10-12T16:09:42Z"), dateService
.iso8601SecondsDateParse("2009-10-19T04:37:00Z"), "rootr", dateService

15
atmos/src/test/java/org/jclouds/atmosonline/saas/internal/StubAtmosStorageAsyncClient.java
View File

@ -135,15 +135,17 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
}
public ListenableFuture<Void> deletePath(String path) {
if (path.indexOf('/') == -1)
return compose(blobStore.deleteContainerImpl(path), new Function<Boolean, Void>() {
if (path.indexOf('/') == path.length() - 1) {
// chop off the trailing slash
return compose(blobStore.deleteContainerImpl(path.substring(0, path.length() - 1)),
new Function<Boolean, Void>() {
public Void apply(Boolean from) {
return null;
}
});
else {
} else {
String container = path.substring(0, path.indexOf('/'));
path = path.substring(path.indexOf('/') + 1);
return blobStore.removeBlob(container, path);
@ -205,9 +207,10 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
}
public ListenableFuture<Boolean> pathExists(final String path) {
if (path.indexOf('/') == -1 )
return blobStore.containerExists(path);
else {
if (path.indexOf('/') == path.length() - 1) {
// chop off the trailing slash
return blobStore.containerExists(path.substring(0, path.length() - 1));
} else {
String container = path.substring(0, path.indexOf('/'));
String blobName = path.substring(path.indexOf('/') + 1);
try {

69
atmos/src/test/resources/log4j.xml
View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Cloud Conscious, LLC.
<info@cloudconscious.com>
====================================================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0 Unless required by
applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions
and limitations under the License.
====================================================================
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<!--
For more configuration infromation and examples see the Apache
Log4j website: http://logging.apache.org/log4j/
-->
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
debug="false">
@ -46,9 +71,17 @@
</layout>
</appender>
<appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="FILE" />
</appender>
<appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="WIREFILE" />
</appender>
<!-- A time/date based rolling appender -->
<appender name="COMPUTEFILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="target/test-data/jclouds-compute.log" />
<appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="target/test-data/jclouds-blobstore.log" />
<param name="Append" value="true" />
<!-- Rollover at midnight each day -->
@ -61,28 +94,23 @@
<param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
<!--
The full pattern: Date MS Priority [Category] (Thread:NDC) Message\n
<param name="ConversionPattern" value="%d %-5r %-5p [%c] (%t:%x)
%m%n"/>
The full pattern: Date MS Priority [Category]
(Thread:NDC) Message\n <param name="ConversionPattern"
value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
-->
</layout>
</appender>
<appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="COMPUTEFILE" />
<appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="BLOBSTOREFILE" />
</appender>
<appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="FILE" />
</appender>
<appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="WIREFILE" />
</appender>
<!-- ================ -->
<!-- Limit categories -->
<!-- ================ -->
<category name="jclouds.blobstore">
<priority value="TRACE" />
<appender-ref ref="ASYNCBLOBSTORE" />
</category>
<category name="org.jclouds">
<priority value="DEBUG" />
@ -92,20 +120,15 @@
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
</category>
</category><!--
<category name="jclouds.wire">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
</category><!--
<category name="jclouds.signature">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
</category>
--><!-- ================ -->
--><!-- ======================= -->
<!-- Setup the Root category -->
<!-- ======================= -->

4
aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3BlobStore.java
View File

@ -143,14 +143,14 @@ public class S3BlobStore extends BaseBlobStore {
*/
@Override
public void deleteContainer(String container) {
deleteAndEnsurePathGone(container);
clearAndDeleteContainer(container);
}
/**
* This implementation invokes {@link #clearContainer} then {@link S3Client#deleteBucketIfEmpty}
* until it is true.
*/
public void deleteAndEnsurePathGone(final String container) {
public void clearAndDeleteContainer(final String container) {
try {
if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
public Boolean get() {

5
aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3BlobMapIntegrationTest.java
View File

@ -27,9 +27,4 @@ import org.testng.annotations.Test;
@Test(groups = { "integration", "live" }, testName = "s3.S3BlobMapIntegrationTest")
public class S3BlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
@Override
protected int maxList() {
return 1000;
}
}

5
aws/core/src/test/java/org/jclouds/aws/s3/blobstore/integration/S3InputStreamMapIntegrationTest.java
View File

@ -26,8 +26,5 @@ import org.testng.annotations.Test;
*/
@Test(groups = { "integration", "live" }, testName = "s3.S3InputStreamMapIntegrationTest")
public class S3InputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
@Override
protected int maxList() {
return 1000;
}
}

6
azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/strategy/FindMD5InBlobProperties.java
View File

@ -30,7 +30,7 @@ import org.jclouds.blobstore.functions.ObjectMD5;
import org.jclouds.blobstore.internal.BlobRuntimeException;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
import org.jclouds.blobstore.strategy.ListBlobsInContainer;
import com.google.common.base.Throwables;
@ -43,12 +43,12 @@ import com.google.common.base.Throwables;
public class FindMD5InBlobProperties implements ContainsValueInListStrategy {
protected final ObjectMD5 objectMD5;
protected final ListBlobMetadataStrategy getAllBlobMetadata;
protected final ListBlobsInContainer getAllBlobMetadata;
private final AzureBlobClient client;
@Inject
private FindMD5InBlobProperties(ObjectMD5 objectMD5,
ListBlobMetadataStrategy getAllBlobMetadata, AzureBlobClient client) {
ListBlobsInContainer getAllBlobMetadata, AzureBlobClient client) {
this.objectMD5 = objectMD5;
this.getAllBlobMetadata = getAllBlobMetadata;
this.client = client;

74
azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobClientLiveTest.java
View File

@ -18,6 +18,8 @@
*/
package org.jclouds.azure.storage.blob;
import static org.jclouds.azure.storage.blob.options.CreateContainerOptions.Builder.withMetadata;
import static org.jclouds.azure.storage.blob.options.CreateContainerOptions.Builder.withPublicAcl;
import static org.jclouds.azure.storage.options.ListOptions.Builder.includeMetadata;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
@ -33,7 +35,6 @@ import org.jclouds.azure.storage.blob.domain.AzureBlob;
import org.jclouds.azure.storage.blob.domain.BlobProperties;
import org.jclouds.azure.storage.blob.domain.ContainerProperties;
import org.jclouds.azure.storage.blob.domain.ListBlobsResponse;
import org.jclouds.azure.storage.blob.options.CreateContainerOptions;
import org.jclouds.azure.storage.blob.options.ListBlobsOptions;
import org.jclouds.azure.storage.domain.BoundedSet;
import org.jclouds.azure.storage.options.ListOptions;
@ -59,7 +60,7 @@ import com.google.inject.internal.Iterables;
@Test(groups = "live", sequential = true, testName = "azureblob.AzureBlobClientLiveTest")
public class AzureBlobClientLiveTest {
protected AzureBlobClient connection;
protected AzureBlobClient client;
private String containerPrefix = System.getProperty("user.name") + "-azureblob";
private EncryptionService encryptionService = new JCEEncryptionService();
@ -68,13 +69,13 @@ public class AzureBlobClientLiveTest {
public void setupClient() {
account = System.getProperty("jclouds.test.user");
String key = System.getProperty("jclouds.test.key");
connection = (AzureBlobClient) AzureBlobContextFactory.createContext(account, key,
client = (AzureBlobClient) AzureBlobContextFactory.createContext(account, key,
new Log4JLoggingModule()).getProviderSpecificContext().getApi();
}
@Test
public void testListContainers() throws Exception {
Set<ContainerProperties> response = connection.listContainers();
Set<ContainerProperties> response = client.listContainers();
assert null != response;
long initialContainerCount = response.size();
assertTrue(initialContainerCount >= 0);
@ -91,8 +92,8 @@ public class AzureBlobClientLiveTest {
while (!created) {
privateContainer = containerPrefix + new SecureRandom().nextInt();
try {
created = connection.createContainer(privateContainer, CreateContainerOptions.Builder
.withMetadata(ImmutableMultimap.of("foo", "bar")));
created = client.createContainer(privateContainer, withMetadata(ImmutableMultimap
.of("foo", "bar")));
} catch (UndeclaredThrowableException e) {
HttpResponseException htpe = (HttpResponseException) e.getCause().getCause();
if (htpe.getResponse().getStatusCode() == 409)
@ -100,11 +101,11 @@ public class AzureBlobClientLiveTest {
throw e;
}
}
Set<ContainerProperties> response = connection.listContainers(includeMetadata());
Set<ContainerProperties> response = client.listContainers(includeMetadata());
assert null != response;
long containerCount = response.size();
assertTrue(containerCount >= 1);
ListBlobsResponse list = connection.listBlobs(privateContainer);
ListBlobsResponse list = client.listBlobs(privateContainer);
assertEquals(list.getUrl(), URI.create(String.format("https://%s.blob.core.windows.net/%s",
account, privateContainer)));
// TODO ... check to see the container actually exists
@ -116,8 +117,7 @@ public class AzureBlobClientLiveTest {
while (!created) {
publicContainer = containerPrefix + new SecureRandom().nextInt();
try {
created = connection.createContainer(publicContainer, CreateContainerOptions.Builder
.withPublicAcl());
created = client.createContainer(publicContainer, withPublicAcl());
} catch (UndeclaredThrowableException e) {
HttpResponseException htpe = (HttpResponseException) e.getCause().getCause();
if (htpe.getResponse().getStatusCode() == 409)
@ -134,7 +134,7 @@ public class AzureBlobClientLiveTest {
@Test(timeOut = 5 * 60 * 1000)
public void testCreatePublicRootContainer() throws Exception {
try {
connection.deleteRootContainer();
client.deleteRootContainer();
} catch (ContainerNotFoundException e) {
Thread.sleep(5000);
} catch (AzureStorageResponseException htpe) {
@ -148,7 +148,7 @@ public class AzureBlobClientLiveTest {
boolean created = false;
while (!created) {
try {
created = connection.createRootContainer();
created = client.createRootContainer();
} catch (AzureStorageResponseException htpe) {
if (htpe.getResponse().getStatusCode() == 409) {// TODO look for specific message
Thread.sleep(5000);
@ -158,7 +158,7 @@ public class AzureBlobClientLiveTest {
}
}
}
ListBlobsResponse list = connection.listBlobs();
ListBlobsResponse list = client.listBlobs();
assertEquals(list.getUrl(), URI.create(String.format(
"https://%s.blob.core.windows.net/%%24root", account)));
}
@ -166,7 +166,7 @@ public class AzureBlobClientLiveTest {
@Test
public void testListContainersWithOptions() throws Exception {
BoundedSet<ContainerProperties> response = connection.listContainers(ListOptions.Builder
BoundedSet<ContainerProperties> response = client.listContainers(ListOptions.Builder
.prefix(privateContainer).maxResults(1).includeMetadata());
assert null != response;
long initialContainerCount = response.size();
@ -177,7 +177,7 @@ public class AzureBlobClientLiveTest {
@Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testCreatePublicRootContainer" })
public void testDeleteRootContainer() throws Exception {
connection.deleteRootContainer();
client.deleteRootContainer();
// TODO loop for up to 30 seconds checking if they are really gone
}
@ -186,19 +186,19 @@ public class AzureBlobClientLiveTest {
public void testListOwnedContainers() throws Exception {
// Test default listing
Set<ContainerProperties> response = connection.listContainers();
Set<ContainerProperties> response = client.listContainers();
// assertEquals(response.size(), initialContainerCount + 2);// if the containers already
// exist, this will fail
// Test listing with options
response = connection.listContainers(ListOptions.Builder.prefix(
response = client.listContainers(ListOptions.Builder.prefix(
privateContainer.substring(0, privateContainer.length() - 1)).maxResults(1)
.includeMetadata());
assertEquals(response.size(), 1);
assertEquals(Iterables.getOnlyElement(response).getName(), privateContainer);
assertEquals(Iterables.getOnlyElement(response).getMetadata(), ImmutableMap.of("foo", "bar"));
response = connection.listContainers(ListOptions.Builder.prefix(publicContainer)
response = client.listContainers(ListOptions.Builder.prefix(publicContainer)
.maxResults(1));
assertEquals(response.size(), 1);
assertEquals(Iterables.getOnlyElement(response).getName(), publicContainer);
@ -207,14 +207,14 @@ public class AzureBlobClientLiveTest {
@Test
public void testDeleteOneContainer() throws Exception {
connection.deleteContainer("does-not-exist");
client.deleteContainer("does-not-exist");
}
@Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testListOwnedContainers",
"testObjectOperations" })
public void testDeleteContainer() throws Exception {
connection.deleteContainer(privateContainer);
connection.deleteContainer(publicContainer);
client.deleteContainer(privateContainer);
client.deleteContainer(publicContainer);
// TODO loop for up to 30 seconds checking if they are really gone
}
@ -224,7 +224,7 @@ public class AzureBlobClientLiveTest {
String data = "Here is my data";
// Test PUT with string data, ETag hash, and a piece of metadata
AzureBlob object = connection.newBlob();
AzureBlob object = client.newBlob();
object.getProperties().setName("object");
object.setPayload(data);
object.setContentLength(data.length());
@ -232,15 +232,15 @@ public class AzureBlobClientLiveTest {
object.getProperties().setContentType("text/plain");
object.getProperties().getMetadata().put("mykey", "metadata-value");
byte[] md5 = object.getProperties().getContentMD5();
String newEtag = connection.putBlob(privateContainer, object);
String newEtag = client.putBlob(privateContainer, object);
assertEquals(encryptionService.toHexString(md5), encryptionService.toHexString(object
.getProperties().getContentMD5()));
// Test HEAD of missing object
assert connection.getBlobProperties(privateContainer, "non-existent-object") == null;
assert client.getBlobProperties(privateContainer, "non-existent-object") == null;
// Test HEAD of object
BlobProperties metadata = connection.getBlobProperties(privateContainer, object
BlobProperties metadata = client.getBlobProperties(privateContainer, object
.getProperties().getName());
// TODO assertEquals(metadata.getName(), object.getProperties().getName());
// we can't check this while hacking around lack of content-md5, as GET of the first byte will
@ -261,14 +261,14 @@ public class AzureBlobClientLiveTest {
// Multimap<String, String> userMetadata = LinkedHashMultimap.create();
// userMetadata.put("New-Metadata-1", "value-1");
// userMetadata.put("New-Metadata-2", "value-2");
// assertTrue(connection.setBlobProperties(privateContainer, object.getProperties().getName(),
// assertTrue(client.setBlobProperties(privateContainer, object.getProperties().getName(),
// userMetadata));
// Test GET of missing object
assert connection.getBlob(privateContainer, "non-existent-object") == null;
assert client.getBlob(privateContainer, "non-existent-object") == null;
// Test GET of object (including updated metadata)
AzureBlob getBlob = connection.getBlob(privateContainer, object.getProperties().getName());
AzureBlob getBlob = client.getBlob(privateContainer, object.getProperties().getName());
assertEquals(Utils.toStringAndClose(getBlob.getContent()), data);
// TODO assertEquals(getBlob.getName(), object.getProperties().getName());
assertEquals(getBlob.getContentLength(), new Long(data.length()));
@ -288,7 +288,7 @@ public class AzureBlobClientLiveTest {
assertEquals(metadata.getMetadata().get("mykey"), "metadata-value");
// test listing
ListBlobsResponse response = connection.listBlobs(privateContainer, ListBlobsOptions.Builder
ListBlobsResponse response = client.listBlobs(privateContainer, ListBlobsOptions.Builder
.prefix(
object.getProperties().getName().substring(0,
object.getProperties().getName().length() - 1)).maxResults(1)
@ -303,25 +303,25 @@ public class AzureBlobClientLiveTest {
String incorrectEtag = "0" + correctEtag.substring(1);
object.getProperties().setETag(incorrectEtag);
try {
connection.putBlob(privateContainer, object);
client.putBlob(privateContainer, object);
} catch (Throwable e) {
assertEquals(e.getCause().getClass(), HttpResponseException.class);
assertEquals(((HttpResponseException) e.getCause()).getResponse().getStatusCode(), 422);
}
ByteArrayInputStream bais = new ByteArrayInputStream(data.getBytes("UTF-8"));
object = connection.newBlob();
object = client.newBlob();
object.getProperties().setName("chunked-object");
object.setPayload(bais);
object.setContentLength(new Long(data.getBytes().length));
newEtag = connection.putBlob(privateContainer, object);
newEtag = client.putBlob(privateContainer, object);
assertEquals(encryptionService.toHexString(md5), encryptionService.toHexString(getBlob
.getProperties().getContentMD5()));
// Test GET with options
// Non-matching ETag
try {
connection.getBlob(privateContainer, object.getProperties().getName(), GetOptions.Builder
client.getBlob(privateContainer, object.getProperties().getName(), GetOptions.Builder
.ifETagDoesntMatch(newEtag));
} catch (Exception e) {
assertEquals(e.getCause().getClass(), HttpResponseException.class);
@ -330,7 +330,7 @@ public class AzureBlobClientLiveTest {
// Matching ETag TODO this shouldn't fail!!!
try {
getBlob = connection.getBlob(privateContainer, object.getProperties().getName(),
getBlob = client.getBlob(privateContainer, object.getProperties().getName(),
GetOptions.Builder.ifETagMatches(newEtag));
assertEquals(getBlob.getProperties().getETag(), newEtag);
} catch (HttpResponseException e) {
@ -340,13 +340,13 @@ public class AzureBlobClientLiveTest {
// Range
// doesn't work per
// http://social.msdn.microsoft.com/Forums/en-US/windowsazure/thread/479fa63f-51df-4b66-96b5-33ae362747b6
// getBlob = connection
// getBlob = client
// .getBlob(privateContainer, object.getProperties().getName(),
// GetOptions.Builder.startAt(8)).get(120,
// TimeUnit.SECONDS);
// assertEquals(Utils.toStringAndClose((InputStream) getBlob.getData()), data.substring(8));
connection.deleteBlob(privateContainer, "object");
connection.deleteBlob(privateContainer, "chunked-object");
client.deleteBlob(privateContainer, "object");
client.deleteBlob(privateContainer, "chunked-object");
}
}

5
azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobInputStreamMapIntegrationTest.java
View File

@ -27,9 +27,4 @@ import org.testng.annotations.Test;
@Test(groups = { "integration", "live" }, testName = "azureblob.AzureBlobInputStreamMapIntegrationTest")
public class AzureBlobInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
@Override
protected int maxList() {
return 5000;
}
}

5
azure/src/test/java/org/jclouds/azure/storage/blob/blobstore/integration/AzureBlobMapIntegrationTest.java
View File

@ -27,9 +27,4 @@ import org.testng.annotations.Test;
@Test(groups = { "integration", "live" }, testName = "azureblob.AzureBlobMapIntegrationTest")
public class AzureBlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
@Override
protected int maxList() {
return 5000;
}
}

5
blobstore/src/main/java/org/jclouds/blobstore/BlobMap.java
View File

@ -18,10 +18,9 @@
*/
package org.jclouds.blobstore;
import javax.annotation.Nullable;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.internal.BlobMapImpl;
import org.jclouds.blobstore.options.ListContainerOptions;
import com.google.inject.ImplementedBy;
@ -38,7 +37,7 @@ public interface BlobMap extends ListableMap<String, Blob> {
Blob newBlob(String name);
public static interface Factory {
BlobMap create(String containerName, @Nullable String dir);
BlobMap create(String containerName, ListContainerOptions options);
}
}

5
blobstore/src/main/java/org/jclouds/blobstore/BlobStoreContext.java
View File

@ -20,6 +20,7 @@ package org.jclouds.blobstore;
import org.jclouds.blobstore.attr.ConsistencyModels;
import org.jclouds.blobstore.internal.BlobStoreContextImpl;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.rest.RestContext;
import com.google.inject.ImplementedBy;
@ -39,6 +40,8 @@ public interface BlobStoreContext {
*
* @param container
*/
InputStreamMap createInputStreamMap(String container, ListContainerOptions options);
InputStreamMap createInputStreamMap(String container);
/**
@ -46,6 +49,8 @@ public interface BlobStoreContext {
*
* @param container
*/
BlobMap createBlobMap(String container, ListContainerOptions options);
BlobMap createBlobMap(String container);
AsyncBlobStore getAsyncBlobStore();

4
blobstore/src/main/java/org/jclouds/blobstore/ContainerNotFoundException.java
View File

@ -18,12 +18,14 @@
*/
package org.jclouds.blobstore;
import org.jclouds.rest.ResourceNotFoundException;
/**
* Thrown when a container cannot be located.
*
* @author Adrian Cole
*/
public class ContainerNotFoundException extends RuntimeException {
public class ContainerNotFoundException extends ResourceNotFoundException {
private String container;

5
blobstore/src/main/java/org/jclouds/blobstore/InputStreamMap.java
View File

@ -22,9 +22,8 @@ import java.io.File;
import java.io.InputStream;
import java.util.Map;
import javax.annotation.Nullable;
import org.jclouds.blobstore.internal.InputStreamMapImpl;
import org.jclouds.blobstore.options.ListContainerOptions;
import com.google.inject.ImplementedBy;
@ -43,7 +42,7 @@ import com.google.inject.ImplementedBy;
@ImplementedBy(InputStreamMapImpl.class)
public interface InputStreamMap extends ListableMap<String, InputStream> {
public static interface Factory {
InputStreamMap create(String containerName, @Nullable String dir);
InputStreamMap create(String containerName, ListContainerOptions options);
}
InputStream putString(String key, String value);

16
blobstore/src/main/java/org/jclouds/blobstore/config/BlobStoreMapModule.java
View File

@ -18,7 +18,6 @@
*/
package org.jclouds.blobstore.config;
import javax.annotation.Nullable;
import javax.inject.Inject;
import org.jclouds.blobstore.BlobMap;
@ -27,10 +26,11 @@ import org.jclouds.blobstore.InputStreamMap;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.internal.BlobMapImpl;
import org.jclouds.blobstore.internal.InputStreamMapImpl;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
import org.jclouds.blobstore.strategy.PutBlobsStrategy;
import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
import com.google.inject.AbstractModule;
import com.google.inject.Scopes;
@ -62,11 +62,11 @@ public class BlobStoreMapModule extends AbstractModule {
@Inject
PutBlobsStrategy putBlobsStrategy;
@Inject
ListBlobMetadataInContainer listStrategy;
ListContainerAndRecurseThroughFolders listStrategy;
public BlobMap create(String containerName, @Nullable String dir) {
public BlobMap create(String containerName, ListContainerOptions options) {
return new BlobMapImpl(connection, getAllBlobs, containsValueStrategy, putBlobsStrategy,
listStrategy, containerName, dir);
listStrategy, containerName, options);
}
}
@ -83,11 +83,11 @@ public class BlobStoreMapModule extends AbstractModule {
@Inject
PutBlobsStrategy putBlobsStrategy;
@Inject
ListBlobMetadataInContainer listStrategy;
ListContainerAndRecurseThroughFolders listStrategy;
public InputStreamMap create(String containerName, @Nullable String dir) {
public InputStreamMap create(String containerName, ListContainerOptions options) {
return new InputStreamMapImpl(connection, blobFactory, getAllBlobs, listStrategy,
containsValueStrategy, putBlobsStrategy, containerName, dir);
containsValueStrategy, putBlobsStrategy, containerName, options);
}
}

7
blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java
View File

@ -32,6 +32,7 @@ import javax.inject.Named;
import org.jclouds.Constants;
import org.jclouds.blobstore.AsyncBlobStore;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
@ -238,8 +239,12 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore {
try {
if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
public Boolean get() {
blobUtils.clearContainer(container, recursive());
try {
clearContainer(container, recursive());
return deleteAndVerifyContainerGone(container);
} catch (ContainerNotFoundException e) {
return true;
}
}
}, 30000)) {

17
blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java
View File

@ -20,13 +20,10 @@ package org.jclouds.blobstore.internal;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.inDirectory;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
@ -37,7 +34,7 @@ import org.jclouds.blobstore.options.ListContainerOptions.ImmutableListContainer
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
import org.jclouds.blobstore.strategy.PutBlobsStrategy;
import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
@ -58,7 +55,7 @@ public abstract class BaseBlobMap<V> implements Map<String, V> {
protected final ListContainerOptions options;
protected final GetBlobsInListStrategy getAllBlobs;
protected final ContainsValueInListStrategy containsValueStrategy;
protected final ListBlobMetadataInContainer listStrategy;
protected final ListContainerAndRecurseThroughFolders listStrategy;
protected final PutBlobsStrategy putBlobsStrategy;
static class StripPath implements Function<String, String> {
@ -98,11 +95,15 @@ public abstract class BaseBlobMap<V> implements Map<String, V> {
@Inject
public BaseBlobMap(BlobStore blobstore, GetBlobsInListStrategy getAllBlobs,
ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy,
ListBlobMetadataInContainer listStrategy, String containerName, @Nullable String dir) {
ListContainerAndRecurseThroughFolders listStrategy, String containerName,
ListContainerOptions options) {
this.blobstore = checkNotNull(blobstore, "blobstore");
this.containerName = checkNotNull(containerName, "container");
this.options = new ImmutableListContainerOptions(dir != null ? inDirectory(dir)
: ListContainerOptions.NONE);
checkArgument(containerName.indexOf('/') == -1,
"please specify directory path using the option: inDirectory, not encoded in the container name");
this.options = checkNotNull(options, "options") instanceof ImmutableListContainerOptions ? options
: new ImmutableListContainerOptions(options);
String dir = options.getDir();
if (dir == null) {
prefixer = new PassThrough<String>();
pathStripper = prefixer;

9
blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java
View File

@ -24,6 +24,7 @@ import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursi
import javax.inject.Inject;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
@ -172,15 +173,19 @@ public abstract class BaseBlobStore implements BlobStore {
*/
@Override
public void deleteContainer(final String container) {
deleteAndEnsurePathGone(container);
clearAndDeleteContainer(container);
}
protected void deleteAndEnsurePathGone(final String container) {
protected void clearAndDeleteContainer(final String container) {
try {
if (!Utils.enventuallyTrue(new Supplier<Boolean>() {
public Boolean get() {
try {
clearContainer(container, recursive());
return deleteAndVerifyContainerGone(container);
} catch (ContainerNotFoundException e) {
return true;
}
}
}, 30000)) {

9
blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java
View File

@ -21,17 +21,17 @@ package org.jclouds.blobstore.internal;
import java.util.Collection;
import java.util.Map;
import javax.annotation.Nullable;
import javax.inject.Inject;
import org.jclouds.blobstore.BlobMap;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.KeyNotFoundException;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
import org.jclouds.blobstore.strategy.PutBlobsStrategy;
import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
/**
* Map representation of a live connection to a Blob Service.
@ -46,9 +46,10 @@ public class BlobMapImpl extends BaseBlobMap<Blob> implements BlobMap {
@Inject
public BlobMapImpl(BlobStore blobstore, GetBlobsInListStrategy getAllBlobs,
ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy,
ListBlobMetadataInContainer listStrategy, String containerName, @Nullable String dir) {
ListContainerAndRecurseThroughFolders listStrategy, String containerName,
ListContainerOptions options) {
super(blobstore, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy,
containerName, dir);
containerName, options);
}
@Override

24
blobstore/src/main/java/org/jclouds/blobstore/internal/BlobStoreContextImpl.java
View File

@ -29,7 +29,7 @@ import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.InputStreamMap;
import org.jclouds.blobstore.attr.ConsistencyModel;
import org.jclouds.blobstore.attr.ConsistencyModels;
import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.rest.RestContext;
/**
@ -67,18 +67,20 @@ public class BlobStoreContextImpl<X, Y> implements BlobStoreContext {
this.blobStore = checkNotNull(blobStore, "blobStore");
}
public BlobMap createBlobMap(String path) {
checkNotNull(path, "path");
String container = BlobStoreUtilsImpl.parseContainerFromPath(path);
String dir = BlobStoreUtilsImpl.parsePrefixFromPath(path);
return blobMapFactory.create(container, dir);
public BlobMap createBlobMap(String container, ListContainerOptions options) {
return blobMapFactory.create(container, options);
}
public InputStreamMap createInputStreamMap(String path) {
checkNotNull(path, "path");
String container = BlobStoreUtilsImpl.parseContainerFromPath(path);
String dir = BlobStoreUtilsImpl.parsePrefixFromPath(path);
return inputStreamMapFactory.create(container, dir);
public BlobMap createBlobMap(String container) {
return blobMapFactory.create(container, ListContainerOptions.NONE);
}
public InputStreamMap createInputStreamMap(String container, ListContainerOptions options) {
return inputStreamMapFactory.create(container, options);
}
public InputStreamMap createInputStreamMap(String container) {
return inputStreamMapFactory.create(container, ListContainerOptions.NONE);
}
public BlobStore getBlobStore() {

10
blobstore/src/main/java/org/jclouds/blobstore/internal/InputStreamMapImpl.java
View File

@ -23,17 +23,17 @@ import java.io.InputStream;
import java.util.Collection;
import java.util.Map;
import javax.annotation.Nullable;
import javax.inject.Inject;
import org.jclouds.blobstore.BlobMap;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.InputStreamMap;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
import org.jclouds.blobstore.strategy.PutBlobsStrategy;
import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
import org.jclouds.http.Payload;
import org.jclouds.http.Payloads;
import org.jclouds.http.payloads.ByteArrayPayload;
@ -60,11 +60,11 @@ public class InputStreamMapImpl extends BaseBlobMap<InputStream> implements Inpu
@Inject
public InputStreamMapImpl(BlobStore connection, Blob.Factory blobFactory,
GetBlobsInListStrategy getAllBlobs, ListBlobMetadataInContainer listStrategy,
GetBlobsInListStrategy getAllBlobs, ListContainerAndRecurseThroughFolders listStrategy,
ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy,
String containerName, @Nullable String dir) {
String containerName, ListContainerOptions options) {
super(connection, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy,
containerName, dir);
containerName, options);
}
@Override

5
blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java
View File

@ -99,6 +99,11 @@ public class ListContainerOptions extends ListOptions implements Cloneable {
return delegate.clone();
}
@Override
public String toString() {
return delegate.toString();
}
}
public static final ImmutableListContainerOptions NONE = new ImmutableListContainerOptions(

5
blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java
View File

@ -68,6 +68,11 @@ public class ListOptions implements Cloneable {
public ListOptions clone() {
return delegate.clone();
}
@Override
public String toString() {
return delegate.toString();
}
}
public static final ImmutableListOptions NONE = new ImmutableListOptions(new ListOptions());

6
blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobMetadataStrategy.java → blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java
View File

@ -20,7 +20,7 @@ package org.jclouds.blobstore.strategy;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.internal.ListBlobMetadataInContainer;
import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
import com.google.inject.ImplementedBy;
@ -29,8 +29,8 @@ import com.google.inject.ImplementedBy;
*
* @author Adrian Cole
*/
@ImplementedBy(ListBlobMetadataInContainer.class)
public interface ListBlobMetadataStrategy {
@ImplementedBy(ListContainerAndRecurseThroughFolders.class)
public interface ListBlobsInContainer {
Iterable<? extends BlobMetadata> execute(String containerName, ListContainerOptions options);

6
blobstore/src/main/java/org/jclouds/blobstore/strategy/ListMetadataStrategy.java → blobstore/src/main/java/org/jclouds/blobstore/strategy/ListContainerStrategy.java
View File

@ -20,7 +20,7 @@ package org.jclouds.blobstore.strategy;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.internal.ListAllMetadataInContainer;
import org.jclouds.blobstore.strategy.internal.ConcatenateContainerLists;
import com.google.inject.ImplementedBy;
@ -29,8 +29,8 @@ import com.google.inject.ImplementedBy;
*
* @author Adrian Cole
*/
@ImplementedBy(ListAllMetadataInContainer.class)
public interface ListMetadataStrategy {
@ImplementedBy(ConcatenateContainerLists.class)
public interface ListContainerStrategy {
Iterable<? extends StorageMetadata> execute(String containerName, ListContainerOptions options);

6
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListAllMetadataInContainer.java → blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ConcatenateContainerLists.java
View File

@ -28,7 +28,7 @@ import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.internal.BlobRuntimeException;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.options.ListContainerOptions.ImmutableListContainerOptions;
import org.jclouds.blobstore.strategy.ListMetadataStrategy;
import org.jclouds.blobstore.strategy.ListContainerStrategy;
import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
@ -41,12 +41,12 @@ import com.google.inject.Inject;
* @author Adrian Cole
*/
@Singleton
public class ListAllMetadataInContainer implements ListMetadataStrategy {
public class ConcatenateContainerLists implements ListContainerStrategy {
protected final BlobStore connection;
@Inject
public ListAllMetadataInContainer(BlobStore connection) {
public ConcatenateContainerLists(BlobStore connection) {
this.connection = connection;
}

6
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/CountBlobTypeInList.java
View File

@ -23,7 +23,7 @@ import javax.inject.Singleton;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.CountListStrategy;
import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
import org.jclouds.blobstore.strategy.ListBlobsInContainer;
import com.google.common.collect.Iterables;
@ -34,10 +34,10 @@ import com.google.common.collect.Iterables;
*/
@Singleton
public class CountBlobTypeInList implements CountListStrategy {
protected final ListBlobMetadataStrategy getAllBlobMetadata;
protected final ListBlobsInContainer getAllBlobMetadata;
@Inject
CountBlobTypeInList(ListBlobMetadataStrategy getAllBlobMetadata) {
CountBlobTypeInList(ListBlobsInContainer getAllBlobMetadata) {
this.getAllBlobMetadata = getAllBlobMetadata;
}

39
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java
View File

@ -36,6 +36,7 @@ import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.reference.BlobStoreConstants;
import org.jclouds.blobstore.strategy.ClearContainerStrategy;
import org.jclouds.blobstore.strategy.ClearListStrategy;
import org.jclouds.blobstore.strategy.ListContainerStrategy;
import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
import org.jclouds.logging.Logger;
@ -56,7 +57,7 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
@Named(BlobStoreConstants.BLOBSTORE_LOGGER)
protected Logger logger = Logger.NULL;
protected final ListAllMetadataInContainer getAllMetadata;
protected final ListContainerStrategy listContainer;
protected final BackoffLimitedRetryHandler retryHandler;
private final ExecutorService userExecutor;
@ -70,12 +71,12 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
@Inject
DeleteAllKeysInList(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
AsyncBlobStore connection, ListAllMetadataInContainer getAllMetadata,
AsyncBlobStore connection, ListContainerStrategy listContainer,
BackoffLimitedRetryHandler retryHandler) {
this.userExecutor = userExecutor;
this.connection = connection;
this.getAllMetadata = getAllMetadata;
this.listContainer = listContainer;
this.retryHandler = retryHandler;
}
@ -84,23 +85,34 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
}
public void execute(final String containerName, final ListContainerOptions options) {
String message = options.getDir() != null ? String.format("deleting from path: %s/%s",
containerName, options.getDir()) : String.format("deleting from containerName: %s",
String message = options.getDir() != null ? String.format("clearing path %s/%s",
containerName, options.getDir()) : String.format("clearing container %s",
containerName);
if (options.isRecursive())
message = message + " recursively";
Map<StorageMetadata, Exception> exceptions = Maps.newHashMap();
Iterable<? extends StorageMetadata> toDelete = getResourcesToDelete(containerName, options);
for (int i = 0; i < 3; i++) { // TODO parameterize
Map<StorageMetadata, ListenableFuture<?>> responses = Maps.newHashMap();
try {
for (StorageMetadata md : toDelete) {
for (final StorageMetadata md : toDelete) {
String fullPath = parentIsFolder(options, md) ? options.getDir() + "/"
+ md.getName() : md.getName();
switch (md.getType()) {
case BLOB:
responses.put(md, connection.removeBlob(containerName, md.getName()));
responses.put(md, connection.removeBlob(containerName, fullPath));
break;
case FOLDER:
if (options.isRecursive() && !fullPath.equals(options.getDir())) {
execute(containerName, options.clone().inDirectory(fullPath));
}
connection.deleteDirectory(containerName, fullPath);
break;
case RELATIVE_PATH:
if (options.isRecursive())
responses.put(md, connection.deleteDirectory(containerName, md.getName()));
if (options.isRecursive() && !fullPath.equals(options.getDir())) {
execute(containerName, options.clone().inDirectory(fullPath));
}
connection.deleteDirectory(containerName, md.getName());
break;
case CONTAINER:
throw new IllegalArgumentException("Container type not supported");
@ -113,17 +125,24 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
break;
}
if (exceptions.size() > 0) {
toDelete = Iterables.concat(exceptions.keySet(), toDelete);
retryHandler.imposeBackoffExponentialDelay(i + 1, message);
}
}
}
if (exceptions.size() > 0)
throw new BlobRuntimeException(String.format("error %s: %s", message, exceptions));
assert Iterables.size(toDelete) == 0 : String.format("items remaining %s: %s", message,
toDelete);
}
private boolean parentIsFolder(final ListContainerOptions options, final StorageMetadata md) {
return (options.getDir() != null && md.getName().indexOf('/') == -1);
}
private Iterable<? extends StorageMetadata> getResourcesToDelete(final String containerName,
final ListContainerOptions options) {
Iterable<? extends StorageMetadata> toDelete = Iterables.filter(getAllMetadata.execute(
Iterable<? extends StorageMetadata> toDelete = Iterables.filter(listContainer.execute(
containerName, options), new Predicate<StorageMetadata>() {
@Override

6
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FindMD5InList.java
View File

@ -28,7 +28,7 @@ import org.jclouds.blobstore.functions.ObjectMD5;
import org.jclouds.blobstore.internal.BlobRuntimeException;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
import org.jclouds.blobstore.strategy.ListBlobsInContainer;
import com.google.common.base.Throwables;
@ -41,10 +41,10 @@ import com.google.common.base.Throwables;
public class FindMD5InList implements ContainsValueInListStrategy {
protected final ObjectMD5 objectMD5;
protected final ListBlobMetadataStrategy getAllBlobMetadata;
protected final ListBlobsInContainer getAllBlobMetadata;
@Inject
private FindMD5InList(ObjectMD5 objectMD5, ListBlobMetadataStrategy getAllBlobMetadata) {
private FindMD5InList(ObjectMD5 objectMD5, ListBlobsInContainer getAllBlobMetadata) {
this.objectMD5 = objectMD5;
this.getAllBlobMetadata = getAllBlobMetadata;
}

6
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java
View File

@ -37,7 +37,7 @@ import org.jclouds.blobstore.internal.BlobRuntimeException;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.reference.BlobStoreConstants;
import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
import org.jclouds.blobstore.strategy.ListBlobsInContainer;
import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
import org.jclouds.logging.Logger;
@ -57,7 +57,7 @@ import com.google.inject.Inject;
@Singleton
public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrategy {
protected final ListBlobMetadataStrategy getAllBlobMetadata;
protected final ListBlobsInContainer getAllBlobMetadata;
protected final BackoffLimitedRetryHandler retryHandler;
protected final AsyncBlobStore ablobstore;
protected final ExecutorService userExecutor;
@ -74,7 +74,7 @@ public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrateg
@Inject
GetAllBlobsInListAndRetryOnFailure(
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
ListBlobMetadataStrategy getAllBlobMetadata, AsyncBlobStore ablobstore,
ListBlobsInContainer getAllBlobMetadata, AsyncBlobStore ablobstore,
BackoffLimitedRetryHandler retryHandler) {
this.userExecutor = userExecutor;
this.ablobstore = ablobstore;

66
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListBlobMetadataInContainer.java
View File

@ -1,66 +0,0 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.blobstore.strategy.internal;
import javax.inject.Singleton;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ListBlobMetadataStrategy;
import org.jclouds.blobstore.strategy.ListMetadataStrategy;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.inject.Inject;
/**
* Retrieves all blobs in the blobstore by the most efficient means possible.
*
* @author Adrian Cole
*/
@Singleton
public class ListBlobMetadataInContainer implements ListBlobMetadataStrategy {
protected final ListMetadataStrategy lister;
@Inject
ListBlobMetadataInContainer(ListMetadataStrategy lister) {
this.lister = lister;
}
@Override
public Iterable<? extends BlobMetadata> execute(String container, ListContainerOptions options) {
return Iterables.transform(Iterables.filter(lister.execute(container, options),
new Predicate<StorageMetadata>() {
@Override
public boolean apply(StorageMetadata input) {
return input.getType() == StorageType.BLOB;
}
}), new Function<StorageMetadata, BlobMetadata>() {
@Override
public BlobMetadata apply(StorageMetadata from) {
return (BlobMetadata) from;
}
});
}
}

83
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java Normal file
View File

@ -0,0 +1,83 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.blobstore.strategy.internal;
import java.util.List;
import javax.inject.Singleton;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ListBlobsInContainer;
import org.jclouds.blobstore.strategy.ListContainerStrategy;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
/**
* Retrieves all blobs in the blobstore by the most efficient means possible.
*
* @author Adrian Cole
*/
@Singleton
public class ListContainerAndRecurseThroughFolders implements ListBlobsInContainer {
protected final ListContainerStrategy lister;
@Inject
ListContainerAndRecurseThroughFolders(ListContainerStrategy lister) {
this.lister = lister;
}
@Override
public Iterable<? extends BlobMetadata> execute(final String containerName,
final ListContainerOptions options) {
final List<Iterable<? extends BlobMetadata>> lists = Lists.newArrayList();
Iterable<? extends StorageMetadata> pwdList = lister.execute(containerName, options);
for (StorageMetadata md : Iterables.filter(pwdList, new Predicate<StorageMetadata>() {
@Override
public boolean apply(StorageMetadata input) {
return (input.getType() == StorageType.FOLDER || input.getType() == StorageType.RELATIVE_PATH)
&& options.isRecursive();
}
})) {
String directory = (options.getDir() != null) ? options.getDir() + "/" + md.getName() : md
.getName();
lists.add(execute(containerName, options.clone().inDirectory(directory)));
}
lists.add(Iterables.transform(Iterables.filter(pwdList, new Predicate<StorageMetadata>() {
@Override
public boolean apply(StorageMetadata input) {
return input.getType() == StorageType.BLOB;
}
}), new Function<StorageMetadata, BlobMetadata>() {
@Override
public BlobMetadata apply(StorageMetadata from) {
return (BlobMetadata) from;
}
}));
return Sets.newHashSet(Iterables.concat(lists));
}
}

14
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java
View File

@ -30,6 +30,7 @@ import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.blobstore.AsyncBlobStore;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.internal.BlobRuntimeException;
import org.jclouds.blobstore.reference.BlobStoreConstants;
import org.jclouds.blobstore.strategy.DeleteDirectoryStrategy;
@ -64,6 +65,7 @@ import com.google.inject.Inject;
public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy {
private final AsyncBlobStore ablobstore;
private final BlobStore blobstore;
private final ExecutorService userExecutor;
@Resource
@Named(BlobStoreConstants.BLOBSTORE_LOGGER)
@ -78,9 +80,10 @@ public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy {
@Inject
MarkersDeleteDirectoryStrategy(
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor,
AsyncBlobStore ablobstore) {
AsyncBlobStore ablobstore, BlobStore blobstore) {
this.userExecutor = userExecutor;
this.ablobstore = ablobstore;
this.blobstore = blobstore;
}
public void execute(String containerName, String directory) {
@ -93,10 +96,13 @@ public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy {
for (String name : names) {
responses.put(name, ablobstore.removeBlob(containerName, name));
}
String message = String.format("deleting directory %s in containerName: %s", directory,
containerName);
Map<String, Exception> exceptions = awaitCompletion(responses, userExecutor, maxTime, logger,
String.format("deleting directory in containerName: %s", containerName));
message);
if (exceptions.size() > 0)
throw new BlobRuntimeException(String.format("error deleting from container %s: %s",
containerName, exceptions));
throw new BlobRuntimeException(String.format("error %s: %s", message, exceptions));
assert !blobstore.directoryExists(containerName, directory) : String.format(
"still exists %s: %s", message, exceptions);
}
}

5
blobstore/src/test/java/org/jclouds/blobstore/integration/StubBlobMapIntegrationTest.java
View File

@ -28,9 +28,4 @@ import org.testng.annotations.Test;
@Test(groups = { "integration", "live" }, testName = "blobstore.StubBlobMapIntegrationTest")
public class StubBlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
@Override
protected int maxList() {
return 1;
}
}

5
blobstore/src/test/java/org/jclouds/blobstore/integration/StubInputStreamMapIntegrationTest.java
View File

@ -27,9 +27,4 @@ import org.testng.annotations.Test;
@Test(groups = { "integration", "live" }, testName = "blobstore.StubInputStreamMapIntegrationTest")
public class StubInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
@Override
protected int maxList() {
return 1;
}
}

24
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobMapIntegrationTest.java
View File

@ -18,6 +18,7 @@
*/
package org.jclouds.blobstore.integration.internal;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
@ -30,8 +31,10 @@ import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import org.jclouds.blobstore.BlobMap;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl;
import org.jclouds.util.Utils;
import org.testng.annotations.Test;
@ -204,18 +207,16 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
}
}
protected abstract int maxList();
@Test(enabled = false, groups = { "integration", "live" })
@Test(groups = { "integration", "live" })
public void testPutMoreThanSingleListing() throws InterruptedException, ExecutionException,
TimeoutException {
if (maxList() == 0)
if (maxResultsForTestListings() == 0)
return;
String bucketName = getContainerName();
try {
Map<String, Blob> map = createMap(context, bucketName);
Set<String> keySet = Sets.newHashSet();
for (int i = 0; i < maxList() + 1; i++) {
for (int i = 0; i < maxResultsForTestListings() + 1; i++) {
keySet.add(i + "");
}
@ -228,7 +229,7 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
map.putAll(newMap);
newMap.clear();
assertConsistencyAwareMapSize(map, maxList() + 1);
assertConsistencyAwareMapSize(map, maxResultsForTestListings() + 1);
assertConsistencyAwareKeySetEquals(map, keySet);
map.clear();
assertConsistencyAwareMapSize(map, 0);
@ -265,8 +266,15 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
map.putAll(newMap);
}
protected Map<String, Blob> createMap(BlobStoreContext context, String bucket) {
return context.createBlobMap(bucket);
protected int maxResultsForTestListings() {
return 100;
}
protected BlobMap createMap(BlobStoreContext context, String bucket) {
return createMap(context, bucket, maxResults(maxResultsForTestListings()));
}
protected BlobMap createMap(BlobStoreContext context, String bucket, ListContainerOptions options) {
return context.createBlobMap(bucket, options);
}
}

2
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java
View File

@ -207,8 +207,8 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
addAlphabetUnderRoot(containerName);
PageSet<? extends StorageMetadata> container = context.getBlobStore().list(containerName,
maxResults(5));
assert container.getNextMarker() != null;
assertEquals(container.size(), 5);
assert container.getNextMarker() != null;
} finally {
returnContainer(containerName);
}

118
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseInputStreamMapIntegrationTest.java
View File

@ -18,6 +18,7 @@
*/
package org.jclouds.blobstore.integration.internal;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
@ -34,6 +35,7 @@ import java.util.concurrent.TimeoutException;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.InputStreamMap;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.util.Utils;
import org.testng.annotations.Test;
@ -49,9 +51,9 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
@Override
@Test(groups = { "integration", "live" })
public void testValues() throws InterruptedException, IOException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
map.putAll(this.fiveInputs);
// this will cause us to block until the bucket updates.
assertConsistencyAwareMapSize(map, 5);
@ -65,20 +67,18 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
assert valuesAsString.size() == 0 : valuesAsString.size() + ": " + values + ": "
+ valuesAsString;
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
protected abstract int maxList();
@Test(enabled = false, groups = { "integration", "live" })
@Test(groups = { "integration", "live" })
public void testPutMoreThanSingleListing() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
InputStreamMap map = createMap(context, bucketName);
InputStreamMap map = createMap(context, containerName);
Set<String> keySet = Sets.newHashSet();
for (int i = 0; i < maxList() + 1; i++) {
for (int i = 0; i < maxResultsForTestListings() + 1; i++) {
keySet.add(i + "");
}
@ -89,20 +89,20 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
map.putAllStrings(newMap);
newMap.clear();
assertConsistencyAwareMapSize(map, maxList() + 1);
assertConsistencyAwareMapSize(map, maxResultsForTestListings() + 1);
assertConsistencyAwareKeySetEquals(map, keySet);
map.clear();
assertConsistencyAwareMapSize(map, 0);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testRemove() throws InterruptedException, IOException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
putStringWithMD5(map, "one", "two");
InputStream old = map.remove("one");
assertEquals(Utils.toStringAndClose(old), "two");
@ -113,16 +113,16 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
assert old == null;
assertConsistencyAwareKeySize(map, 0);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Override
@Test(groups = { "integration", "live" })
public void testEntrySet() throws InterruptedException, IOException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
((InputStreamMap) map).putAllStrings(this.fiveStrings);
// this will cause us to block until the bucket updates.
assertConsistencyAwareKeySize(map, 5);
@ -137,134 +137,134 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
assertEquals(Utils.toStringAndClose(value), "");
}
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testContainsStringValue() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple"));
assertConsistencyAwareContainsValue(map, fiveStrings.get("one"));
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testContainsFileValue() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple"));
assertConsistencyAwareContainsValue(map, fiveFiles.get("one"));
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testContainsInputStreamValue() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple"));
assertConsistencyAwareContainsValue(map, this.fiveInputs.get("one"));
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testContainsBytesValue() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
((InputStreamMap) map).putString("one", String.format(XML_STRING_FORMAT, "apple"));
assertConsistencyAwareContainsValue(map, this.fiveBytes.get("one"));
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Override
@Test(groups = { "integration", "live" })
public void testPutAll() throws InterruptedException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
map.putAll(this.fiveInputs);
assertConsistencyAwareMapSize(map, 5);
assertConsistencyAwareKeySetEquals(map, new TreeSet<String>(fiveInputs.keySet()));
fourLeftRemovingOne(map);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testPutAllBytes() throws InterruptedException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
((InputStreamMap) map).putAllBytes(this.fiveBytes);
assertConsistencyAwareMapSize(map, 5);
assertConsistencyAwareKeySetEquals(map, new TreeSet<String>(fiveBytes.keySet()));
fourLeftRemovingOne(map);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testPutAllFiles() throws InterruptedException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
((InputStreamMap) map).putAllFiles(this.fiveFiles);
assertConsistencyAwareMapSize(map, 5);
assertConsistencyAwareKeySetEquals(map, new TreeSet<String>(fiveFiles.keySet()));
fourLeftRemovingOne(map);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testPutAllStrings() throws InterruptedException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
((InputStreamMap) map).putAllStrings(this.fiveStrings);
assertConsistencyAwareMapSize(map, 5);
assertConsistencyAwareKeySetEquals(map, new TreeSet<String>(fiveStrings.keySet()));
fourLeftRemovingOne(map);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testPutString() throws InterruptedException, IOException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
InputStream old = ((InputStreamMap) map).putString("one", fiveStrings.get("one"));
getOneReturnsAppleAndOldValueIsNull(map, old);
InputStream apple = ((InputStreamMap) map).putString("one", fiveStrings.get("two"));
getOneReturnsBearAndOldValueIsApple(map, apple);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@ -285,46 +285,46 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
@Test(groups = { "integration", "live" })
public void testPutFile() throws IOException, InterruptedException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
InputStream old = ((InputStreamMap) map).putFile("one", fiveFiles.get("one"));
getOneReturnsAppleAndOldValueIsNull(map, old);
InputStream apple = ((InputStreamMap) map).putFile("one", fiveFiles.get("two"));
getOneReturnsBearAndOldValueIsApple(map, apple);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testPutBytes() throws InterruptedException, IOException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
InputStream old = ((InputStreamMap) map).putBytes("one", fiveBytes.get("one"));
getOneReturnsAppleAndOldValueIsNull(map, old);
InputStream apple = ((InputStreamMap) map).putBytes("one", fiveBytes.get("two"));
getOneReturnsBearAndOldValueIsApple(map, apple);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testPut() throws InterruptedException, IOException {
String bucketName = getContainerName();
String containerName = getContainerName();
try {
Map<String, InputStream> map = createMap(context, bucketName);
Map<String, InputStream> map = createMap(context, containerName);
InputStream old = map.put("one", fiveInputs.get("one"));
getOneReturnsAppleAndOldValueIsNull(map, old);
InputStream apple = map.put("one", fiveInputs.get("two"));
getOneReturnsBearAndOldValueIsApple(map, apple);
} finally {
returnContainer(bucketName);
returnContainer(containerName);
}
}
@ -334,8 +334,16 @@ public abstract class BaseInputStreamMapIntegrationTest extends BaseMapIntegrati
((InputStreamMap) map).putString(key, value);
}
protected int maxResultsForTestListings() {
return 100;
}
protected InputStreamMap createMap(BlobStoreContext context, String bucket) {
InputStreamMap map = context.createInputStreamMap(bucket);
return map;
return createMap(context, bucket, maxResults(maxResultsForTestListings()));
}
protected InputStreamMap createMap(BlobStoreContext context, String bucket,
ListContainerOptions options) {
return context.createInputStreamMap(bucket, options);
}
}

114
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java
View File

@ -18,6 +18,8 @@
*/
package org.jclouds.blobstore.integration.internal;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.inDirectory;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
@ -25,6 +27,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@ -34,6 +37,8 @@ import java.util.concurrent.TimeoutException;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.ListableMap;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.util.Utils;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
@ -97,20 +102,23 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
}
}
protected abstract Map<String, V> createMap(BlobStoreContext context, String bucket);
protected abstract Map<String, V> createMap(BlobStoreContext context, String containerName);
protected abstract Map<String, V> createMap(BlobStoreContext context, String containerName,
ListContainerOptions options);
@Test(groups = { "integration", "live" })
public void testClear() throws InterruptedException, ExecutionException, TimeoutException {
String bucketName = getContainerName();
String containerNameName = getContainerName();
try {
Map<String, V> map = createMap(context, bucketName);
Map<String, V> map = createMap(context, containerNameName);
assertConsistencyAwareMapSize(map, 0);
putStringWithMD5(map, "one", "apple");
assertConsistencyAwareMapSize(map, 1);
map.clear();
assertConsistencyAwareMapSize(map, 0);
} finally {
returnContainer(bucketName);
returnContainer(containerNameName);
}
}
@ -120,18 +128,84 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
@Test(groups = { "integration", "live" })
public void testKeySet() throws InterruptedException, ExecutionException, TimeoutException {
String bucketName = getContainerName();
String containerNameName = getContainerName();
try {
Map<String, V> map = createMap(context, bucketName);
Map<String, V> map = createMap(context, containerNameName);
assertConsistencyAwareKeySize(map, 0);
putStringWithMD5(map, "one", "two");
assertConsistencyAwareKeySize(map, 1);
assertConsistencyAwareKeySetEquals(map, ImmutableSet.of("one"));
} finally {
returnContainer(bucketName);
returnContainer(containerNameName);
}
}
protected void addTenObjectsUnderPrefix(String containerName, String prefix)
throws InterruptedException {
for (int i = 0; i < 10; i++) {
Blob blob = context.getBlobStore().newBlob(prefix + "/" + i);
blob.setPayload(i + "content");
context.getBlobStore().putBlob(containerName, blob);
}
}
protected void addTenObjectsUnderRoot(String containerName) throws InterruptedException {
for (int i = 0; i < 10; i++) {
Blob blob = context.getBlobStore().newBlob(i + "");
blob.setPayload(i + "content");
context.getBlobStore().putBlob(containerName, blob);
}
}
@Test(groups = { "integration", "live" })
public void testDirectory() throws InterruptedException, UnsupportedEncodingException {
String containerName = getContainerName();
try {
String directory = "apps";
Map<String, V> rootMap = createMap(context, containerName);
Map<String, V> rootRecursiveMap = createMap(context, containerName, recursive());
Map<String, V> inDirectoryMap = createMap(context, containerName, inDirectory(directory));
Map<String, V> inDirectoryRecursiveMap = createMap(context, containerName, inDirectory(
directory).recursive());
context.getBlobStore().createDirectory(containerName, directory);
addTenObjectsUnderRoot(containerName);
assertEquals(rootMap.size(), 10);
assertEquals(rootRecursiveMap.size(), 10);
assertEquals(inDirectoryMap.size(), 0);
assertEquals(inDirectoryRecursiveMap.size(), 0);
addTenObjectsUnderPrefix(containerName, directory);
assertEquals(rootMap.size(), 10);
assertEquals(rootRecursiveMap.size(), 20);
assertEquals(inDirectoryMap.size(), 10);
assertEquals(inDirectoryRecursiveMap.size(), 10);
context.getBlobStore().createDirectory(containerName, directory + "/" + directory);
assertEquals(rootMap.size(), 10);
assertEquals(rootRecursiveMap.size(), 20);
assertEquals(inDirectoryMap.size(), 10);
assertEquals(inDirectoryRecursiveMap.size(), 10);
rootMap.clear();
assertEquals(rootMap.size(), 0);
assertEquals(rootRecursiveMap.size(), 10);
assertEquals(inDirectoryMap.size(), 10);
assertEquals(inDirectoryRecursiveMap.size(), 10);
inDirectoryMap.clear();
assertEquals(rootMap.size(), 0);
assertEquals(rootRecursiveMap.size(), 0);
assertEquals(inDirectoryMap.size(), 0);
assertEquals(inDirectoryRecursiveMap.size(), 0);
} finally {
returnContainer(containerName);
}
}
protected void assertConsistencyAwareKeySetEquals(final Map<String, V> map,
final Set<String> expected) throws InterruptedException {
assertConsistencyAware(new Runnable() {
@ -177,20 +251,20 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
@Test(groups = { "integration", "live" })
public void testContainsKey() throws InterruptedException, ExecutionException, TimeoutException {
String bucketName = getContainerName();
String containerNameName = getContainerName();
try {
Map<String, V> map = createMap(context, bucketName);
Map<String, V> map = createMap(context, containerNameName);
assertConsistencyAwareDoesntContainKey(map);
putStringWithMD5(map, "one", "apple");
assertConsistencyAwareContainsKey(map);
} finally {
returnContainer(bucketName);
returnContainer(containerNameName);
}
}
/**
* containsValue() uses eTag comparison to bucket contents, so this can be subject to eventual
* consistency problems.
* containsValue() uses eTag comparison to containerName contents, so this can be subject to
* eventual consistency problems.
*/
protected void assertConsistencyAwareContainsValue(final Map<String, V> map, final Object value)
throws InterruptedException {
@ -221,14 +295,14 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
@Test(groups = { "integration", "live" })
public void testIsEmpty() throws InterruptedException, ExecutionException, TimeoutException {
String bucketName = getContainerName();
String containerNameName = getContainerName();
try {
Map<String, V> map = createMap(context, bucketName);
Map<String, V> map = createMap(context, containerNameName);
assertConsistencyAwareEmpty(map);
putStringWithMD5(map, "one", "apple");
assertConsistencyAwareNotEmpty(map);
} finally {
returnContainer(bucketName);
returnContainer(containerNameName);
}
}
@ -275,17 +349,17 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
@Test(groups = { "integration", "live" })
public void testListContainer() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getContainerName();
String containerNameName = getContainerName();
try {
ListableMap<?, ?> map = (ListableMap<?, ?>) createMap(context, bucketName);
assertConsistencyAwareListContainer(map, bucketName);
ListableMap<?, ?> map = (ListableMap<?, ?>) createMap(context, containerNameName);
assertConsistencyAwareListContainer(map, containerNameName);
} finally {
returnContainer(bucketName);
returnContainer(containerNameName);
}
}
protected void assertConsistencyAwareListContainer(final ListableMap<?, ?> map,
final String bucketName) throws InterruptedException {
final String containerNameName) throws InterruptedException {
assertConsistencyAware(new Runnable() {
public void run() {
assertTrue(Iterables.size(map.list()) >= 0);

12
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/StubAsyncBlobStore.java
View File

@ -19,6 +19,7 @@
package org.jclouds.blobstore.integration.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
import static com.google.common.util.concurrent.Futures.immediateFuture;
@ -157,7 +158,12 @@ public class StubAsyncBlobStore extends BaseAsyncBlobStore {
SortedSet<StorageMetadata> contents = Sets.newTreeSet(Iterables.transform(realContents
.keySet(), new Function<String, StorageMetadata>() {
public StorageMetadata apply(String key) {
MutableBlobMetadata md = copy(realContents.get(key).getMetadata());
Blob oldBlob = realContents.get(key);
checkState(oldBlob != null, "blob " + key
+ " is not present although it was in the list of " + name);
checkState(oldBlob.getMetadata() != null, "blob " + name + "/" + key
+ " has no metadata");
MutableBlobMetadata md = copy(oldBlob.getMetadata());
String directoryName = ifDirectoryReturnName.execute(md);
if (directoryName != null) {
md.setName(directoryName);
@ -242,7 +248,9 @@ public class StubAsyncBlobStore extends BaseAsyncBlobStore {
convertUserMetadataKeysToLowercase(metadata);
return metadata;
} catch (Exception e) {
throw new RuntimeException(e);
Throwables.propagate(e);
assert false : "exception should have propagated: " + e;
return null;
}
}

3
core/src/main/java/org/jclouds/http/HttpRequest.java
View File

@ -103,9 +103,6 @@ public class HttpRequest extends HttpMessage {
return payload;
}
/**
* {@inheritDoc}
*/
public void setPayload(Payload data) {
closeContentIfPresent();
this.payload = checkNotNull(data, "data");

47
core/src/main/java/org/jclouds/rest/functions/ReturnFalseOnResourceNotFound.java Normal file
View File

@ -0,0 +1,47 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.rest.functions;
import static org.jclouds.util.Utils.propagateOrNull;
import java.util.List;
import javax.inject.Singleton;
import org.jclouds.rest.ResourceNotFoundException;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
/**
*
* @author Adrian Cole
*/
@Singleton
public class ReturnFalseOnResourceNotFound implements Function<Exception, Boolean> {
public Boolean apply(Exception from) {
List<Throwable> throwables = Throwables.getCausalChain(from);
if (Iterables.size(Iterables.filter(throwables, ResourceNotFoundException.class)) >= 1) {
return false;
}
return Boolean.class.cast(propagateOrNull(from));
}
}

42
core/src/main/java/org/jclouds/rest/functions/ReturnNullOnResourceNotFound.java Normal file
View File

@ -0,0 +1,42 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.rest.functions;
import static org.jclouds.util.Utils.propagateOrNull;
import javax.inject.Singleton;
import org.jclouds.rest.ResourceNotFoundException;
import com.google.common.base.Function;
/**
*
* @author Adrian Cole
*/
@Singleton
public class ReturnNullOnResourceNotFound implements Function<Exception, Object> {
public Object apply(Exception from) {
if (from instanceof ResourceNotFoundException) {
return null;
}
return propagateOrNull(from);
}
}

3
core/src/main/java/org/jclouds/util/Jsr330.java
View File

@ -31,10 +31,9 @@ import com.google.inject.Key;
* Utility methods for use with {@code @}{@link Named}.
*
* @author crazybob@google.com (Bob Lee) - original code taken from
* {@link com.google.inject.name.Names}
* {@code com.google.inject.name.Names}
*
* @see com.google.inject.util.Jsr330#named
* @see com.google.inject.name.Names
* @author Adrian Cole
*/
public class Jsr330 {

9
core/src/main/java/org/jclouds/util/Utils.java
View File

@ -85,12 +85,8 @@ public class Utils {
}
/**
* Returns a factory that will supply instances of {@link OutputStream} that read from the given
* outputStream.
* converts an {@link OutputStream} to an {@link OutputSupplier}
*
* @param url
* the URL to read from
* @return the factory
*/
public static OutputSupplier<OutputStream> newOutputStreamSupplier(final OutputStream output) {
checkNotNull(output, "output");
@ -146,9 +142,6 @@ public class Utils {
* {@link UnsupportedEncodingException}, log a warning and fall back to the system's default
* encoding.
*
* @see {@link String#getBytes(String)}
* @see {@link String#getBytes()} - used as fall-back.
*
* @param str
* what to encode
* @param charsetName

11
rackspace/src/main/java/org/jclouds/rackspace/cloudfiles/blobstore/functions/ContainerToResourceList.java
View File

@ -24,7 +24,9 @@ import javax.inject.Singleton;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.domain.internal.StorageMetadataImpl;
import org.jclouds.rackspace.cloudfiles.domain.ObjectInfo;
import com.google.common.base.Function;
@ -46,8 +48,13 @@ public class ContainerToResourceList implements
public PageSet<? extends StorageMetadata> apply(PageSet<ObjectInfo> from) {
return new PageSetImpl<StorageMetadata>(Iterables.transform(Iterables.transform(from,
object2blobMd), new Function<BlobMetadata, StorageMetadata>() {
public StorageMetadata apply(BlobMetadata arg0) {
return arg0;
public StorageMetadata apply(BlobMetadata input) {
if (input.getContentType().equals("application/directory")) {
return new StorageMetadataImpl(StorageType.RELATIVE_PATH, input.getId(), input
.getName(), input.getLocationId(), input.getUri(), input.getETag(), input
.getSize(), input.getLastModified(), input.getUserMetadata());
}
return input;
}
}), from.getNextMarker());

16
rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationTest.java
View File

@ -18,10 +18,7 @@
*/
package org.jclouds.rackspace.cloudfiles.blobstore.integration;
import java.io.IOException;
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
/**
@ -38,17 +35,4 @@ public class CloudFilesBlobIntegrationTest extends BaseBlobIntegrationTest {
// not supported in cloud files
}
// TODO this should work, not sure why my encoding is set to Mac Roman and not UTF-8
@DataProvider(name = "delete")
public Object[][] createData() {
return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" },
{ "path/foo" }, { "colon:" }, { "asteri*k" }, { "quote\"" }, { "p|pe" } };
}
@Override
@Test(enabled = false)
public void testPutObject(String key, String type, Object content, Object realObject)
throws InterruptedException, IOException {
// TODO relative path keeps showing up
}
}

5
rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesBlobMapIntegrationTest.java
View File

@ -26,8 +26,5 @@ import org.testng.annotations.Test;
*/
@Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesBlobMapIntegrationTest")
public class CloudFilesBlobMapIntegrationTest extends BaseBlobMapIntegrationTest {
@Override
protected int maxList() {
return 10000;
}
}

16
rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesContainerIntegrationTest.java
View File

@ -18,8 +18,6 @@
*/
package org.jclouds.rackspace.cloudfiles.blobstore.integration;
import java.io.UnsupportedEncodingException;
import org.jclouds.blobstore.integration.internal.BaseContainerIntegrationTest;
import org.testng.annotations.Test;
@ -30,18 +28,4 @@ import org.testng.annotations.Test;
@Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesContainerIntegrationTest")
public class CloudFilesContainerIntegrationTest extends BaseContainerIntegrationTest {
@Override
@Test(enabled = false)
public void testListRootUsesDelimiter() throws InterruptedException,
UnsupportedEncodingException {
// TODO occasionally fails due to virtual directories not deleting from the prior run
}
@Override
@Test(enabled = false)
public void testListContainerMarker() throws InterruptedException,
UnsupportedEncodingException {
// TODO occasionally fails due to virtual directories not deleting from the prior run
}
}

5
rackspace/src/test/java/org/jclouds/rackspace/cloudfiles/blobstore/integration/CloudFilesInputStreamMapIntegrationTest.java
View File

@ -26,8 +26,5 @@ import org.testng.annotations.Test;
*/
@Test(groups = { "integration", "live" }, testName = "cloudfiles.CloudFilesInputStreamMapIntegrationTest")
public class CloudFilesInputStreamMapIntegrationTest extends BaseInputStreamMapIntegrationTest {
@Override
protected int maxList() {
return 10000;
}
}

6
tools/vfs/src/main/java/org/jclouds/vfs/provider/blobstore/BlobStoreFileObject.java
View File

@ -56,7 +56,7 @@ import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.internal.ListAllMetadataInContainer;
import org.jclouds.blobstore.strategy.internal.ConcatenateContainerLists;
import org.jclouds.blobstore.util.internal.BlobStoreUtilsImpl;
import org.jclouds.util.Utils;
@ -70,7 +70,7 @@ import com.google.common.collect.Sets;
*/
public class BlobStoreFileObject extends AbstractFileObject {
private final BlobStoreContext context;
private final ListAllMetadataInContainer lister;
private final ConcatenateContainerLists lister;
private final String container;
private StorageMetadata metadata;
private static final Logger logger = Logger.getLogger(BlobStoreFileObject.class);
@ -81,7 +81,7 @@ public class BlobStoreFileObject extends AbstractFileObject {
super(fileName, fileSystem);
this.context = checkNotNull(context, "context");
this.container = checkNotNull(container, "container");
this.lister = checkNotNull(new ListAllMetadataInContainer(context.getBlobStore()), "lister");
this.lister = checkNotNull(new ConcatenateContainerLists(context.getBlobStore()), "lister");
}

1
vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/TerremarkVCloudClient.java
View File

@ -129,6 +129,7 @@ public interface TerremarkVCloudClient extends VCloudClient {
void deleteNode(int nodeId);
@Timeout(duration = 180, timeUnit = TimeUnit.SECONDS)
SortedSet<Node> getNodes(int internetServiceId);
SortedSet<IpAddress> getIpAddressesForNetwork(String networkId);