From 1d5c0ed5ebded9c4d69a4f05967fa4b960665c6a Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Thu, 29 Mar 2012 10:22:46 +0400 Subject: [PATCH 01/50] Stubs for multipart upload support in swift. --- .../blobstore/CloudFilesBlobStore.java | 3 +- .../swift/blobstore/SwiftBlobStore.java | 12 ++++- .../blobstore/strategy/MultipartUpload.java | 21 +++++++++ .../internal/MultipartUploadStrategy.java | 12 +++++ .../SequentialMultipartUploadStrategy.java | 46 +++++++++++++++++++ .../HPCloudObjectStorageBlobStore.java | 2 +- 6 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java index 42be6d884c..40ba250abd 100644 --- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java +++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java @@ -42,6 +42,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; import com.google.common.base.Supplier; +import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; /** * @@ -62,7 +63,7 @@ public class CloudFilesBlobStore extends SwiftBlobStore { Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions, - fetchBlobMetadataProvider); + fetchBlobMetadataProvider, null); this.enableCDNAndCache = enableCDNAndCache; } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java index e96909ba18..6cc3329861 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java @@ -50,6 +50,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList; import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; +import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; import org.jclouds.openstack.swift.domain.ContainerMetadata; import com.google.common.base.Function; @@ -71,6 +72,7 @@ public class SwiftBlobStore extends BaseBlobStore { private final ObjectToBlobMetadata object2BlobMd; private final BlobToHttpGetOptions blob2ObjectGetOptions; private final Provider fetchBlobMetadataProvider; + private final Provider multipartUploadStrategy; @Inject protected SwiftBlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier defaultLocation, @@ -79,7 +81,8 @@ public class SwiftBlobStore extends BaseBlobStore { BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions, - Provider fetchBlobMetadataProvider) { + Provider fetchBlobMetadataProvider, + Provider multipartUploadStrategy) { super(context, blobUtils, defaultLocation, locations); this.sync = sync; this.container2ResourceMd = container2ResourceMd; @@ -90,6 +93,7 @@ public class SwiftBlobStore extends BaseBlobStore { this.object2BlobMd = object2BlobMd; this.blob2ObjectGetOptions = blob2ObjectGetOptions; this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider"); + this.multipartUploadStrategy = multipartUploadStrategy; } /** @@ -207,7 +211,11 @@ public class SwiftBlobStore extends BaseBlobStore { @Override public String putBlob(String container, Blob blob, PutOptions options) { // TODO implement options - return putBlob(container, blob); + if (options.isMultipart()) { + return multipartUploadStrategy.get().execute(container, blob, options); + } else { + return putBlob(container, blob); + } } /** diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java new file mode 100644 index 0000000000..839f4774e2 --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java @@ -0,0 +1,21 @@ +package org.jclouds.openstack.swift.blobstore.strategy; + +/* +@author Roman Bogorodskiy + */ + +public interface MultipartUpload { + + /* Maximum number of parts per upload */ + public static final int MAX_NUMBER_OF_PARTS = 10000; + /* Maximum number of parts returned for a list parts request */ + public static final int MAX_LIST_PARTS_RETURNED = 1000; + /* Maximum number of multipart uploads returned in a list multipart uploads request */ + public static final int MAX_LIST_MPU_RETURNED = 1000; + + /* + * part size 5 MB to 5 GB, last part can be < 5 MB + */ + public static final long MIN_PART_SIZE = 5242880L; + public static final long MAX_PART_SIZE = 5368709120L; +} diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java new file mode 100644 index 0000000000..40a79ad45a --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java @@ -0,0 +1,12 @@ +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import com.google.inject.ImplementedBy; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload; + +@ImplementedBy(SequentialMultipartUploadStrategy.class) +public interface MultipartUploadStrategy extends MultipartUpload { + + String execute(String container, Blob blob, PutOptions options); +} diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java new file mode 100644 index 0000000000..693cee51be --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java @@ -0,0 +1,46 @@ +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import javax.annotation.Resource; +import javax.inject.Named; + +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.reference.BlobStoreConstants; +import org.jclouds.io.Payload; +import org.jclouds.io.PayloadSlicer; +import org.jclouds.logging.Logger; +import org.jclouds.openstack.swift.blobstore.SwiftBlobStore; + +import static com.google.common.base.Preconditions.checkNotNull; + + +public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy { + @Resource + @Named(BlobStoreConstants.BLOBSTORE_LOGGER) + protected Logger logger = Logger.NULL; + + protected final SwiftBlobStore ablobstore; + protected final PayloadSlicer slicer; + + public SequentialMultipartUploadStrategy(SwiftBlobStore ablobstore, PayloadSlicer slicer) { + this.ablobstore = checkNotNull(ablobstore, "ablobstore"); + this.slicer = checkNotNull(slicer, "slicer"); + } + + @Override + public String execute(String container, Blob blob, PutOptions options) { + String key = blob.getMetadata().getName(); + Payload payload = blob.getPayload(); + /*MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(); + algorithm + .calculateChunkSize(checkNotNull( + payload.getContentMetadata().getContentLength(), + "contentLength required on all uploads to amazon s3; please invoke payload.getContentMetadata().setContentLength(length) first")); + int parts = algorithm.getParts(); + long chunkSize = algorithm.getChunkSize(); + if (parts > 0) { + + } */ + return "NOT IMPLEMENTED"; + } +} diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java index e05119ed8c..cd11feb720 100644 --- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java +++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java @@ -62,7 +62,7 @@ public class HPCloudObjectStorageBlobStore extends SwiftBlobStore { Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions, - fetchBlobMetadataProvider); + fetchBlobMetadataProvider, null); this.enableCDNAndCache = enableCDNAndCache; } From 5f83e6af3d331821455c6e9b5d05ee84819ebdc3 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Fri, 6 Apr 2012 17:26:19 +0400 Subject: [PATCH 02/50] First working implementation of swift multipart upload. Async client TDB. --- .../swift/CommonSwiftAsyncClient.java | 16 +- .../openstack/swift/CommonSwiftClient.java | 2 + .../swift/blobstore/SwiftAsyncBlobStore.java | 9 +- .../swift/blobstore/SwiftBlobStore.java | 2 +- .../MultipartUploadSlicingAlgorithm.java | 152 ++++++++++++++++++ .../internal/MultipartUploadStrategy.java | 3 +- .../SequentialMultipartUploadStrategy.java | 65 ++++++-- .../swift/internal/StubSwiftAsyncClient.java | 13 +- .../java/org/jclouds/aws/s3/AWSS3Client.java | 1 + 9 files changed, 242 insertions(+), 21 deletions(-) create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftAsyncClient.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftAsyncClient.java index 2fff5ae24e..71aae2d781 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftAsyncClient.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftAsyncClient.java @@ -52,14 +52,7 @@ import org.jclouds.openstack.swift.functions.ParseObjectInfoFromHeaders; import org.jclouds.openstack.swift.functions.ParseObjectInfoListFromJsonResponse; import org.jclouds.openstack.swift.functions.ReturnTrueOn404FalseOn409; import org.jclouds.openstack.swift.options.ListContainerOptions; -import org.jclouds.rest.annotations.BinderParam; -import org.jclouds.rest.annotations.Endpoint; -import org.jclouds.rest.annotations.ExceptionParser; -import org.jclouds.rest.annotations.ParamParser; -import org.jclouds.rest.annotations.QueryParams; -import org.jclouds.rest.annotations.RequestFilters; -import org.jclouds.rest.annotations.ResponseParser; -import org.jclouds.rest.annotations.SkipEncoding; +import org.jclouds.rest.annotations.*; import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404; import com.google.common.util.concurrent.ListenableFuture; @@ -183,4 +176,11 @@ public interface CommonSwiftAsyncClient { @Path("/{container}/{name}") ListenableFuture removeObject(@PathParam("container") String container, @PathParam("name") String name); + @PUT + @Path("/{container}/{name}") + @ResponseParser(ParseETagHeader.class) + @Headers(keys = "X-Object-Manifest", values="{container}/{name}") + ListenableFuture putObjectManifest(@PathParam("container") String container, + @PathParam("name") String name); + } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftClient.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftClient.java index a8b5210a32..5ad663bacf 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftClient.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftClient.java @@ -26,6 +26,7 @@ import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.concurrent.Timeout; import org.jclouds.http.options.GetOptions; +import org.jclouds.io.Payload; import org.jclouds.openstack.swift.domain.AccountMetadata; import org.jclouds.openstack.swift.domain.ContainerMetadata; import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata; @@ -113,4 +114,5 @@ public interface CommonSwiftClient { */ boolean objectExists(String container, String name); + String putObjectManifest(String container, String name); } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java index b0873c9af6..e59b86f95e 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java @@ -55,6 +55,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList; import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; +import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; import org.jclouds.openstack.swift.domain.ContainerMetadata; import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata; import org.jclouds.openstack.swift.domain.ObjectInfo; @@ -81,6 +82,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore { private final ObjectToBlobMetadata object2BlobMd; private final BlobToHttpGetOptions blob2ObjectGetOptions; private final Provider fetchBlobMetadataProvider; + //private final Provider multipartUploadStrategy; @Inject protected SwiftAsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, @@ -102,6 +104,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore { this.object2BlobMd = object2BlobMd; this.blob2ObjectGetOptions = blob2ObjectGetOptions; this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider"); + //this.multipartUploadStrategy = multipartUploadStrategy; } /** @@ -239,7 +242,11 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore { @Override public ListenableFuture putBlob(String container, Blob blob, PutOptions options) { // TODO implement options - return putBlob(container, blob); + //if (options.isMultipart()) { + // return null; //Lis multipartUploadStrategy.get().execute(container, blob, options); + //} else { + return putBlob(container, blob); + //} } @Override diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java index 6cc3329861..d19ab2ae44 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java @@ -212,7 +212,7 @@ public class SwiftBlobStore extends BaseBlobStore { public String putBlob(String container, Blob blob, PutOptions options) { // TODO implement options if (options.isMultipart()) { - return multipartUploadStrategy.get().execute(container, blob, options); + return multipartUploadStrategy.get().execute(container, blob, options, blob2Object); } else { return putBlob(container, blob); } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java new file mode 100644 index 0000000000..26c5979dea --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java @@ -0,0 +1,152 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * MultipartUploadSlicingAlgorithm.java + * + * + * Created by: tibor + * + * History + * + */ + +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import org.jclouds.blobstore.reference.BlobStoreConstants; +import org.jclouds.logging.Logger; +import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload; + +import javax.annotation.Resource; +import javax.inject.Named; + +public class MultipartUploadSlicingAlgorithm { + + @Resource + @Named(BlobStoreConstants.BLOBSTORE_LOGGER) + protected Logger logger = Logger.NULL; + + @VisibleForTesting + static final long DEFAULT_PART_SIZE = 33554432; // 32MB + + @VisibleForTesting + static final int DEFAULT_MAGNITUDE_BASE = 100; + + @Inject(optional = true) + @Named("jclouds.mpu.parts.size") + @VisibleForTesting + long defaultPartSize = DEFAULT_PART_SIZE; + + @Inject(optional = true) + @Named("jclouds.mpu.parts.magnitude") + @VisibleForTesting + int magnitudeBase = DEFAULT_MAGNITUDE_BASE; + + // calculated only once, but not from the constructor + private volatile int parts; // required number of parts with chunkSize + private volatile long chunkSize; + private volatile long remaining; // number of bytes remained for the last part + + // sequentially updated values + private volatile int part; + private volatile long chunkOffset; + private volatile long copied; + + @VisibleForTesting + protected long calculateChunkSize(long length) { + long unitPartSize = defaultPartSize; // first try with default part size + int parts = (int)(length / unitPartSize); + long partSize = unitPartSize; + int magnitude = (int) (parts / magnitudeBase); + if (magnitude > 0) { + partSize = magnitude * unitPartSize; + if (partSize > MultipartUpload.MAX_PART_SIZE) { + partSize = MultipartUpload.MAX_PART_SIZE; + unitPartSize = MultipartUpload.MAX_PART_SIZE; + } + parts = (int)(length / partSize); + if (parts * partSize < length) { + partSize = (magnitude + 1) * unitPartSize; + if (partSize > MultipartUpload.MAX_PART_SIZE) { + partSize = MultipartUpload.MAX_PART_SIZE; + unitPartSize = MultipartUpload.MAX_PART_SIZE; + } + parts = (int)(length / partSize); + } + } + if (parts > MultipartUpload.MAX_NUMBER_OF_PARTS) { // if splits in too many parts or + // cannot be split + unitPartSize = MultipartUpload.MIN_PART_SIZE; // take the minimum part size + parts = (int)(length / unitPartSize); + } + if (parts > MultipartUpload.MAX_NUMBER_OF_PARTS) { // if still splits in too many parts + parts = MultipartUpload.MAX_NUMBER_OF_PARTS - 1; // limit them. do not care about not + // covering + } + long remainder = length % unitPartSize; + if (remainder == 0 && parts > 0) { + parts -= 1; + } + this.chunkSize = partSize; + this.parts = parts; + this.remaining = length - partSize * parts; + logger.debug(" %d bytes partitioned in %d parts of part size: %d, remaining: %d%s", length, parts, chunkSize, + remaining, (remaining > MultipartUpload.MAX_PART_SIZE ? " overflow!" : "")); + return this.chunkSize; + } + + public long getCopied() { + return copied; + } + + public void setCopied(long copied) { + this.copied = copied; + } + + @VisibleForTesting + protected int getParts() { + return parts; + } + + protected int getNextPart() { + return ++part; + } + + protected void addCopied(long copied) { + this.copied += copied; + } + + protected long getNextChunkOffset() { + long next = chunkOffset; + chunkOffset += getChunkSize(); + return next; + } + + @VisibleForTesting + protected long getChunkSize() { + return chunkSize; + } + + @VisibleForTesting + protected long getRemaining() { + return remaining; + } + +} diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java index 40a79ad45a..c536a8e8c2 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java @@ -3,10 +3,11 @@ package org.jclouds.openstack.swift.blobstore.strategy.internal; import com.google.inject.ImplementedBy; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.openstack.swift.blobstore.functions.BlobToObject; import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload; @ImplementedBy(SequentialMultipartUploadStrategy.class) public interface MultipartUploadStrategy extends MultipartUpload { - String execute(String container, Blob blob, PutOptions options); + String execute(String container, Blob blob, PutOptions options, BlobToObject blob2Object); } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java index 693cee51be..e11bd41ce3 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java @@ -3,44 +3,91 @@ package org.jclouds.openstack.swift.blobstore.strategy.internal; import javax.annotation.Resource; import javax.inject.Named; +import com.google.common.collect.Maps; +import com.google.inject.Inject; +import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.io.Payload; import org.jclouds.io.PayloadSlicer; import org.jclouds.logging.Logger; +import org.jclouds.openstack.swift.SwiftClient; import org.jclouds.openstack.swift.blobstore.SwiftBlobStore; +import org.jclouds.openstack.swift.blobstore.functions.BlobToObject; +import org.jclouds.openstack.swift.domain.SwiftObject; +import org.jclouds.util.Throwables2; + +import java.util.SortedMap; import static com.google.common.base.Preconditions.checkNotNull; public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy { - @Resource - @Named(BlobStoreConstants.BLOBSTORE_LOGGER) - protected Logger logger = Logger.NULL; + public static final String PART_SEPARATOR = "/"; - protected final SwiftBlobStore ablobstore; - protected final PayloadSlicer slicer; + @Resource + @Named(BlobStoreConstants.BLOBSTORE_LOGGER) + protected Logger logger = Logger.NULL; + protected final SwiftBlobStore ablobstore; + protected final PayloadSlicer slicer; + + @Inject public SequentialMultipartUploadStrategy(SwiftBlobStore ablobstore, PayloadSlicer slicer) { this.ablobstore = checkNotNull(ablobstore, "ablobstore"); this.slicer = checkNotNull(slicer, "slicer"); } @Override - public String execute(String container, Blob blob, PutOptions options) { + public String execute(String container, Blob blob, PutOptions options, BlobToObject blob2Object) { + System.out.println("here we go"); String key = blob.getMetadata().getName(); Payload payload = blob.getPayload(); - /*MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(); + MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(); algorithm .calculateChunkSize(checkNotNull( payload.getContentMetadata().getContentLength(), - "contentLength required on all uploads to amazon s3; please invoke payload.getContentMetadata().setContentLength(length) first")); + "contentLength required on all uploads to swift; please invoke payload.getContentMetadata().setContentLength(length) first")); int parts = algorithm.getParts(); long chunkSize = algorithm.getChunkSize(); if (parts > 0) { + SwiftClient client = (SwiftClient) ablobstore.getContext() + .getProviderSpecificContext().getApi(); - } */ + try { + SortedMap etags = Maps.newTreeMap(); + int part; + while ((part = algorithm.getNextPart()) <= parts) { + System.out.println("Uploading part " + part); + Payload chunkedPart = slicer.slice(payload, + algorithm.getNextChunkOffset(), chunkSize); + Blob blobPart = ablobstore.blobBuilder(blob.getMetadata().getName() + PART_SEPARATOR + + String.valueOf(part)).payload(chunkedPart).contentDisposition( + blob.getMetadata().getName() + PART_SEPARATOR + String.valueOf(part)).build(); + client.putObject(container, blob2Object.apply(blobPart)); + } + long remaining = algorithm.getRemaining(); + if (remaining > 0) { + System.out.println("Uploading tail."); + Payload chunkedPart = slicer.slice(payload, + algorithm.getNextChunkOffset(), remaining); + Blob blobPart = ablobstore.blobBuilder(blob.getMetadata().getName() + PART_SEPARATOR + + String.valueOf(part)).payload(chunkedPart).contentDisposition( + blob.getMetadata().getName() + PART_SEPARATOR + String.valueOf(part)).build(); + client.putObject(container, blob2Object.apply(blobPart)); + } + return client.putObjectManifest(container, key); + } catch (Exception ex) { + RuntimeException rtex = Throwables2.getFirstThrowableOfType(ex, RuntimeException.class); + if (rtex == null) { + rtex = new RuntimeException(ex); + } + //client.abortMultipartUpload(container, key, uploadId); + throw rtex; + } + + } return "NOT IMPLEMENTED"; } } diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java index 01ccfe458b..9a3b189990 100644 --- a/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java @@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorService; import javax.inject.Inject; import javax.inject.Named; import javax.inject.Singleton; +import javax.ws.rs.PathParam; import org.jclouds.Constants; import org.jclouds.blobstore.TransientAsyncBlobStore; @@ -167,7 +168,12 @@ public class StubSwiftAsyncClient implements CommonSwiftAsyncClient { return blobStore.removeBlob(container, key); } - public ListenableFuture setObjectInfo(String container, String key, Map userMetadata) { + @Override + public ListenableFuture putObjectManifest(String container, String name) { + return null; + } + + public ListenableFuture setObjectInfo(String container, String key, Map userMetadata) { throw new UnsupportedOperationException(); } @@ -179,6 +185,11 @@ public class StubSwiftAsyncClient implements CommonSwiftAsyncClient { return objectProvider.create(null); } + + /*public String putObjectManifest(String container, String name) { + return "stub"; + } */ + @Override public ListenableFuture objectExists(String bucketName, String key) { return immediateFuture(containerToBlobs.get(bucketName).containsKey(key)); diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java index ea1ae6548c..5426847372 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java @@ -56,6 +56,7 @@ public interface AWSS3Client extends S3Client { */ String initiateMultipartUpload(String bucketName, ObjectMetadata objectMetadata, PutObjectOptions... options); + /** * This operation aborts a multipart upload. After a multipart upload is aborted, no additional * parts can be uploaded using that upload ID. The storage consumed by any previously uploaded From 13f6d98060db1c72c82159263400d823ad9bf15e Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Tue, 10 Apr 2012 14:07:26 +0400 Subject: [PATCH 03/50] Initial implementation of MPU for SwiftAsyncBlobStore. --- .../blobstore/CloudFilesAsyncBlobStore.java | 2 +- .../swift/blobstore/SwiftAsyncBlobStore.java | 17 +- .../swift/blobstore/SwiftBlobStore.java | 1 - .../AsyncMultipartUploadStrategy.java | 12 + .../ParallelMultipartUploadStrategy.java | 268 ++++++++++++++++++ .../HPCloudObjectStorageAsyncBlobStore.java | 2 +- 6 files changed, 291 insertions(+), 11 deletions(-) create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/AsyncMultipartUploadStrategy.java create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java index 34b61b03de..68ad22f587 100644 --- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java +++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java @@ -69,7 +69,7 @@ public class CloudFilesAsyncBlobStore extends SwiftAsyncBlobStore { Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { super(context, blobUtils, service, defaultLocation, locations, sync, async, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, - blob2ObjectGetOptions, fetchBlobMetadataProvider); + blob2ObjectGetOptions, fetchBlobMetadataProvider, null); this.enableCDNAndCache = enableCDNAndCache; } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java index e59b86f95e..1ffd600bf0 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java @@ -55,6 +55,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList; import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; +import org.jclouds.openstack.swift.blobstore.strategy.internal.AsyncMultipartUploadStrategy; import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; import org.jclouds.openstack.swift.domain.ContainerMetadata; import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata; @@ -82,7 +83,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore { private final ObjectToBlobMetadata object2BlobMd; private final BlobToHttpGetOptions blob2ObjectGetOptions; private final Provider fetchBlobMetadataProvider; - //private final Provider multipartUploadStrategy; + private final Provider multipartUploadStrategy; @Inject protected SwiftAsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, @@ -92,7 +93,8 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore { BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions, - Provider fetchBlobMetadataProvider) { + Provider fetchBlobMetadataProvider, + Provider multipartUploadStrategy) { super(context, blobUtils, service, defaultLocation, locations); this.sync = sync; this.async = async; @@ -104,7 +106,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore { this.object2BlobMd = object2BlobMd; this.blob2ObjectGetOptions = blob2ObjectGetOptions; this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider"); - //this.multipartUploadStrategy = multipartUploadStrategy; + this.multipartUploadStrategy = multipartUploadStrategy; } /** @@ -241,12 +243,11 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore { @Override public ListenableFuture putBlob(String container, Blob blob, PutOptions options) { - // TODO implement options - //if (options.isMultipart()) { - // return null; //Lis multipartUploadStrategy.get().execute(container, blob, options); - //} else { + if (options.isMultipart()) { + return multipartUploadStrategy.get().execute(container, blob, options, blob2Object); + } else { return putBlob(container, blob); - //} + } } @Override diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java index d19ab2ae44..2afc789227 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java @@ -210,7 +210,6 @@ public class SwiftBlobStore extends BaseBlobStore { */ @Override public String putBlob(String container, Blob blob, PutOptions options) { - // TODO implement options if (options.isMultipart()) { return multipartUploadStrategy.get().execute(container, blob, options, blob2Object); } else { diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/AsyncMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/AsyncMultipartUploadStrategy.java new file mode 100644 index 0000000000..017d1dac22 --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/AsyncMultipartUploadStrategy.java @@ -0,0 +1,12 @@ +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.inject.ImplementedBy; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.openstack.swift.blobstore.functions.BlobToObject; + +@ImplementedBy(ParallelMultipartUploadStrategy.class) +public interface AsyncMultipartUploadStrategy { + ListenableFuture execute(String container, Blob blob, PutOptions options, BlobToObject blob2Object); +} diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java new file mode 100644 index 0000000000..bdc86344c2 --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java @@ -0,0 +1,268 @@ +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.inject.Inject; +import org.jclouds.Constants; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.internal.BlobRuntimeException; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.reference.BlobStoreConstants; +import org.jclouds.concurrent.Futures; +import org.jclouds.io.Payload; +import org.jclouds.io.PayloadSlicer; +import org.jclouds.logging.Logger; +import org.jclouds.openstack.swift.SwiftAsyncClient; +import org.jclouds.openstack.swift.SwiftClient; +import org.jclouds.openstack.swift.blobstore.SwiftAsyncBlobStore; +import org.jclouds.openstack.swift.blobstore.functions.BlobToObject; +import org.jclouds.util.Throwables2; + +import javax.annotation.Resource; +import javax.inject.Named; +import java.util.Map; +import java.util.Queue; +import java.util.SortedMap; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStrategy { + @Resource + @Named(BlobStoreConstants.BLOBSTORE_LOGGER) + protected Logger logger = Logger.NULL; + + public static final String PART_SEPARATOR = "/"; + @VisibleForTesting + static final int DEFAULT_PARALLEL_DEGREE = 4; + @VisibleForTesting + static final int DEFAULT_MIN_RETRIES = 5; + @VisibleForTesting + static final int DEFAULT_MAX_PERCENT_RETRIES = 10; + + @Inject(optional = true) + @Named("jclouds.mpu.parallel.degree") + @VisibleForTesting + int parallelDegree = DEFAULT_PARALLEL_DEGREE; + + @Inject(optional = true) + @Named("jclouds.mpu.parallel.retries.min") + @VisibleForTesting + int minRetries = DEFAULT_MIN_RETRIES; + + @Inject(optional = true) + @Named("jclouds.mpu.parallel.retries.maxpercent") + @VisibleForTesting + int maxPercentRetries = DEFAULT_MAX_PERCENT_RETRIES; + + /** + * maximum duration of an blob Request + */ + @Inject(optional = true) + @Named(Constants.PROPERTY_REQUEST_TIMEOUT) + protected Long maxTime; + + private final ExecutorService ioWorkerExecutor; + + protected final SwiftAsyncBlobStore ablobstore; + protected final PayloadSlicer slicer; + + @Inject + public ParallelMultipartUploadStrategy(SwiftAsyncBlobStore ablobstore, PayloadSlicer slicer, + @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor) { + this.ablobstore = checkNotNull(ablobstore, "ablobstore"); + this.slicer = checkNotNull(slicer, "slicer"); + this.ioWorkerExecutor = checkNotNull(ioWorkerExecutor, "ioWorkerExecutor"); + } + + + protected void prepareUploadPart(final String container, final Blob blob, final String key, + final Integer part, final Payload payload, + final long offset, final long size, final SortedMap etags, + final BlockingQueue activeParts, + final Map> futureParts, + final AtomicInteger errors, final int maxRetries, final Map errorMap, + final Queue toRetry, final CountDownLatch latch, + BlobToObject blob2Object) { + if (errors.get() > maxRetries) { + activeParts.remove(part); // remove part from the bounded-queue without blocking + latch.countDown(); + return; + } + final SwiftAsyncClient client = (SwiftAsyncClient) ablobstore.getContext() + .getProviderSpecificContext().getAsyncApi(); + Payload chunkedPart = slicer.slice(payload, offset, size); + logger.debug(String.format("async uploading part %s of %s to container %s", part, key, container)); + final long start = System.currentTimeMillis(); + String blobPartName = blob.getMetadata().getName() + PART_SEPARATOR + + String.valueOf(part); + + Blob blobPart = ablobstore.blobBuilder(blobPartName).payload(chunkedPart). + contentDisposition(blobPartName).build(); + final ListenableFuture futureETag = client.putObject(container, blob2Object.apply(blobPart)); + futureETag.addListener(new Runnable() { + @Override + public void run() { + try { + etags.put(part, futureETag.get()); + logger.debug(String.format("async uploaded part %s of %s to container %s in %sms", + part, key, container, (System.currentTimeMillis() - start))); + } catch (CancellationException e) { + errorMap.put(part, e); + String message = String.format("%s while uploading part %s - [%s,%s] to container %s with running since %dms", + e.getMessage(), part, offset, size, container, (System.currentTimeMillis() - start)); + logger.debug(message); + } catch (Exception e) { + errorMap.put(part, e); + String message = String.format("%s while uploading part %s - [%s,%s] to container %s running since %dms", + e.getMessage(), part, offset, size, container, (System.currentTimeMillis() - start)); + logger.error(message, e); + if (errors.incrementAndGet() <= maxRetries) + toRetry.add(new Part(part, offset, size)); + } finally { + activeParts.remove(part); // remove part from the bounded-queue without blocking + futureParts.remove(part); + latch.countDown(); + } + } + }, ioWorkerExecutor); + futureParts.put(part, futureETag); + } + + @Override + public ListenableFuture execute(final String container, final Blob blob, final PutOptions options, final BlobToObject blob2Object) { + return Futures.makeListenable( + ioWorkerExecutor.submit(new Callable() { + @Override + public String call() throws Exception { + String key = blob.getMetadata().getName(); + Payload payload = blob.getPayload(); + MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(); + algorithm.calculateChunkSize(payload.getContentMetadata() + .getContentLength()); + int parts = algorithm.getParts(); + long chunkSize = algorithm.getChunkSize(); + long remaining = algorithm.getRemaining(); + if (parts > 0) { + SwiftClient client = (SwiftClient) ablobstore + .getContext().getProviderSpecificContext().getApi(); + final Map> futureParts = + new ConcurrentHashMap>(); + final Map errorMap = Maps.newHashMap(); + AtomicInteger errors = new AtomicInteger(0); + int maxRetries = Math.max(minRetries, parts * maxPercentRetries / 100); + int effectiveParts = remaining > 0 ? parts + 1 : parts; + try { + logger.debug(String.format("initiated multipart upload of %s to container %s" + + " consisting from %s part (possible max. retries: %d)", + key, container, effectiveParts, maxRetries)); + // we need a bounded-blocking queue to control the amount of parallel jobs + ArrayBlockingQueue activeParts = new ArrayBlockingQueue(parallelDegree); + Queue toRetry = new ConcurrentLinkedQueue(); + SortedMap etags = new ConcurrentSkipListMap(); + CountDownLatch latch = new CountDownLatch(effectiveParts); + int part; + while ((part = algorithm.getNextPart()) <= parts) { + Integer partKey = new Integer(part); + activeParts.put(partKey); + + prepareUploadPart(container, blob, key, partKey, payload, + algorithm.getNextChunkOffset(), chunkSize, etags, + activeParts, futureParts, errors, maxRetries, errorMap, toRetry, latch, + blob2Object); + } + if (remaining > 0) { + Integer partKey = new Integer(part); + activeParts.put(partKey); + prepareUploadPart(container, blob, key, partKey, payload, + algorithm.getNextChunkOffset(), remaining, etags, + activeParts, futureParts, errors, maxRetries, errorMap, toRetry, latch, + blob2Object); + } + latch.await(); + // handling retries + while (errors.get() <= maxRetries && toRetry.size() > 0) { + int atOnce = Math.min(Math.min(toRetry.size(), errors.get()), parallelDegree); + CountDownLatch retryLatch = new CountDownLatch(atOnce); + for (int i = 0; i < atOnce; i++) { + Part failedPart = toRetry.poll(); + Integer partKey = new Integer(failedPart.getPart()); + activeParts.put(partKey); + prepareUploadPart(container, blob, key, partKey, payload, + failedPart.getOffset(), failedPart.getSize(), etags, + activeParts, futureParts, errors, maxRetries, errorMap, toRetry, retryLatch, + blob2Object); + } + retryLatch.await(); + } + if (errors.get() > maxRetries) { + throw new BlobRuntimeException(String.format( + "Too many failed parts: %s while multipart upload of %s to container %s", + errors.get(), key, container)); + } + + String eTag = client.putObjectManifest(container, key); + logger.debug(String.format("multipart upload of %s to container %s" + + " succeffully finished with %s retries", key, container, errors.get())); + return eTag; + } catch (Exception ex) { + RuntimeException rtex = Throwables2.getFirstThrowableOfType(ex, RuntimeException.class); + if (rtex == null) { + rtex = new RuntimeException(ex); + } + for (Map.Entry> entry : futureParts.entrySet()) { + entry.getValue().cancel(false); + } + /* + if (uploadId != null) { + client.abortMultipartUpload(container, key, uploadId); + } */ + throw rtex; + } + } else { + ListenableFuture futureETag = ablobstore.putBlob(container, blob, options); + return maxTime != null ? + futureETag.get(maxTime, TimeUnit.SECONDS) : futureETag.get(); + } + } + }), ioWorkerExecutor); + } + + class Part { + private int part; + private long offset; + private long size; + + Part(int part, long offset, long size) { + this.part = part; + this.offset = offset; + this.size = size; + } + + public int getPart() { + return part; + } + + public void setPart(int part) { + this.part = part; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + } +} diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java index ed305371de..fdaaa3ec4d 100644 --- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java +++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java @@ -69,7 +69,7 @@ public class HPCloudObjectStorageAsyncBlobStore extends SwiftAsyncBlobStore { Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { super(context, blobUtils, service, defaultLocation, locations, sync, async, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, - blob2ObjectGetOptions, fetchBlobMetadataProvider); + blob2ObjectGetOptions, fetchBlobMetadataProvider, null); this.enableCDNAndCache = enableCDNAndCache; } From 9d5f242e6c8f452e21afab929071da476578f626 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Tue, 10 Apr 2012 14:28:24 +0400 Subject: [PATCH 04/50] Clean up commented out useless function. --- .../openstack/swift/internal/StubSwiftAsyncClient.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java index 9a3b189990..7178760509 100644 --- a/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java @@ -185,11 +185,6 @@ public class StubSwiftAsyncClient implements CommonSwiftAsyncClient { return objectProvider.create(null); } - - /*public String putObjectManifest(String container, String name) { - return "stub"; - } */ - @Override public ListenableFuture objectExists(String bucketName, String key) { return immediateFuture(containerToBlobs.get(bucketName).containsKey(key)); From 9e8bc44285b32650d07f94bfe7e83cabffc5861d Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Tue, 17 Apr 2012 15:21:26 +0400 Subject: [PATCH 05/50] Use CommonSwiftClient instead of SwiftClient in multipart code. --- .../cloudfiles/blobstore/CloudFilesAsyncBlobStore.java | 6 ++++-- .../strategy/internal/ParallelMultipartUploadStrategy.java | 3 ++- .../internal/SequentialMultipartUploadStrategy.java | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java index 68ad22f587..4d1cbb63e0 100644 --- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java +++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java @@ -49,6 +49,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.util.concurrent.ListenableFuture; +import org.jclouds.openstack.swift.blobstore.strategy.internal.AsyncMultipartUploadStrategy; /** * @@ -66,10 +67,11 @@ public class CloudFilesAsyncBlobStore extends SwiftAsyncBlobStore { BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions, - Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { + Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache, + Provider multipartUploadStrategy) { super(context, blobUtils, service, defaultLocation, locations, sync, async, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, - blob2ObjectGetOptions, fetchBlobMetadataProvider, null); + blob2ObjectGetOptions, fetchBlobMetadataProvider, multipartUploadStrategy); this.enableCDNAndCache = enableCDNAndCache; } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java index bdc86344c2..95e4fc8983 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java @@ -13,6 +13,7 @@ import org.jclouds.concurrent.Futures; import org.jclouds.io.Payload; import org.jclouds.io.PayloadSlicer; import org.jclouds.logging.Logger; +import org.jclouds.openstack.swift.CommonSwiftAsyncClient; import org.jclouds.openstack.swift.SwiftAsyncClient; import org.jclouds.openstack.swift.SwiftClient; import org.jclouds.openstack.swift.blobstore.SwiftAsyncBlobStore; @@ -91,7 +92,7 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra latch.countDown(); return; } - final SwiftAsyncClient client = (SwiftAsyncClient) ablobstore.getContext() + final CommonSwiftAsyncClient client = (CommonSwiftAsyncClient) ablobstore.getContext() .getProviderSpecificContext().getAsyncApi(); Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s", part, key, container)); diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java index e11bd41ce3..d77012661b 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java @@ -12,6 +12,7 @@ import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.io.Payload; import org.jclouds.io.PayloadSlicer; import org.jclouds.logging.Logger; +import org.jclouds.openstack.swift.CommonSwiftClient; import org.jclouds.openstack.swift.SwiftClient; import org.jclouds.openstack.swift.blobstore.SwiftBlobStore; import org.jclouds.openstack.swift.blobstore.functions.BlobToObject; @@ -52,7 +53,7 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg int parts = algorithm.getParts(); long chunkSize = algorithm.getChunkSize(); if (parts > 0) { - SwiftClient client = (SwiftClient) ablobstore.getContext() + CommonSwiftClient client = (CommonSwiftClient) ablobstore.getContext() .getProviderSpecificContext().getApi(); try { From 9d76fd1aa4c5103a7b95021bd9cabaf18c9dfcc2 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Tue, 17 Apr 2012 18:22:34 +0400 Subject: [PATCH 06/50] Enable multipart for HPCloud. --- .../jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java | 5 +++-- .../strategy/internal/ParallelMultipartUploadStrategy.java | 5 ++--- .../blobstore/HPCloudObjectStorageAsyncBlobStore.java | 6 ++++-- .../blobstore/HPCloudObjectStorageBlobStore.java | 6 ++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java index 40ba250abd..c0d8f2d099 100644 --- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java +++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java @@ -60,10 +60,11 @@ public class CloudFilesBlobStore extends SwiftBlobStore { BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions, - Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { + Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache, + Provider multipartUploadStrategy) { super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions, - fetchBlobMetadataProvider, null); + fetchBlobMetadataProvider, multipartUploadStrategy); this.enableCDNAndCache = enableCDNAndCache; } diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java index 95e4fc8983..d4331caaa5 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java @@ -14,8 +14,7 @@ import org.jclouds.io.Payload; import org.jclouds.io.PayloadSlicer; import org.jclouds.logging.Logger; import org.jclouds.openstack.swift.CommonSwiftAsyncClient; -import org.jclouds.openstack.swift.SwiftAsyncClient; -import org.jclouds.openstack.swift.SwiftClient; +import org.jclouds.openstack.swift.CommonSwiftClient; import org.jclouds.openstack.swift.blobstore.SwiftAsyncBlobStore; import org.jclouds.openstack.swift.blobstore.functions.BlobToObject; import org.jclouds.util.Throwables2; @@ -147,7 +146,7 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra long chunkSize = algorithm.getChunkSize(); long remaining = algorithm.getRemaining(); if (parts > 0) { - SwiftClient client = (SwiftClient) ablobstore + CommonSwiftClient client = (CommonSwiftClient) ablobstore .getContext().getProviderSpecificContext().getApi(); final Map> futureParts = new ConcurrentHashMap>(); diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java index fdaaa3ec4d..847eb686ea 100644 --- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java +++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java @@ -45,6 +45,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList; import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; +import org.jclouds.openstack.swift.blobstore.strategy.internal.AsyncMultipartUploadStrategy; import com.google.common.base.Function; import com.google.common.base.Supplier; @@ -66,10 +67,11 @@ public class HPCloudObjectStorageAsyncBlobStore extends SwiftAsyncBlobStore { BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions, - Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { + Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache, + Provider multipartUploadStrategy) { super(context, blobUtils, service, defaultLocation, locations, sync, async, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, - blob2ObjectGetOptions, fetchBlobMetadataProvider, null); + blob2ObjectGetOptions, fetchBlobMetadataProvider, multipartUploadStrategy); this.enableCDNAndCache = enableCDNAndCache; } diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java index cd11feb720..365cb294d0 100644 --- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java +++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java @@ -40,6 +40,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList; import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; +import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; import com.google.common.base.Supplier; @@ -59,10 +60,11 @@ public class HPCloudObjectStorageBlobStore extends SwiftBlobStore { BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions, ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions, - Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) { + Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache, + Provider multipartUploadStrategy) { super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions, - fetchBlobMetadataProvider, null); + fetchBlobMetadataProvider, multipartUploadStrategy); this.enableCDNAndCache = enableCDNAndCache; } From 1534e4ff37fca523af3cbf1b06b3153000857c8a Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Wed, 18 Apr 2012 12:41:09 +0400 Subject: [PATCH 07/50] Remove deub prints and fall back to traditional upload if file is not large enough. --- .../internal/SequentialMultipartUploadStrategy.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java index d77012661b..c723a51742 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java @@ -42,7 +42,6 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg @Override public String execute(String container, Blob blob, PutOptions options, BlobToObject blob2Object) { - System.out.println("here we go"); String key = blob.getMetadata().getName(); Payload payload = blob.getPayload(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(); @@ -57,10 +56,8 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg .getProviderSpecificContext().getApi(); try { - SortedMap etags = Maps.newTreeMap(); int part; while ((part = algorithm.getNextPart()) <= parts) { - System.out.println("Uploading part " + part); Payload chunkedPart = slicer.slice(payload, algorithm.getNextChunkOffset(), chunkSize); Blob blobPart = ablobstore.blobBuilder(blob.getMetadata().getName() + PART_SEPARATOR + @@ -70,7 +67,6 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg } long remaining = algorithm.getRemaining(); if (remaining > 0) { - System.out.println("Uploading tail."); Payload chunkedPart = slicer.slice(payload, algorithm.getNextChunkOffset(), remaining); Blob blobPart = ablobstore.blobBuilder(blob.getMetadata().getName() + PART_SEPARATOR + @@ -84,11 +80,10 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg if (rtex == null) { rtex = new RuntimeException(ex); } - //client.abortMultipartUpload(container, key, uploadId); throw rtex; } - + } else { + return ablobstore.putBlob(container, blob, PutOptions.NONE); } - return "NOT IMPLEMENTED"; } } From 7d39b056ba1364d644d2d2d24bbd2ddd86b47ed2 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Wed, 18 Apr 2012 14:28:47 +0400 Subject: [PATCH 08/50] Remove useless comment. --- .../openstack/swift/blobstore/strategy/MultipartUpload.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java index 839f4774e2..9c2ead8253 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java @@ -1,9 +1,5 @@ package org.jclouds.openstack.swift.blobstore.strategy; -/* -@author Roman Bogorodskiy - */ - public interface MultipartUpload { /* Maximum number of parts per upload */ From dbf59adce139e525841b451abe03cb7a1734c00a Mon Sep 17 00:00:00 2001 From: danikov Date: Wed, 18 Apr 2012 16:59:01 +0100 Subject: [PATCH 09/50] bugfix for self-referential loop + test --- .../org/jclouds/util/ConcreteFunction.java | 45 +++++++++++++++++++ .../test/java/org/jclouds/util/Maps2Test.java | 15 +++++++ 2 files changed, 60 insertions(+) create mode 100644 core/src/main/java/org/jclouds/util/ConcreteFunction.java diff --git a/core/src/main/java/org/jclouds/util/ConcreteFunction.java b/core/src/main/java/org/jclouds/util/ConcreteFunction.java new file mode 100644 index 0000000000..27ac98c67d --- /dev/null +++ b/core/src/main/java/org/jclouds/util/ConcreteFunction.java @@ -0,0 +1,45 @@ +/** + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jclouds.util; + +import com.google.common.base.Function; + + +/** + * For wrapping covariant functions for passing to non-covariant methods + * + * @author danikov + */ +public class ConcreteFunction implements Function { + private final Function delegate; + + public static ConcreteFunction wrap(Function delegate) { + return new ConcreteFunction(delegate); + } + + public ConcreteFunction(Function delegate) { + this.delegate = delegate; + } + + @Override + public T apply(F input) { + return delegate.apply(input); + } + +} diff --git a/core/src/test/java/org/jclouds/util/Maps2Test.java b/core/src/test/java/org/jclouds/util/Maps2Test.java index d0f3cbf1fd..8c52536fa7 100644 --- a/core/src/test/java/org/jclouds/util/Maps2Test.java +++ b/core/src/test/java/org/jclouds/util/Maps2Test.java @@ -31,6 +31,7 @@ import org.testng.annotations.Test; import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; /** @@ -86,4 +87,18 @@ public class Maps2Test { }), expected); } + @Test + public void testCovariantUniqueIndex() { + Iterable values = Lists.newArrayList(1, 2, 3, 4, 5); + Map map = Maps2.uniqueIndex(values, new Function() { + + @Override + public Double apply(Object input) { + return (Integer)input + 0.1; + } + }); + + assertEquals(map.get(1.1), 1); + } + } From 437bb83496d061abf1e884a77f37444d6be3c594 Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Fri, 13 Apr 2012 16:01:21 +0100 Subject: [PATCH 10/50] Do not create new users just for the tests, run them with the passed in credentials --- .../org/jclouds/apis/BaseContextLiveTest.java | 3 +- .../v1_5/VCloudDirectorMediaType.java | 4 +- .../BaseVCloudDirectorClientLiveTest.java | 2 - .../internal/VCloudDirectorTestSession.java | 83 +++++-------------- 4 files changed, 27 insertions(+), 65 deletions(-) diff --git a/core/src/test/java/org/jclouds/apis/BaseContextLiveTest.java b/core/src/test/java/org/jclouds/apis/BaseContextLiveTest.java index f71ac1e121..deb4c08ce2 100644 --- a/core/src/test/java/org/jclouds/apis/BaseContextLiveTest.java +++ b/core/src/test/java/org/jclouds/apis/BaseContextLiveTest.java @@ -82,8 +82,7 @@ public abstract class BaseContextLiveTest { } protected void initializeContext() { - if (context != null) - Closeables.closeQuietly(context); + Closeables.closeQuietly(context); context = createContext(setupProperties(), setupModules()); } diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/VCloudDirectorMediaType.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/VCloudDirectorMediaType.java index 68d93a1b09..d12ed4fdb1 100644 --- a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/VCloudDirectorMediaType.java +++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/VCloudDirectorMediaType.java @@ -196,6 +196,8 @@ public class VCloudDirectorMediaType { public static final String NETWORK_POOL = "application/vnd.vmware.admin.networkPool+xml"; public static final String ENTITY = "application/vnd.vmware.vcloud.entity+xml"; + + public static final String ADMIN = "application/vnd.vmware.admin.vcloud+xml"; /** * All acceptable media types. @@ -220,7 +222,7 @@ public class VCloudDirectorMediaType { ADMIN_ORG_NETWORK, USER, ROLE, DEPLOY_VAPP_PARAMS, RECOMPOSE_VAPP_PARAMS, RELOCATE_VM_PARAMS, UNDEPLOY_VAPP_PARAMS, ADMIN_VDC, MEDIA_PARAMS, RUNTIME_INFO_SECTION, SCREEN_TICKET, VAPP_NETWORK, - TEXT_XML, ADMIN_VDC, NETWORK_POOL, ADMIN_ORG, ENTITY + TEXT_XML, ADMIN_VDC, NETWORK_POOL, ADMIN_ORG, ENTITY, ADMIN ); // NOTE These lists must be updated whenever a new media type constant is added. diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java index 12b093af8e..44cbe2b2b3 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java @@ -179,8 +179,6 @@ public abstract class BaseVCloudDirectorClientLiveTest extends BaseContextLiveTe .overrides(overrides) .build(); - System.err.println("*** " + endpoint + " ***"); - context = testSession.getUserContext(); adminContext = testSession.getAdminContext(); diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java index 858e98d7db..ed8bc0a351 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java @@ -2,6 +2,7 @@ package org.jclouds.vcloud.director.v1_5.internal; import static com.google.common.base.Objects.equal; import static org.jclouds.vcloud.director.v1_5.VCloudDirectorLiveTestConstants.REF_REQ_LIVE; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; @@ -14,16 +15,19 @@ import org.jclouds.logging.log4j.config.Log4JLoggingModule; import org.jclouds.rest.RestContext; import org.jclouds.sshj.config.SshjSshClientModule; import org.jclouds.vcloud.director.v1_5.VCloudDirectorContext; +import org.jclouds.vcloud.director.v1_5.VCloudDirectorMediaType; import org.jclouds.vcloud.director.v1_5.admin.VCloudDirectorAdminAsyncClient; import org.jclouds.vcloud.director.v1_5.admin.VCloudDirectorAdminClient; import org.jclouds.vcloud.director.v1_5.domain.Link; import org.jclouds.vcloud.director.v1_5.domain.Reference; import org.jclouds.vcloud.director.v1_5.domain.Role.DefaultRoles; import org.jclouds.vcloud.director.v1_5.domain.User; +import org.jclouds.vcloud.director.v1_5.predicates.LinkPredicates; import org.jclouds.vcloud.director.v1_5.predicates.ReferencePredicates; import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorAsyncClient; import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorClient; +import com.google.common.base.Predicates; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.io.Closeables; @@ -73,11 +77,8 @@ public class VCloudDirectorTestSession implements Closeable { } } + private VCloudDirectorContext userContext; private RestContext adminContext; - private RestContext userContext; - - private User createdAdminUser; - private User createdUser; private VCloudDirectorTestSession(String provider, String identity, String credential, Properties overrides, String endpoint) { ContextBuilder builder = ContextBuilder.newBuilder(provider) @@ -85,74 +86,36 @@ public class VCloudDirectorTestSession implements Closeable { .endpoint(endpoint) .modules(ImmutableSet. of(new Log4JLoggingModule(), new SshjSshClientModule())) .overrides(overrides); - VCloudDirectorContext rootContext = VCloudDirectorContext.class.cast(builder.build()); + userContext = VCloudDirectorContext.class.cast(builder.build()); - if (rootContext.getApi().getCurrentSession().getLinks().contains(Link.builder() - .rel("down") - .type("application/vnd.vmware.admin.vcloud+xml") - .href(URI.create(endpoint+"/admin/")) - .build())) { + // Look for the admin link in the current session + Link admin = Iterables.tryFind( + userContext.getApi().getCurrentSession().getLinks(), + Predicates.and(LinkPredicates.relEquals(Link.Rel.DOWN), ReferencePredicates.typeEquals(VCloudDirectorMediaType.ADMIN))) + .orNull(); - adminContext = rootContext.getAdminContext(); + // Get the admin context if the link exists + if (admin != null) { + adminContext = userContext.getAdminContext(); - Reference orgRef = Iterables.getFirst(rootContext.getApi().getOrgClient().getOrgList().getOrgs(), null) + // Lookup the user details + Reference orgRef = Iterables.getFirst(userContext.getApi().getOrgClient().getOrgList().getOrgs(), null) .toAdminReference(endpoint); - assertNotNull(orgRef, String.format(REF_REQ_LIVE, "admin org")); - - Reference userRef = Iterables.find(adminContext.getApi().getOrgClient().getOrg(orgRef.getHref()).getUsers(), + Reference userRef = Iterables.find( + adminContext.getApi().getOrgClient().getOrg(orgRef.getHref()).getUsers(), ReferencePredicates.nameEquals(adminContext.getApi().getCurrentSession().getUser())); - User user = adminContext.getApi().getUserClient().getUser(userRef.getHref()); - Reference orgAdmin = user.getRole(); - assertTrue(equal(orgAdmin.getName(), DefaultRoles.ORG_ADMIN.value()), "must give org admin or user-only credentials"); - String adminIdentity = "testAdmin"+BaseVCloudDirectorClientLiveTest.getTestDateTimeStamp(); - String adminCredential = "testAdminPassword"; - - createdAdminUser = rootContext.getAdminContext().getApi().getUserClient().createUser(orgRef.getHref(), User.builder() - .name(adminIdentity) - .password(adminCredential) - .description("test user with user-level privileges") - .role(orgAdmin) - .deployedVmQuota(BaseVCloudDirectorClientLiveTest.REQUIRED_ADMIN_VM_QUOTA) - .isEnabled(true) - .build()); - - Closeables.closeQuietly(rootContext); - - builder.credentials(adminIdentity, adminCredential); - adminContext = VCloudDirectorContext.class.cast(builder.build()).getAdminContext(); - - String userIdentity = "test"+BaseVCloudDirectorClientLiveTest.getTestDateTimeStamp(); - String userCredential = "testPassword"; - - createdUser = adminContext.getApi().getUserClient().createUser(orgRef.getHref(), User.builder() - .name(userIdentity) - .password(userCredential) - .description("test user with user-level privileges") - .role(BaseVCloudDirectorClientLiveTest.getRoleReferenceFor(DefaultRoles.USER.value(), adminContext)) - .deployedVmQuota(BaseVCloudDirectorClientLiveTest.REQUIRED_USER_VM_QUOTA) - .isEnabled(true) - .build()); - - builder.credentials(userIdentity, userCredential); - userContext = VCloudDirectorContext.class.cast(builder.build()); - } else { - userContext = rootContext; + // Check that the user has the org admin role + Reference userRole = user.getRole(); + assertEquals(userRole.getName(), DefaultRoles.ORG_ADMIN.value()); } } @Override public void close() { - if (createdUser != null) { - adminContext.getApi().getUserClient().deleteUser(createdUser.getHref()); - } - if (userContext != null) userContext.close(); - if (createdAdminUser != null) { - // TODO: may have to preserve root context if we can't delete the user for it's own context here - adminContext.getApi().getUserClient().deleteUser(createdAdminUser.getHref()); - } - if (adminContext != null) adminContext.close(); + Closeables.closeQuietly(userContext); + Closeables.closeQuietly(adminContext); } public RestContext getUserContext() { From a8e5938b6a9d90a161bea168f8afc640cec6460d Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Mon, 16 Apr 2012 10:47:00 +0100 Subject: [PATCH 11/50] Wrong check for not found --- .../v1_5/features/VAppClientLiveTest.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/features/VAppClientLiveTest.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/features/VAppClientLiveTest.java index 476fa75c4e..edb255169c 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/features/VAppClientLiveTest.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/features/VAppClientLiveTest.java @@ -19,8 +19,8 @@ package org.jclouds.vcloud.director.v1_5.features; import static com.google.common.base.Predicates.and; -import static com.google.common.collect.Iterables.find; import static com.google.common.collect.Iterables.contains; +import static com.google.common.collect.Iterables.find; import static com.google.common.collect.Iterables.getFirst; import static org.jclouds.vcloud.director.v1_5.VCloudDirectorLiveTestConstants.CONDITION_FMT; import static org.jclouds.vcloud.director.v1_5.VCloudDirectorLiveTestConstants.CORRECT_VALUE_OBJECT_FMT; @@ -58,8 +58,8 @@ import static org.jclouds.vcloud.director.v1_5.predicates.LinkPredicates.typeEqu import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; import java.math.BigInteger; import java.net.URI; @@ -68,15 +68,14 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.jclouds.io.Payloads; import org.jclouds.dmtf.cim.OSType; import org.jclouds.dmtf.cim.ResourceAllocationSettingData; import org.jclouds.dmtf.ovf.MsgType; import org.jclouds.dmtf.ovf.NetworkSection; import org.jclouds.dmtf.ovf.ProductSection; import org.jclouds.dmtf.ovf.StartupSection; +import org.jclouds.io.Payloads; import org.jclouds.vcloud.director.v1_5.AbstractVAppClientLiveTest; -import org.jclouds.vcloud.director.v1_5.VCloudDirectorException; import org.jclouds.vcloud.director.v1_5.VCloudDirectorMediaType; import org.jclouds.vcloud.director.v1_5.domain.AccessSetting; import org.jclouds.vcloud.director.v1_5.domain.Checks; @@ -1286,11 +1285,7 @@ public class VAppClientLiveTest extends AbstractVAppClientLiveTest { Task deleteVApp = vAppClient.deleteVApp(temp.getHref()); assertTrue(retryTaskSuccess.apply(deleteVApp), String.format(TASK_COMPLETE_TIMELY, "deleteVApp")); - try { - vAppClient.getVApp(temp.getHref()); - fail("The VApp "+temp+" should have been deleted"); - } catch (VCloudDirectorException vcde) { - assertEquals(vcde.getError().getMajorErrorCode(), Integer.valueOf(403), "The error code for deleted vApp should have been 'Forbidden' (403)"); - } + VApp deleted = vAppClient.getVApp(temp.getHref()); + assertNull(deleted, "The VApp "+temp.getName()+" should have been deleted"); } } From d30f6548eef059b5aa0e98fd5d2fe771a77f3d1c Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Mon, 16 Apr 2012 10:47:44 +0100 Subject: [PATCH 12/50] Use Status enum not value --- .../v1_5/internal/BaseVCloudDirectorClientLiveTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java index 44cbe2b2b3..4995d55bca 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java @@ -450,7 +450,7 @@ public abstract class BaseVCloudDirectorClientLiveTest extends BaseContextLiveTe } // Shutdown and power off the VApp if necessary - if (vApp.getStatus().equals(Status.POWERED_ON.getValue())) { + if (vApp.getStatus() == Status.POWERED_ON) { try { Task shutdownTask = vAppClient.shutdown(vAppURI); taskDoneEventually(shutdownTask); From 18c6313eb41a2a04df4f9b2f255c2948701089f4 Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Mon, 16 Apr 2012 10:48:49 +0100 Subject: [PATCH 13/50] Change context return type --- .../director/v1_5/internal/VCloudDirectorTestSession.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java index ed8bc0a351..bb367b38ed 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java @@ -1,13 +1,8 @@ package org.jclouds.vcloud.director.v1_5.internal; -import static com.google.common.base.Objects.equal; -import static org.jclouds.vcloud.director.v1_5.VCloudDirectorLiveTestConstants.REF_REQ_LIVE; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; import java.io.Closeable; -import java.net.URI; import java.util.Properties; import org.jclouds.ContextBuilder; @@ -118,7 +113,7 @@ public class VCloudDirectorTestSession implements Closeable { Closeables.closeQuietly(adminContext); } - public RestContext getUserContext() { + public VCloudDirectorContext getUserContext() { return userContext; } From 27fb042c113c9d3cd874525399c5d9d58505d6da Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Mon, 16 Apr 2012 10:49:14 +0100 Subject: [PATCH 14/50] Misc --- .../vcloud/director/v1_5/AbstractVAppClientLiveTest.java | 2 +- .../v1_5/internal/BaseVCloudDirectorClientLiveTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/AbstractVAppClientLiveTest.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/AbstractVAppClientLiveTest.java index df302c2c27..a0d47cb00c 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/AbstractVAppClientLiveTest.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/AbstractVAppClientLiveTest.java @@ -246,7 +246,7 @@ public abstract class AbstractVAppClientLiveTest extends BaseVCloudDirectorClien } /** - * Power on a VApp. + * Power on a {@link VApp}s {@link Vm}s. */ protected VApp powerOn(final URI testVAppURI) { VApp testVApp = vAppClient.getVApp(testVAppURI); diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java index 4995d55bca..6489316f66 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/BaseVCloudDirectorClientLiveTest.java @@ -405,7 +405,7 @@ public abstract class BaseVCloudDirectorClientLiveTest extends BaseContextLiveTe // Build the configuration object NetworkConfiguration networkConfiguration = NetworkConfiguration.builder() .parentNetwork(parentNetwork.get()) - .fenceMode(Network.FenceMode.ISOLATED) + .fenceMode(Network.FenceMode.BRIDGED) .build(); return networkConfiguration; From d7bf98017b1864b843b8b2d64aba2f1136c55775 Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Mon, 16 Apr 2012 16:23:08 +0100 Subject: [PATCH 15/50] Fixup session and login live testing --- .../director/v1_5/HttpClientLiveTest.java | 3 +- .../v1_5/login/SessionClientLiveTest.java | 29 ++++--------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/HttpClientLiveTest.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/HttpClientLiveTest.java index d924ddb906..f590ff30d6 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/HttpClientLiveTest.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/HttpClientLiveTest.java @@ -32,7 +32,6 @@ import org.jclouds.http.HttpResponse; import org.jclouds.util.Strings2; import org.jclouds.vcloud.director.v1_5.domain.SessionWithToken; import org.jclouds.vcloud.director.v1_5.domain.org.OrgList; -import org.jclouds.vcloud.director.v1_5.features.admin.AdminCatalogClient; import org.jclouds.vcloud.director.v1_5.internal.BaseVCloudDirectorClientLiveTest; import org.jclouds.xml.internal.JAXBParser; import org.testng.annotations.Test; @@ -41,7 +40,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Iterables; /** - * Tests live behavior of {@link AdminCatalogClient}. + * Tests live behavior of operations that use {@link HttpClient}. * * @author danikov */ diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/login/SessionClientLiveTest.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/login/SessionClientLiveTest.java index f733e733d4..f2c7be0033 100644 --- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/login/SessionClientLiveTest.java +++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/login/SessionClientLiveTest.java @@ -23,9 +23,7 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import java.net.URI; -import java.util.Properties; -import org.jclouds.Constants; import org.jclouds.apis.ApiMetadata; import org.jclouds.apis.BaseContextLiveTest; import org.jclouds.rest.AnonymousRestApiMetadata; @@ -39,18 +37,20 @@ import org.testng.annotations.Test; import com.google.common.reflect.TypeToken; /** - * Tests behavior of {@code SessionClient}. Note this class is tested completely independently of - * VCloudClient as it is a dependency of the VCloud context working. + * Tests behavior of {@link SessionClient}. Note this class is tested completely independently of + * {@link VCloudDirectorClient} as it is a dependency of the {@code vcloud-director} context working. * * @author Adrian Cole */ @Listeners(FormatApiResultsListener.class) -@Test(groups = { "live", "user", "login" }, testName = "SessionClientLiveTest") +@Test(groups = { "live", "user" }, testName = "SessionClientLiveTest") public class SessionClientLiveTest extends BaseContextLiveTest> { + public SessionClientLiveTest() { provider = "vcloud-director"; } + @Override @BeforeGroups(groups = { "live" }) public void setupContext() { super.setupContext(); @@ -60,25 +60,6 @@ public class SessionClientLiveTest extends BaseContextLiveTest Date: Thu, 19 Apr 2012 00:12:56 +0100 Subject: [PATCH 16/50] Fixing some issues with XmlEnum annotated types --- .../domain/network/NetworkConnection.java | 32 ++++++++++--------- .../domain/params/UndeployVAppParams.java | 18 ++++++++--- .../vcloud/director/v1_5/domain/Checks.java | 13 ++++---- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/domain/network/NetworkConnection.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/domain/network/NetworkConnection.java index 60f71eb726..4c5303552d 100644 --- a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/domain/network/NetworkConnection.java +++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/domain/network/NetworkConnection.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to jclouds, Inc. (jclouds) under one or more * contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -37,15 +37,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; - /** * Represents a network connection. - *

- *

- *

Java class for NetworkConnection complex type. - *

- *

The following schema fragment specifies the expected content contained within this class. - *

+ * * 

  * <complexType name="NetworkConnection">
  *   <complexContent>
@@ -83,7 +77,7 @@ public class NetworkConnection {
       @XmlEnumValue("dhcp") DHCP("dhcp"),
       @XmlEnumValue("manual") MANUAL("manual"),
       @XmlEnumValue("none") NONE("none"),
-      UNRECOGNIZED("unrecognized");
+      @XmlEnumValue("") UNRECOGNIZED("unrecognized");
       
       public static final List ALL = ImmutableList.of(POOL, DHCP, MANUAL, NONE);
 
@@ -126,7 +120,7 @@ public class NetworkConnection {
       private String externalIpAddress;
       private boolean isConnected;
       private String macAddress;
-      private String ipAddressAllocationMode;
+      private IpAddressAllocationMode ipAddressAllocationMode;
       private String network;
       private Boolean needsCustomization;
 
@@ -173,11 +167,19 @@ public class NetworkConnection {
       /**
        * @see NetworkConnection#getIpAddressAllocationMode()
        */
-      public Builder ipAddressAllocationMode(String ipAddressAllocationMode) {
+      public Builder ipAddressAllocationMode(IpAddressAllocationMode ipAddressAllocationMode) {
          this.ipAddressAllocationMode = ipAddressAllocationMode;
          return this;
       }
 
+      /**
+       * @see NetworkConnection#getIpAddressAllocationMode()
+       */
+      public Builder ipAddressAllocationMode(String ipAddressAllocationMode) {
+         this.ipAddressAllocationMode = IpAddressAllocationMode.valueOf(ipAddressAllocationMode);
+         return this;
+      }
+
       /**
        * @see NetworkConnection#getNetwork()
        */
@@ -214,11 +216,11 @@ public class NetworkConnection {
    }
 
    public NetworkConnection(int networkConnectionIndex, String ipAddress, String externalIpAddress, boolean connected,
-                            String macAddress, String ipAddressAllocationMode, String network, Boolean needsCustomization) {
+                            String macAddress, IpAddressAllocationMode ipAddressAllocationMode, String network, Boolean needsCustomization) {
       this.networkConnectionIndex = networkConnectionIndex;
       this.ipAddress = ipAddress;
       this.externalIpAddress = externalIpAddress;
-      isConnected = connected;
+      this.isConnected = connected;
       this.macAddress = macAddress;
       this.ipAddressAllocationMode = ipAddressAllocationMode;
       this.network = network;
@@ -241,7 +243,7 @@ public class NetworkConnection {
    @XmlElement(name = "MACAddress")
    protected String macAddress;
    @XmlElement(name = "IpAddressAllocationMode", required = true)
-   protected String ipAddressAllocationMode;
+   protected IpAddressAllocationMode ipAddressAllocationMode;
    @XmlAttribute(required = true)
    protected String network;
    @XmlAttribute
@@ -297,7 +299,7 @@ public class NetworkConnection {
     * @return possible object is
     *         {@link String }
     */
-   public String getIpAddressAllocationMode() {
+   public IpAddressAllocationMode getIpAddressAllocationMode() {
       return ipAddressAllocationMode;
    }
 
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/domain/params/UndeployVAppParams.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/domain/params/UndeployVAppParams.java
index dcb37b6ab9..67ac098e49 100644
--- a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/domain/params/UndeployVAppParams.java
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/domain/params/UndeployVAppParams.java
@@ -59,7 +59,7 @@ public class UndeployVAppParams {
       @XmlEnumValue("suspend") SUSPEND("suspend"),
       @XmlEnumValue("shutdown") SHUTDOWN("shutdown"),
       @XmlEnumValue("force") FORCE("force"),
-      UNRECOGNIZED("unrecognized");
+      @XmlEnumValue("") UNRECOGNIZED("unrecognized");
       
       public static final List ALL = ImmutableList.of( POWER_OFF, SUSPEND, SHUTDOWN, FORCE );
 
@@ -97,13 +97,21 @@ public class UndeployVAppParams {
 
    public static class Builder {
 
-      private String undeployPowerAction;
+      private PowerAction undeployPowerAction;
+
+      /**
+       * @see UndeployVAppParams#getUndeployPowerAction()
+       */
+      public Builder undeployPowerAction(PowerAction undeployPowerAction) {
+         this.undeployPowerAction = undeployPowerAction;
+         return this;
+      }
 
       /**
        * @see UndeployVAppParams#getUndeployPowerAction()
        */
       public Builder undeployPowerAction(String undeployPowerAction) {
-         this.undeployPowerAction = undeployPowerAction;
+         this.undeployPowerAction = PowerAction.valueOf(undeployPowerAction);
          return this;
       }
 
@@ -123,7 +131,7 @@ public class UndeployVAppParams {
    }
 
    @XmlElement(name = "UndeployPowerAction")
-   protected String undeployPowerAction;
+   protected PowerAction undeployPowerAction;
 
    /**
     * The specified action is applied to all VMs in the vApp.
@@ -140,7 +148,7 @@ public class UndeployVAppParams {
     *
     * @since 1.5
     */
-   public String getUndeployPowerAction() {
+   public PowerAction getUndeployPowerAction() {
       return undeployPowerAction;
    }
 
diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/domain/Checks.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/domain/Checks.java
index 15b9f8a2dd..2f4345c595 100644
--- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/domain/Checks.java
+++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/domain/Checks.java
@@ -36,6 +36,7 @@ import static org.jclouds.vcloud.director.v1_5.VCloudDirectorLiveTestConstants.R
 import static org.jclouds.vcloud.director.v1_5.VCloudDirectorLiveTestConstants.REQUIRED_VALUE_OBJECT_FMT;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotEquals;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
@@ -73,23 +74,22 @@ import org.jclouds.vcloud.director.v1_5.domain.network.NetworkServiceType;
 import org.jclouds.vcloud.director.v1_5.domain.network.RouterInfo;
 import org.jclouds.vcloud.director.v1_5.domain.network.SyslogServerSettings;
 import org.jclouds.vcloud.director.v1_5.domain.network.VAppNetworkConfiguration;
-import org.jclouds.vcloud.director.v1_5.domain.network.NetworkConnection.IpAddressAllocationMode;
 import org.jclouds.vcloud.director.v1_5.domain.org.AdminOrg;
 import org.jclouds.vcloud.director.v1_5.domain.org.CustomOrgLdapSettings;
+import org.jclouds.vcloud.director.v1_5.domain.org.CustomOrgLdapSettings.AuthenticationMechanism;
+import org.jclouds.vcloud.director.v1_5.domain.org.CustomOrgLdapSettings.ConnectorType;
 import org.jclouds.vcloud.director.v1_5.domain.org.Org;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgEmailSettings;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgGeneralSettings;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgLdapGroupAttributes;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgLdapSettings;
+import org.jclouds.vcloud.director.v1_5.domain.org.OrgLdapSettings.LdapMode;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgLdapUserAttributes;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgLeaseSettings;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgNetwork;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgPasswordPolicySettings;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgSettings;
 import org.jclouds.vcloud.director.v1_5.domain.org.OrgVAppTemplateLeaseSettings;
-import org.jclouds.vcloud.director.v1_5.domain.org.CustomOrgLdapSettings.AuthenticationMechanism;
-import org.jclouds.vcloud.director.v1_5.domain.org.CustomOrgLdapSettings.ConnectorType;
-import org.jclouds.vcloud.director.v1_5.domain.org.OrgLdapSettings.LdapMode;
 import org.jclouds.vcloud.director.v1_5.domain.params.ControlAccessParams;
 import org.jclouds.vcloud.director.v1_5.domain.query.ContainerType;
 import org.jclouds.vcloud.director.v1_5.domain.query.QueryResultRecordType;
@@ -1255,9 +1255,8 @@ public class Checks {
       // Check required fields
       assertNotNull(val.getNetwork(), String.format(NOT_NULL_OBJ_FIELD_FMT, "Network", "NetworkConnection"));
       assertNotNull(val.getIpAddressAllocationMode(), String.format(NOT_NULL_OBJ_FIELD_FMT, "IpAddressAllocationMode", "NetworkConnection"));
-      IpAddressAllocationMode mode = NetworkConnection.IpAddressAllocationMode.valueOf(val.getIpAddressAllocationMode());
-      assertTrue(NetworkConnection.IpAddressAllocationMode.ALL.contains(mode), 
-               String.format(REQUIRED_VALUE_OBJECT_FMT, "IpAddressAllocationMode", "NetworkConnection", val.getIpAddressAllocationMode(), Iterables.toString(NetworkConnection.IpAddressAllocationMode.ALL)));
+      assertNotEquals(val.getIpAddressAllocationMode(), NetworkConnection.IpAddressAllocationMode.UNRECOGNIZED,
+            String.format(REQUIRED_VALUE_OBJECT_FMT, "IpAddressAllocationMode", "NetworkConnection", val.getIpAddressAllocationMode(), Iterables.toString(NetworkConnection.IpAddressAllocationMode.ALL)));
       
       // Check optional fields
       if (val.getIpAddress() != null) {

From 6b10fb8f3251e35959039bdae8b5655901b144f7 Mon Sep 17 00:00:00 2001
From: Andrew Donald Kennedy 
Date: Thu, 19 Apr 2012 00:13:54 +0100
Subject: [PATCH 17/50] Problem when instantiating the SessionClient via
 ContextBuilder, solved by forcing the getProviderMetadata override method to
 be used

---
 .../v1_5/VCloudDirectorConstants.java         | 13 +++++++++
 .../internal/VCloudDirectorTestSession.java   |  2 --
 .../v1_5/login/SessionClientLiveTest.java     | 29 ++++++++++++-------
 3 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/VCloudDirectorConstants.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/VCloudDirectorConstants.java
index 3ac2dee73b..91088d307e 100644
--- a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/VCloudDirectorConstants.java
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/VCloudDirectorConstants.java
@@ -18,6 +18,12 @@
  */
 package org.jclouds.vcloud.director.v1_5;
 
+import org.jclouds.rest.RestContext;
+import org.jclouds.vcloud.director.v1_5.login.SessionAsyncClient;
+import org.jclouds.vcloud.director.v1_5.login.SessionClient;
+
+import com.google.common.reflect.TypeToken;
+
 /**
  * Constants used by VCloudDirector clients
  *
@@ -63,4 +69,11 @@ public class VCloudDirectorConstants {
 
    /** TODO javadoc */
    public static final String PROPERTY_NS_NAME_LEN_MAX = "jclouds.dns_name_length_max";
+
+   /** TODO javadoc */
+   public static final TypeToken> SESSION_CONTEXT_TYPE =
+         new TypeToken>() {
+				/** The serialVersionUID */
+				private static final long serialVersionUID = -3625362618882122604L;
+		   };
 }
diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java
index bb367b38ed..4199435c16 100644
--- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java
+++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/internal/VCloudDirectorTestSession.java
@@ -19,8 +19,6 @@ import org.jclouds.vcloud.director.v1_5.domain.Role.DefaultRoles;
 import org.jclouds.vcloud.director.v1_5.domain.User;
 import org.jclouds.vcloud.director.v1_5.predicates.LinkPredicates;
 import org.jclouds.vcloud.director.v1_5.predicates.ReferencePredicates;
-import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorAsyncClient;
-import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorClient;
 
 import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableSet;
diff --git a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/login/SessionClientLiveTest.java b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/login/SessionClientLiveTest.java
index f2c7be0033..ba581fd874 100644
--- a/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/login/SessionClientLiveTest.java
+++ b/labs/vcloud-director/src/test/java/org/jclouds/vcloud/director/v1_5/login/SessionClientLiveTest.java
@@ -24,11 +24,13 @@ import static org.testng.Assert.assertTrue;
 
 import java.net.URI;
 
-import org.jclouds.apis.ApiMetadata;
+import org.jclouds.ContextBuilder;
 import org.jclouds.apis.BaseContextLiveTest;
-import org.jclouds.rest.AnonymousRestApiMetadata;
+import org.jclouds.providers.AnonymousProviderMetadata;
+import org.jclouds.providers.ProviderMetadata;
 import org.jclouds.rest.RestContext;
 import org.jclouds.vcloud.director.testng.FormatApiResultsListener;
+import org.jclouds.vcloud.director.v1_5.VCloudDirectorConstants;
 import org.jclouds.vcloud.director.v1_5.domain.SessionWithToken;
 import org.testng.annotations.BeforeGroups;
 import org.testng.annotations.Listeners;
@@ -84,17 +86,22 @@ public class SessionClientLiveTest extends BaseContextLiveTest> contextType() {
-      return new TypeToken>(){
-
-         /** The serialVersionUID */
-         private static final long serialVersionUID = -3625362618882122604L;};
+      return VCloudDirectorConstants.SESSION_CONTEXT_TYPE;
    }
+
+   @Override
+   protected ProviderMetadata createProviderMetadata() {
+      return AnonymousProviderMetadata.forClientMappedToAsyncClientOnEndpoint(SessionClient.class, SessionAsyncClient.class, endpoint);
+   }
+
+   @Override
+   protected ContextBuilder newBuilder() {
+      ProviderMetadata pm = createProviderMetadata();
+      ContextBuilder builder = ContextBuilder.newBuilder(pm);
+      return builder;
+   }
+
 }

From 7dbba2a4e1aeafd4b075275f868eb236d6dfac38 Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 07:20:56 +0100
Subject: [PATCH 18/50] Adjusting iso8601SecondsDateParse to replace ' ' with
 'T" in the same manner as iso8601DateParse

---
 .../org/jclouds/date/internal/SimpleDateFormatDateService.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/main/java/org/jclouds/date/internal/SimpleDateFormatDateService.java b/core/src/main/java/org/jclouds/date/internal/SimpleDateFormatDateService.java
index 78b54a24d6..b6a0108dc8 100644
--- a/core/src/main/java/org/jclouds/date/internal/SimpleDateFormatDateService.java
+++ b/core/src/main/java/org/jclouds/date/internal/SimpleDateFormatDateService.java
@@ -157,6 +157,8 @@ public class SimpleDateFormatDateService implements DateService {
       toParse = trimToMillis(toParse);
       toParse = trimTZ(toParse);
       toParse += tz;
+      if (toParse.charAt(10) == ' ')
+         toParse = new StringBuilder(toParse).replace(10, 11, "T").toString();
       synchronized (iso8601SecondsSimpleDateFormat) {
          try {
             return iso8601SecondsSimpleDateFormat.parse(toParse);

From eef27bbe4ee0f60dbfc7d5e7be588fd3dad91f3a Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 07:24:56 +0100
Subject: [PATCH 19/50] Adding full extension list

---
 .../v1_1/internal/BaseNovaExpectTest.java     |   2 +-
 .../test/resources/extension_list_full.json   | 123 ++++++++++++++++++
 2 files changed, 124 insertions(+), 1 deletion(-)
 create mode 100644 apis/openstack-nova/src/test/resources/extension_list_full.json

diff --git a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/internal/BaseNovaExpectTest.java b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/internal/BaseNovaExpectTest.java
index c311dbcbe0..336f507ba7 100644
--- a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/internal/BaseNovaExpectTest.java
+++ b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/internal/BaseNovaExpectTest.java
@@ -63,7 +63,7 @@ public class BaseNovaExpectTest extends BaseRestClientExpectTest {
                         .put("X-Auth-Token", authToken).build()).build();
 
       extensionsOfNovaResponse = HttpResponse.builder().statusCode(200)
-            .payload(payloadFromResource("/extension_list_normal.json")).build();
+            .payload(payloadFromResource("/extension_list_full.json")).build();
       
       unmatchedExtensionsOfNovaResponse = HttpResponse.builder().statusCode(200)
             .payload(payloadFromResource("/extension_list.json")).build();
diff --git a/apis/openstack-nova/src/test/resources/extension_list_full.json b/apis/openstack-nova/src/test/resources/extension_list_full.json
new file mode 100644
index 0000000000..298a2f2f4a
--- /dev/null
+++ b/apis/openstack-nova/src/test/resources/extension_list_full.json
@@ -0,0 +1,123 @@
+{
+    "extensions": [{
+        "updated": "2011-06-09T00:00:00+00:00",
+        "name": "Multinic",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/multinic/api/v1.1",
+        "alias": "NMN",
+        "description": "Multiple network support"
+    }, {
+        "updated": "2011-06-29T00:00:00+00:00",
+        "name": "Hosts",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/hosts/api/v1.1",
+        "alias": "os-hosts",
+        "description": "Host administration"
+    }, {
+        "updated": "2011-03-25T00:00:00+00:00",
+        "name": "Volumes",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/volumes/api/v1.1",
+        "alias": "os-volumes",
+        "description": "Volumes support"
+    }, {
+        "updated": "2011-05-25 16:12:21.656723",
+        "name": "Admin Controller",
+        "links": [],
+        "namespace": "https:TODO/",
+        "alias": "ADMIN",
+        "description": "The Admin API Extension"
+    }, {
+        "updated": "2011-08-08T00:00:00+00:00",
+        "name": "Quotas",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/quotas-sets/api/v1.1",
+        "alias": "os-quota-sets",
+        "description": "Quotas management support"
+    }, {
+        "updated": "2011-08-24T00:00:00+00:00",
+        "name": "VolumeTypes",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/volume_types/api/v1.1",
+        "alias": "os-volume-types",
+        "description": "Volume types support"
+    }, {
+        "updated": "2011-06-23T00:00:00+00:00",
+        "name": "FlavorExtraSpecs",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/flavor_extra_specs/api/v1.1",
+        "alias": "os-flavor-extra-specs",
+        "description": "Instance type (flavor) extra specs"
+    }, {
+        "updated": "2011-09-14T00:00:00+00:00",
+        "name": "FlavorExtraData",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/flavor_extra_data/api/v1.1",
+        "alias": "os-flavor-extra-data",
+        "description": "Provide additional data for flavors"
+    }, {
+        "updated": "2011-08-17T00:00:00+00:00",
+        "name": "VirtualInterfaces",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/virtual_interfaces/api/v1.1",
+        "alias": "virtual_interfaces",
+        "description": "Virtual interface support"
+    }, {
+        "updated": "2011-07-19T00:00:00+00:00",
+        "name": "Createserverext",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/createserverext/api/v1.1",
+        "alias": "os-create-server-ext",
+        "description": "Extended support to the Create Server v1.1 API"
+    }, {
+        "updated": "2011-08-08T00:00:00+00:00",
+        "name": "Keypairs",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/keypairs/api/v1.1",
+        "alias": "os-keypairs",
+        "description": "Keypair Support"
+    }, {
+        "updated": "2011-08-25T00:00:00+00:00",
+        "name": "VSAs",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/vsa/api/v1.1",
+        "alias": "zadr-vsa",
+        "description": "Virtual Storage Arrays support"
+    }, {
+        "updated": "2011-08-19T00:00:00+00:00",
+        "name": "SimpleTenantUsage",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/os-simple-tenant-usage/api/v1.1",
+        "alias": "os-simple-tenant-usage",
+        "description": "Simple tenant usage extension"
+    }, {
+        "updated": "2011-08-18T00:00:00+00:00",
+        "name": "Rescue",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/rescue/api/v1.1",
+        "alias": "os-rescue",
+        "description": "Instance rescue mode"
+    }, {
+        "updated": "2011-07-21T00:00:00+00:00",
+        "name": "SecurityGroups",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/securitygroups/api/v1.1",
+        "alias": "security_groups",
+        "description": "Security group support"
+    }, {
+        "updated": "2011-06-16T00:00:00+00:00",
+        "name": "Floating_ips",
+        "links": [],
+        "namespace": "https://docs.openstack.org/ext/floating_ips/api/v1.1",
+        "alias": "os-floating-ips",
+        "description": "Floating IPs support"
+    }, {
+        "updated": "2011-06-16T00:00:00+00:00",
+        "name": "Users",
+        "links": [],
+        "namespace": "http://docs.openstack.org/compute/ext/users/api/v1.1",
+        "alias": "os-users",
+        "description": "Users support"
+    }
+    ]
+}
\ No newline at end of file

From 400221820a555012df60ff84bbc337705b09e4dc Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 07:25:41 +0100
Subject: [PATCH 20/50] Adding Host Administration extension

---
 .../openstack/nova/v1_1/NovaAsyncClient.java  |  10 ++
 .../openstack/nova/v1_1/NovaClient.java       |   9 +
 .../v1_1/config/NovaRestClientModule.java     |   6 +-
 .../openstack/nova/v1_1/domain/Host.java      | 126 ++++++++++++++
 .../nova/v1_1/domain/HostResourceUsage.java   | 163 ++++++++++++++++++
 .../HostAdministrationAsyncClient.java        |  78 +++++++++
 .../extensions/HostAdministrationClient.java  |  55 ++++++
 ...paceEqualsAnyNamespaceInExtensionsSet.java |   6 +-
 .../src/test/resources/host.json              |   1 +
 .../src/test/resources/hosts_list.json        |   1 +
 10 files changed, 453 insertions(+), 2 deletions(-)
 create mode 100644 apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/Host.java
 create mode 100644 apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/HostResourceUsage.java
 create mode 100644 apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationAsyncClient.java
 create mode 100644 apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClient.java
 create mode 100644 apis/openstack-nova/src/test/resources/host.json
 create mode 100644 apis/openstack-nova/src/test/resources/hosts_list.json

diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
index cb8a4864df..263d9ec745 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
@@ -24,8 +24,10 @@ import org.jclouds.javax.annotation.Nullable;
 import org.jclouds.location.Zone;
 import org.jclouds.location.functions.ZoneToEndpoint;
 import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPAsyncClient;
+import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.KeyPairAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupAsyncClient;
+import org.jclouds.openstack.nova.v1_1.extensions.SimpleTenantUsageAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.ExtensionAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.FlavorAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.ImageAsyncClient;
@@ -103,4 +105,12 @@ public interface NovaAsyncClient {
    @Delegate
    Optional getKeyPairExtensionForZone(
             @EndpointParam(parser = ZoneToEndpoint.class) @Nullable String zone);
+
+   /**
+    * Provides asynchronous access to Host Administration features.
+    */
+   @Delegate
+   Optional getHostAdministrationExtensionForZone(
+         @EndpointParam(parser = ZoneToEndpoint.class) @Nullable String zone);
+
 }
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
index 65725174da..2ffdf21b02 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
@@ -26,8 +26,10 @@ import org.jclouds.javax.annotation.Nullable;
 import org.jclouds.location.Zone;
 import org.jclouds.location.functions.ZoneToEndpoint;
 import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPClient;
+import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationClient;
 import org.jclouds.openstack.nova.v1_1.extensions.KeyPairClient;
 import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupClient;
+import org.jclouds.openstack.nova.v1_1.extensions.SimpleTenantUsageClient;
 import org.jclouds.openstack.nova.v1_1.features.ExtensionClient;
 import org.jclouds.openstack.nova.v1_1.features.FlavorClient;
 import org.jclouds.openstack.nova.v1_1.features.ImageClient;
@@ -106,4 +108,11 @@ public interface NovaClient {
    Optional getKeyPairExtensionForZone(
          @EndpointParam(parser = ZoneToEndpoint.class) @Nullable String zone);
 
+   /**
+    * Provides synchronous access to Host Administration features.
+    */
+   @Delegate
+   Optional getHostAdministrationExtensionForZone(
+         @EndpointParam(parser = ZoneToEndpoint.class) @Nullable String zone);
+
 }
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaRestClientModule.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaRestClientModule.java
index c9f603d178..559cabaae7 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaRestClientModule.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaRestClientModule.java
@@ -37,6 +37,8 @@ import org.jclouds.openstack.nova.v1_1.NovaClient;
 import org.jclouds.openstack.nova.v1_1.domain.Extension;
 import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPClient;
+import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationAsyncClient;
+import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationClient;
 import org.jclouds.openstack.nova.v1_1.extensions.KeyPairAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.KeyPairClient;
 import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupAsyncClient;
@@ -75,7 +77,9 @@ public class NovaRestClientModule extends RestClientModule builder() {
+      return new ConcreteBuilder();
+   }
+
+   public Builder toBuilder() {
+      return new ConcreteBuilder().fromHost(this);
+   }
+
+   public static abstract class Builder>  {
+      protected abstract T self();
+
+      private String name;
+      private String service;
+
+      public T name(String name) {
+         this.name = name;
+         return self();
+      }
+
+      public T service(String service) {
+         this.service = service;
+         return self();
+      }
+
+      public Host build() {
+         return new Host(this);
+      }
+
+      public T fromHost(Host in) {
+         return this
+               .name(in.getName())
+               .service(in.getService())
+               ;
+      }
+
+   }
+
+   private static class ConcreteBuilder extends Builder {
+      @Override
+      protected ConcreteBuilder self() {
+         return this;
+      }
+   }
+
+   @SerializedName(value="host_name")
+   private final String name;
+   private final String service;
+
+   protected Host(Builder builder) {
+      this.name = builder.name;
+      this.service = builder.service;
+   }
+
+   /**
+    */
+   @Nullable
+   public String getName() {
+      return this.name;
+   }
+
+   /**
+    */
+   @Nullable
+   public String getService() {
+      return this.service;
+   }
+
+   @Override
+   public int hashCode() {
+      return Objects.hashCode(name, service);
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      if (this == obj) return true;
+      if (obj == null || getClass() != obj.getClass()) return false;
+      Host that = Host.class.cast(obj);
+      return Objects.equal(this.name, that.name)
+            && Objects.equal(this.service, that.service)
+            ;
+   }
+
+   protected ToStringHelper string() {
+      return Objects.toStringHelper("")
+            .add("name", name)
+            .add("service", service)
+            ;
+   }
+
+   @Override
+   public String toString() {
+      return string().toString();
+   }
+
+}
\ No newline at end of file
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/HostResourceUsage.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/HostResourceUsage.java
new file mode 100644
index 0000000000..95e2b76571
--- /dev/null
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/HostResourceUsage.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.domain;
+
+import org.jclouds.javax.annotation.Nullable;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Objects.ToStringHelper;
+import com.google.gson.annotations.SerializedName;
+
+/**
+ * Class HostResourceUsage
+ */
+public class HostResourceUsage {
+
+   public static Builder builder() {
+      return new ConcreteBuilder();
+   }
+
+   public Builder toBuilder() {
+      return new ConcreteBuilder().fromHostResourceUsage(this);
+   }
+
+   public static abstract class Builder> {
+      protected abstract T self();
+
+      private String host;
+      private String memoryMb;
+      private int cpu;
+      private int diskGb;
+
+      public T host(String host) {
+         this.host = host;
+         return self();
+      }
+
+      public T memoryMb(String memoryMb) {
+         this.memoryMb = memoryMb;
+         return self();
+      }
+
+      public T cpu(int cpu) {
+         this.cpu = cpu;
+         return self();
+      }
+
+      public T diskGb(int diskGb) {
+         this.diskGb = diskGb;
+         return self();
+      }
+
+      public HostResourceUsage build() {
+         return new HostResourceUsage(this);
+      }
+
+      public T fromHostResourceUsage(HostResourceUsage in) {
+         return this
+               .host(in.getHost())
+               .memoryMb(in.getMemoryMb())
+               .cpu(in.getCpu())
+               .diskGb(in.getDiskGb())
+               ;
+      }
+
+   }
+
+   private static class ConcreteBuilder extends Builder {
+      @Override
+      protected ConcreteBuilder self() {
+         return this;
+      }
+   }
+
+   private final String host;
+   @SerializedName(value = "memory_mb")
+   private final String memoryMb;
+   private final int cpu;
+   @SerializedName(value = "disk_gb")
+   private final int diskGb;
+
+   protected HostResourceUsage(Builder builder) {
+      this.host = builder.host;
+      this.memoryMb = builder.memoryMb;
+      this.cpu = builder.cpu;
+      this.diskGb = builder.diskGb;
+   }
+
+   /**
+    */
+   @Nullable
+   public String getHost() {
+      return this.host;
+   }
+
+   /**
+    */
+   @Nullable
+   public String getMemoryMb() {
+      return this.memoryMb;
+   }
+
+   /**
+    */
+   @Nullable
+   public int getCpu() {
+      return this.cpu;
+   }
+
+   /**
+    */
+   @Nullable
+   public int getDiskGb() {
+      return this.diskGb;
+   }
+
+   @Override
+   public int hashCode() {
+      return Objects.hashCode(host, memoryMb, cpu, diskGb);
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      if (this == obj) return true;
+      if (obj == null || getClass() != obj.getClass()) return false;
+      HostResourceUsage that = HostResourceUsage.class.cast(obj);
+      return Objects.equal(this.host, that.host)
+            && Objects.equal(this.memoryMb, that.memoryMb)
+            && Objects.equal(this.cpu, that.cpu)
+            && Objects.equal(this.diskGb, that.diskGb)
+            ;
+   }
+
+   protected ToStringHelper string() {
+      return Objects.toStringHelper("")
+            .add("host", host)
+            .add("memoryMb", memoryMb)
+            .add("cpu", cpu)
+            .add("diskGb", diskGb)
+            ;
+   }
+
+   @Override
+   public String toString() {
+      return string().toString();
+   }
+
+}
\ No newline at end of file
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationAsyncClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationAsyncClient.java
new file mode 100644
index 0000000000..9e1819f52d
--- /dev/null
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationAsyncClient.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.extensions;
+
+import java.util.Set;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.openstack.filters.AuthenticateRequest;
+import org.jclouds.openstack.nova.v1_1.domain.Host;
+import org.jclouds.openstack.nova.v1_1.domain.HostResourceUsage;
+import org.jclouds.openstack.services.Extension;
+import org.jclouds.openstack.services.ServiceType;
+import org.jclouds.rest.annotations.ExceptionParser;
+import org.jclouds.rest.annotations.RequestFilters;
+import org.jclouds.rest.annotations.SelectJson;
+import org.jclouds.rest.annotations.SkipEncoding;
+import org.jclouds.rest.functions.ReturnEmptySetOnNotFoundOr404;
+import org.jclouds.rest.functions.ReturnNullOnNotFoundOr404;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+/**
+ * Provides asynchronous access to Host Administration features via the REST API.
+ * 

+ *
+ * @author Adam Lowe
+ * @see SimpleTenantUsageClient
+ * @see 
+ * @see 
+ * @see 
+ */
+@Extension(of = ServiceType.COMPUTE, namespace = ExtensionNamespaces.HOSTS)
+@SkipEncoding({'/', '='})
+@RequestFilters(AuthenticateRequest.class)
+public interface HostAdministrationAsyncClient {
+
+   /**
+    * @see HostAdministrationClient#listHosts()
+    */
+   @GET
+   @Path("/os-hosts")
+   @SelectJson("hosts")
+   @Consumes(MediaType.APPLICATION_JSON)
+   @ExceptionParser(ReturnEmptySetOnNotFoundOr404.class)
+   ListenableFuture> listHosts();
+
+   /**
+    * @see HostAdministrationClient#getHostResourceUsage(String)
+    */
+   @GET
+   @Path("/os-hosts/{id}")
+   @SelectJson("host")
+   @Consumes(MediaType.APPLICATION_JSON)
+   @ExceptionParser(ReturnNullOnNotFoundOr404.class)
+   ListenableFuture> getHostResourceUsage(@PathParam("id") String hostId);
+
+}
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClient.java
new file mode 100644
index 0000000000..01b3e9ba29
--- /dev/null
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClient.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.extensions;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import javax.ws.rs.PathParam;
+
+import org.jclouds.concurrent.Timeout;
+import org.jclouds.openstack.nova.v1_1.domain.Host;
+import org.jclouds.openstack.nova.v1_1.domain.HostResourceUsage;
+import org.jclouds.openstack.services.Extension;
+import org.jclouds.openstack.services.ServiceType;
+
+/**
+ * Provides asynchronous access to Host Administration features via the REST API.
+ * 

+ *
+ * @author Adam Lowe
+ * @see HostAdministrationAsyncClient
+ */
+@Extension(of = ServiceType.COMPUTE, namespace = ExtensionNamespaces.HOSTS)
+@Timeout(duration = 180, timeUnit = TimeUnit.SECONDS)
+public interface HostAdministrationClient {
+
+   /**
+    * Returns the list of hosts
+    * @return the usage information
+    */
+   Set listHosts();
+
+   /**
+    * Retrieves the physical/usage resource on a specific host
+    * @return the usage information
+    */
+   Set getHostResourceUsage(@PathParam("id") String hostId);
+
+}
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/functions/PresentWhenExtensionAnnotationNamespaceEqualsAnyNamespaceInExtensionsSet.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/functions/PresentWhenExtensionAnnotationNamespaceEqualsAnyNamespaceInExtensionsSet.java
index 30d365d278..5db7bb9d46 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/functions/PresentWhenExtensionAnnotationNamespaceEqualsAnyNamespaceInExtensionsSet.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/functions/PresentWhenExtensionAnnotationNamespaceEqualsAnyNamespaceInExtensionsSet.java
@@ -43,7 +43,7 @@ import com.google.common.collect.Multimap;
 /**
  * We use the annotation {@link org.jclouds.openstack.services.Extension} to
  * bind a class that is an extension to an extension found in the
- * {@link ExtensionsClient#listExtensions} call.
+ * {@link org.jclouds.openstack.nova.v1_1.features.ExtensionClient#listExtensions} call.
  * 
  * @author Adrian Cole
  * 
@@ -62,6 +62,10 @@ public class PresentWhenExtensionAnnotationNamespaceEqualsAnyNamespaceInExtensio
            URI.create("http://docs.openstack.org/compute/ext/floating_ips/api/v1.1"))
       .put(URI.create(ExtensionNamespaces.KEYPAIRS),
            URI.create("http://docs.openstack.org/compute/ext/keypairs/api/v1.1"))
+      .put(URI.create(ExtensionNamespaces.SIMPLE_TENANT_USAGE),
+           URI.create("http://docs.openstack.org/compute/ext/os-simple-tenant-usage/api/v1.1"))
+      .put(URI.create(ExtensionNamespaces.HOSTS),
+           URI.create("http://docs.openstack.org/compute/ext/hosts/api/v1.1"))
       .build();
    
    @Inject
diff --git a/apis/openstack-nova/src/test/resources/host.json b/apis/openstack-nova/src/test/resources/host.json
new file mode 100644
index 0000000000..ad2e217e24
--- /dev/null
+++ b/apis/openstack-nova/src/test/resources/host.json
@@ -0,0 +1 @@
+{"host": [{"resource": {"project": "(total)", "memory_mb": 16083, "host": "ubuntu", "cpu": 4, "disk_gb": 181}}, {"resource": {"project": "(used_now)", "memory_mb": 3396, "host": "ubuntu", "cpu": 3, "disk_gb": 5}}, {"resource": {"project": "(used_max)", "memory_mb": 6144, "host": "ubuntu", "cpu": 3, "disk_gb": 80}}, {"resource": {"project": "f8535069c3fb404cb61c873b1a0b4921", "memory_mb": 6144, "host": "ubuntu", "cpu": 3, "disk_gb": 80}}]}
\ No newline at end of file
diff --git a/apis/openstack-nova/src/test/resources/hosts_list.json b/apis/openstack-nova/src/test/resources/hosts_list.json
new file mode 100644
index 0000000000..b8a2cbd2a8
--- /dev/null
+++ b/apis/openstack-nova/src/test/resources/hosts_list.json
@@ -0,0 +1 @@
+["hosts": [{"host_name": "ubuntu", "service": "compute"}]
\ No newline at end of file

From 80c48193ee2571fccbd0af1ec296d9a54988351a Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 07:32:11 +0100
Subject: [PATCH 21/50] Adding tests of Host Administration extension

---
 .../extensions/HostAdministrationClient.java  |  1 +
 .../HostAdministrationClientExpectTest.java   | 63 +++++++++++++++++++
 .../HostAdministrationClientLiveTest.java     | 55 ++++++++++++++++
 3 files changed, 119 insertions(+)
 create mode 100644 apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientExpectTest.java
 create mode 100644 apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java

diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClient.java
index 01b3e9ba29..fa23767b6b 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClient.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClient.java
@@ -32,6 +32,7 @@ import org.jclouds.openstack.services.ServiceType;
 /**
  * Provides asynchronous access to Host Administration features via the REST API.
  * 

+ * TODO reboot, shutdown, startup, update
  *
  * @author Adam Lowe
  * @see HostAdministrationAsyncClient
diff --git a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientExpectTest.java b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientExpectTest.java
new file mode 100644
index 0000000000..2c0c0f8b7c
--- /dev/null
+++ b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientExpectTest.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.extensions;
+
+import java.net.URI;
+
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.nova.v1_1.internal.BaseNovaClientExpectTest;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMultimap;
+
+/**
+ * Tests HostAdministrationClient guice wiring and parsing
+ * 
+ * @author Adam Lowe
+ */
+@Test(groups = "unit", testName = "HostAdministrationClientExpectTest")
+public class HostAdministrationClientExpectTest extends BaseNovaClientExpectTest {
+   
+   
+   public void testList() throws Exception {
+      URI endpoint = URI.create("https://compute.north.host/v1.1/3456/os-hosts");
+      HostAdministrationClient client = requestsSendResponses(keystoneAuthWithUsernameAndPassword,
+            responseWithKeystoneAccess, extensionsOfNovaRequest, extensionsOfNovaResponse,
+            HttpRequest.builder().method("GET").headers(ImmutableMultimap.of("Accept", MediaType.APPLICATION_JSON, "X-Auth-Token", authToken))
+                  .endpoint(endpoint).build(),
+            HttpResponse.builder().statusCode(200).build()).getHostAdministrationExtensionForZone("az-1.region-a.geo-1").get();
+      
+      client.listHosts();
+   }
+
+   public void testGet() throws Exception {
+      URI endpoint = URI.create("https://compute.north.host/v1.1/3456/os-hosts/xyz");
+      HostAdministrationClient client = requestsSendResponses(keystoneAuthWithUsernameAndPassword,
+            responseWithKeystoneAccess, extensionsOfNovaRequest, extensionsOfNovaResponse,
+            HttpRequest.builder().method("GET").headers(ImmutableMultimap.of("Accept", MediaType.APPLICATION_JSON, "X-Auth-Token", authToken))
+                  .endpoint(endpoint).build(),
+            HttpResponse.builder().statusCode(200).build()).getHostAdministrationExtensionForZone("az-1.region-a.geo-1").get();
+
+      client.getHostResourceUsage("xyz");
+   }
+
+}
diff --git a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java
new file mode 100644
index 0000000000..019068a118
--- /dev/null
+++ b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.extensions;
+
+import static org.testng.Assert.assertNotNull;
+
+import java.util.Set;
+
+import org.jclouds.openstack.nova.v1_1.domain.Host;
+import org.jclouds.openstack.nova.v1_1.domain.SimpleTenantUsage;
+import org.jclouds.openstack.nova.v1_1.internal.BaseNovaClientLiveTest;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Optional;
+
+/**
+ * Tests behavior of HostAdministrationClient
+ *
+ * @author Adam Lowe
+ */
+@Test(groups = "live", testName = "HostAdministrationClientLiveTest")
+public class HostAdministrationClientLiveTest extends BaseNovaClientLiveTest {
+
+
+   public void testListAndGet() throws Exception {
+      for (String zoneId : novaContext.getApi().getConfiguredZones()) {
+         Optional optClient = novaContext.getApi().getHostAdministrationExtensionForZone(zoneId);
+         if (optClient.isPresent()) {
+            HostAdministrationClient client = optClient.get();
+            Set hosts = client.listHosts();
+            assertNotNull(hosts);
+            for(Host host : hosts) {
+               client.getHostResourceUsage(host.getName());
+            }
+         }
+      }
+   }
+
+}

From 522147a9ce04e2f713a9622a3edcdbfcd2ed6676 Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 07:59:06 +0100
Subject: [PATCH 22/50] Fixing imports for Host Administration extension

---
 .../java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java    | 1 -
 .../main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java    | 1 -
 .../nova/v1_1/extensions/HostAdministrationClientLiveTest.java   | 1 -
 3 files changed, 3 deletions(-)

diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
index 263d9ec745..f323161ebb 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
@@ -27,7 +27,6 @@ import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.KeyPairAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupAsyncClient;
-import org.jclouds.openstack.nova.v1_1.extensions.SimpleTenantUsageAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.ExtensionAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.FlavorAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.ImageAsyncClient;
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
index 2ffdf21b02..52b8b5bdda 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
@@ -29,7 +29,6 @@ import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPClient;
 import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationClient;
 import org.jclouds.openstack.nova.v1_1.extensions.KeyPairClient;
 import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupClient;
-import org.jclouds.openstack.nova.v1_1.extensions.SimpleTenantUsageClient;
 import org.jclouds.openstack.nova.v1_1.features.ExtensionClient;
 import org.jclouds.openstack.nova.v1_1.features.FlavorClient;
 import org.jclouds.openstack.nova.v1_1.features.ImageClient;
diff --git a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java
index 019068a118..1092a5b95d 100644
--- a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java
+++ b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java
@@ -23,7 +23,6 @@ import static org.testng.Assert.assertNotNull;
 import java.util.Set;
 
 import org.jclouds.openstack.nova.v1_1.domain.Host;
-import org.jclouds.openstack.nova.v1_1.domain.SimpleTenantUsage;
 import org.jclouds.openstack.nova.v1_1.internal.BaseNovaClientLiveTest;
 import org.testng.annotations.Test;
 

From 1d09fc3400d549f1d7f2e0c3037d0ad761918cd4 Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 10:30:24 +0100
Subject: [PATCH 23/50] Adding project field to HostResourceUsage

---
 .../nova/v1_1/domain/HostResourceUsage.java   | 52 ++++++++++++-------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/HostResourceUsage.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/HostResourceUsage.java
index 95e2b76571..e8dbbd1301 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/HostResourceUsage.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/HostResourceUsage.java
@@ -18,6 +18,8 @@
  */
 package org.jclouds.openstack.nova.v1_1.domain;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import org.jclouds.javax.annotation.Nullable;
 
 import com.google.common.base.Objects;
@@ -37,11 +39,12 @@ public class HostResourceUsage {
       return new ConcreteBuilder().fromHostResourceUsage(this);
    }
 
-   public static abstract class Builder> {
+   public static abstract class Builder>  {
       protected abstract T self();
 
       private String host;
-      private String memoryMb;
+      private String project;
+      private int memoryMb;
       private int cpu;
       private int diskGb;
 
@@ -50,7 +53,12 @@ public class HostResourceUsage {
          return self();
       }
 
-      public T memoryMb(String memoryMb) {
+      public T project(String project) {
+         this.project = project;
+         return self();
+      }
+
+      public T memoryMb(int memoryMb) {
          this.memoryMb = memoryMb;
          return self();
       }
@@ -72,6 +80,7 @@ public class HostResourceUsage {
       public T fromHostResourceUsage(HostResourceUsage in) {
          return this
                .host(in.getHost())
+               .project(in.getProject())
                .memoryMb(in.getMemoryMb())
                .cpu(in.getCpu())
                .diskGb(in.getDiskGb())
@@ -88,22 +97,23 @@ public class HostResourceUsage {
    }
 
    private final String host;
-   @SerializedName(value = "memory_mb")
-   private final String memoryMb;
+   private final String project;
+   @SerializedName(value="memory_mb")
+   private final int memoryMb;
    private final int cpu;
-   @SerializedName(value = "disk_gb")
+   @SerializedName(value="disk_gb")
    private final int diskGb;
 
    protected HostResourceUsage(Builder builder) {
-      this.host = builder.host;
-      this.memoryMb = builder.memoryMb;
-      this.cpu = builder.cpu;
-      this.diskGb = builder.diskGb;
+      this.host = checkNotNull(builder.host, "host");
+      this.project = builder.project;
+      this.memoryMb = checkNotNull(builder.memoryMb, "memoryMb");
+      this.cpu = checkNotNull(builder.cpu, "cpu");
+      this.diskGb = checkNotNull(builder.diskGb, "diskGb");
    }
 
    /**
     */
-   @Nullable
    public String getHost() {
       return this.host;
    }
@@ -111,27 +121,31 @@ public class HostResourceUsage {
    /**
     */
    @Nullable
-   public String getMemoryMb() {
+   public String getProject() {
+      return this.project;
+   }
+
+   /**
+    */
+   public int getMemoryMb() {
       return this.memoryMb;
    }
 
    /**
     */
-   @Nullable
    public int getCpu() {
       return this.cpu;
    }
 
    /**
     */
-   @Nullable
    public int getDiskGb() {
       return this.diskGb;
    }
 
    @Override
    public int hashCode() {
-      return Objects.hashCode(host, memoryMb, cpu, diskGb);
+      return Objects.hashCode(host, project, memoryMb, cpu, diskGb);
    }
 
    @Override
@@ -140,19 +154,19 @@ public class HostResourceUsage {
       if (obj == null || getClass() != obj.getClass()) return false;
       HostResourceUsage that = HostResourceUsage.class.cast(obj);
       return Objects.equal(this.host, that.host)
+            && Objects.equal(this.project, that.project)
             && Objects.equal(this.memoryMb, that.memoryMb)
             && Objects.equal(this.cpu, that.cpu)
-            && Objects.equal(this.diskGb, that.diskGb)
-            ;
+            && Objects.equal(this.diskGb, that.diskGb);
    }
 
    protected ToStringHelper string() {
       return Objects.toStringHelper("")
             .add("host", host)
+            .add("project", project)
             .add("memoryMb", memoryMb)
             .add("cpu", cpu)
-            .add("diskGb", diskGb)
-            ;
+            .add("diskGb", diskGb);
    }
 
    @Override

From 43d258837d998c1f20f35c936ee8c367a5ba0ee2 Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 10:31:23 +0100
Subject: [PATCH 24/50] Adding gson adaptor for HostResourceUsage

---
 .../nova/v1_1/config/NovaParserModule.java    | 35 ++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaParserModule.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaParserModule.java
index 63209fb4f0..e2a50efcc8 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaParserModule.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaParserModule.java
@@ -25,8 +25,15 @@ import javax.inject.Singleton;
 
 import org.jclouds.json.config.GsonModule;
 import org.jclouds.json.config.GsonModule.DateAdapter;
+import org.jclouds.openstack.nova.v1_1.domain.HostResourceUsage;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
 import com.google.inject.AbstractModule;
 import com.google.inject.Provides;
 
@@ -38,7 +45,7 @@ public class NovaParserModule extends AbstractModule {
    @Provides
    @Singleton
    public Map provideCustomAdapterBindings() {
-      return ImmutableMap. of();
+      return ImmutableMap. of(HostResourceUsage.class, new HostResourceUsageAdapter());
    }
 
    @Override
@@ -46,4 +53,30 @@ public class NovaParserModule extends AbstractModule {
       bind(DateAdapter.class).to(GsonModule.Iso8601DateAdapter.class);
    }
 
+   @Singleton
+   public static class HostResourceUsageAdapter implements JsonSerializer, JsonDeserializer {
+      public HostResourceUsage apply(HostResourceUsageWrapper in) {
+         return in.resource.toBuilder().build();
+      }
+      
+      @Override
+      public HostResourceUsage deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext context) throws JsonParseException {
+         return apply((HostResourceUsageWrapper) context.deserialize(jsonElement, HostResourceUsageWrapper.class));
+      }
+
+      @Override
+      public JsonElement serialize(HostResourceUsage hostResourceUsage, Type type, JsonSerializationContext context) {
+         return context.serialize(hostResourceUsage);
+      }
+      
+      private static class HostResourceUsageWrapper {
+         protected HostResourceUsageInternal resource;
+      }
+      private static class HostResourceUsageInternal extends HostResourceUsage {
+         protected HostResourceUsageInternal(Builder builder) {
+            super(builder);
+         }
+      }
+   }
+
 }

From f45b9ca849fbe85160c162f77b3e3c1b0b8a4c3c Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 10:37:17 +0100
Subject: [PATCH 25/50] Improving Host Administration extension tests

---
 .../HostAdministrationClientExpectTest.java   | 29 ++++++++++++++++---
 .../HostAdministrationClientLiveTest.java     | 10 +++++--
 .../src/test/resources/hosts_list.json        |  2 +-
 3 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientExpectTest.java b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientExpectTest.java
index 2c0c0f8b7c..54d7036994 100644
--- a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientExpectTest.java
+++ b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientExpectTest.java
@@ -18,16 +18,23 @@
  */
 package org.jclouds.openstack.nova.v1_1.extensions;
 
+import static org.testng.Assert.assertEquals;
+
 import java.net.URI;
+import java.util.Set;
 
 import javax.ws.rs.core.MediaType;
 
 import org.jclouds.http.HttpRequest;
 import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.nova.v1_1.domain.Host;
+import org.jclouds.openstack.nova.v1_1.domain.HostResourceUsage;
 import org.jclouds.openstack.nova.v1_1.internal.BaseNovaClientExpectTest;
 import org.testng.annotations.Test;
 
 import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 
 /**
  * Tests HostAdministrationClient guice wiring and parsing
@@ -44,9 +51,16 @@ public class HostAdministrationClientExpectTest extends BaseNovaClientExpectTest
             responseWithKeystoneAccess, extensionsOfNovaRequest, extensionsOfNovaResponse,
             HttpRequest.builder().method("GET").headers(ImmutableMultimap.of("Accept", MediaType.APPLICATION_JSON, "X-Auth-Token", authToken))
                   .endpoint(endpoint).build(),
-            HttpResponse.builder().statusCode(200).build()).getHostAdministrationExtensionForZone("az-1.region-a.geo-1").get();
+            HttpResponse.builder().statusCode(200).payload(payloadFromResource("/hosts_list.json")).build()).getHostAdministrationExtensionForZone("az-1.region-a.geo-1").get();
       
-      client.listHosts();
+      Host expected = Host.builder().name("ubuntu").service("compute").build();
+
+      Set result = client.listHosts();
+      Host host = Iterables.getOnlyElement(result);
+      assertEquals(host.getName(), "ubuntu");
+      assertEquals(host.getService(), "compute");
+
+      assertEquals(host, expected);
    }
 
    public void testGet() throws Exception {
@@ -55,9 +69,16 @@ public class HostAdministrationClientExpectTest extends BaseNovaClientExpectTest
             responseWithKeystoneAccess, extensionsOfNovaRequest, extensionsOfNovaResponse,
             HttpRequest.builder().method("GET").headers(ImmutableMultimap.of("Accept", MediaType.APPLICATION_JSON, "X-Auth-Token", authToken))
                   .endpoint(endpoint).build(),
-            HttpResponse.builder().statusCode(200).build()).getHostAdministrationExtensionForZone("az-1.region-a.geo-1").get();
+            HttpResponse.builder().statusCode(200).payload(payloadFromResource("/host.json")).build()).getHostAdministrationExtensionForZone("az-1.region-a.geo-1").get();
 
-      client.getHostResourceUsage("xyz");
+      Set expected = ImmutableSet.of(
+            HostResourceUsage.builder().memoryMb(16083).project("(total)").cpu(4).diskGb(181).host("ubuntu").build(),
+            HostResourceUsage.builder().memoryMb(3396).project("(used_now)").cpu(3).diskGb(5).host("ubuntu").build(),
+            HostResourceUsage.builder().memoryMb(6144).project("(used_max)").cpu(3).diskGb(80).host("ubuntu").build(),
+            HostResourceUsage.builder().memoryMb(6144).project("f8535069c3fb404cb61c873b1a0b4921").cpu(3).diskGb(80).host("ubuntu").build()
+      );
+
+      assertEquals(client.getHostResourceUsage("xyz"), expected);
    }
 
 }
diff --git a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java
index 1092a5b95d..e79dd89891 100644
--- a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java
+++ b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/HostAdministrationClientLiveTest.java
@@ -18,11 +18,13 @@
  */
 package org.jclouds.openstack.nova.v1_1.extensions;
 
+import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
 
 import java.util.Set;
 
 import org.jclouds.openstack.nova.v1_1.domain.Host;
+import org.jclouds.openstack.nova.v1_1.domain.HostResourceUsage;
 import org.jclouds.openstack.nova.v1_1.internal.BaseNovaClientLiveTest;
 import org.testng.annotations.Test;
 
@@ -36,16 +38,18 @@ import com.google.common.base.Optional;
 @Test(groups = "live", testName = "HostAdministrationClientLiveTest")
 public class HostAdministrationClientLiveTest extends BaseNovaClientLiveTest {
 
-
    public void testListAndGet() throws Exception {
       for (String zoneId : novaContext.getApi().getConfiguredZones()) {
          Optional optClient = novaContext.getApi().getHostAdministrationExtensionForZone(zoneId);
-         if (optClient.isPresent()) {
+         if (optClient.isPresent() && identity.endsWith(":admin")) {
             HostAdministrationClient client = optClient.get();
             Set hosts = client.listHosts();
             assertNotNull(hosts);
             for(Host host : hosts) {
-               client.getHostResourceUsage(host.getName());
+               for (HostResourceUsage usage : client.getHostResourceUsage(host.getName())) {
+                  assertEquals(usage.getHost(), host.getName());
+                  assertNotNull(usage);
+               }
             }
          }
       }
diff --git a/apis/openstack-nova/src/test/resources/hosts_list.json b/apis/openstack-nova/src/test/resources/hosts_list.json
index b8a2cbd2a8..30d377016b 100644
--- a/apis/openstack-nova/src/test/resources/hosts_list.json
+++ b/apis/openstack-nova/src/test/resources/hosts_list.json
@@ -1 +1 @@
-["hosts": [{"host_name": "ubuntu", "service": "compute"}]
\ No newline at end of file
+{"hosts": [{"host_name": "ubuntu", "service": "compute"}]}
\ No newline at end of file

From adc6e2aa935e82e0171de1c602d721780e0386b0 Mon Sep 17 00:00:00 2001
From: Adam Lowe 
Date: Thu, 19 Apr 2012 10:42:30 +0100
Subject: [PATCH 26/50] Adding Simple Tenant Usage extension

---
 .../openstack/nova/v1_1/NovaAsyncClient.java  |   8 +
 .../openstack/nova/v1_1/NovaClient.java       |   8 +
 .../v1_1/config/NovaRestClientModule.java     |  10 +-
 .../nova/v1_1/domain/SimpleServerUsage.java   | 302 ++++++++++++++++++
 .../nova/v1_1/domain/SimpleTenantUsage.java   | 239 ++++++++++++++
 .../SimpleTenantUsageAsyncClient.java         |  77 +++++
 .../extensions/SimpleTenantUsageClient.java   |  53 +++
 .../SimpleTenantUsageClientExpectTest.java    | 110 +++++++
 .../SimpleTenantUsageClientLiveTest.java      |  55 ++++
 .../test/resources/simple_tenant_usage.json   |   1 +
 .../test/resources/simple_tenant_usages.json  |   1 +
 11 files changed, 856 insertions(+), 8 deletions(-)
 create mode 100644 apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/SimpleServerUsage.java
 create mode 100644 apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/SimpleTenantUsage.java
 create mode 100644 apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageAsyncClient.java
 create mode 100644 apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClient.java
 create mode 100644 apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClientExpectTest.java
 create mode 100644 apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClientLiveTest.java
 create mode 100644 apis/openstack-nova/src/test/resources/simple_tenant_usage.json
 create mode 100644 apis/openstack-nova/src/test/resources/simple_tenant_usages.json

diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
index f323161ebb..da9d6507b5 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaAsyncClient.java
@@ -27,6 +27,7 @@ import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.KeyPairAsyncClient;
 import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupAsyncClient;
+import org.jclouds.openstack.nova.v1_1.extensions.SimpleTenantUsageAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.ExtensionAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.FlavorAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.ImageAsyncClient;
@@ -112,4 +113,11 @@ public interface NovaAsyncClient {
    Optional getHostAdministrationExtensionForZone(
          @EndpointParam(parser = ZoneToEndpoint.class) @Nullable String zone);
 
+   /**
+    * Provides asynchronous access to Simple Tenant Usage features.
+    */
+   @Delegate
+   Optional getSimpleTenantUsageExtensionForZone(
+         @EndpointParam(parser = ZoneToEndpoint.class) @Nullable String zone);
+
 }
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
index 52b8b5bdda..d33e0ab91b 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/NovaClient.java
@@ -29,6 +29,7 @@ import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPClient;
 import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationClient;
 import org.jclouds.openstack.nova.v1_1.extensions.KeyPairClient;
 import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupClient;
+import org.jclouds.openstack.nova.v1_1.extensions.SimpleTenantUsageClient;
 import org.jclouds.openstack.nova.v1_1.features.ExtensionClient;
 import org.jclouds.openstack.nova.v1_1.features.FlavorClient;
 import org.jclouds.openstack.nova.v1_1.features.ImageClient;
@@ -114,4 +115,11 @@ public interface NovaClient {
    Optional getHostAdministrationExtensionForZone(
          @EndpointParam(parser = ZoneToEndpoint.class) @Nullable String zone);
 
+   /**
+    * Provides synchronous access to Simple Tenant Usage features.
+    */
+   @Delegate
+   Optional getSimpleTenantUsageExtensionForZone(
+         @EndpointParam(parser = ZoneToEndpoint.class) @Nullable String zone);
+
 }
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaRestClientModule.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaRestClientModule.java
index 559cabaae7..fdb3b039a8 100644
--- a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaRestClientModule.java
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/config/NovaRestClientModule.java
@@ -35,14 +35,7 @@ import org.jclouds.openstack.keystone.v2_0.config.KeystoneAuthenticationModule;
 import org.jclouds.openstack.nova.v1_1.NovaAsyncClient;
 import org.jclouds.openstack.nova.v1_1.NovaClient;
 import org.jclouds.openstack.nova.v1_1.domain.Extension;
-import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPAsyncClient;
-import org.jclouds.openstack.nova.v1_1.extensions.FloatingIPClient;
-import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationAsyncClient;
-import org.jclouds.openstack.nova.v1_1.extensions.HostAdministrationClient;
-import org.jclouds.openstack.nova.v1_1.extensions.KeyPairAsyncClient;
-import org.jclouds.openstack.nova.v1_1.extensions.KeyPairClient;
-import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupAsyncClient;
-import org.jclouds.openstack.nova.v1_1.extensions.SecurityGroupClient;
+import org.jclouds.openstack.nova.v1_1.extensions.*;
 import org.jclouds.openstack.nova.v1_1.features.ExtensionAsyncClient;
 import org.jclouds.openstack.nova.v1_1.features.ExtensionClient;
 import org.jclouds.openstack.nova.v1_1.features.FlavorAsyncClient;
@@ -79,6 +72,7 @@ public class NovaRestClientModule extends RestClientModule builder() {
+      return new ConcreteBuilder();
+   }
+
+   public Builder toBuilder() {
+      return new ConcreteBuilder().fromSimpleServerUsage(this);
+   }
+
+   public static abstract class Builder>  {
+      private String instanceName;
+      private double hours;
+      private double flavorMemoryMb;
+      private double flavorLocalGb;
+      private double flavorVcpus;
+      private String tenantId;
+      private String flavorName;
+      private Date instanceCreated;
+      private Date instanceTerminiated;
+      private Status instanceStatus;
+      private long uptime;
+
+      protected abstract T self();
+
+      public T instanceName(String instanceName) {
+         this.instanceName = instanceName;
+         return self();
+      }
+
+      public T hours(double hours) {
+         this.hours = hours;
+         return self();
+      }
+
+      public T flavorMemoryMb(double flavorMemoryMb) {
+         this.flavorMemoryMb = flavorMemoryMb;
+         return self();
+      }
+
+      public T flavorLocalGb(double flavorLocalGb) {
+         this.flavorLocalGb = flavorLocalGb;
+         return self();
+      }
+
+      public T flavorVcpus(double flavorVcpus) {
+         this.flavorVcpus = flavorVcpus;
+         return self();
+      }
+
+      public T tenantId(String tenantId) {
+         this.tenantId = tenantId;
+         return self();
+      }
+
+      public T flavorName(String flavorName) {
+         this.flavorName = flavorName;
+         return self();
+      }
+
+      public T instanceCreated(Date instanceCreated) {
+         this.instanceCreated = instanceCreated;
+         return self();
+      }
+
+      public T instanceTerminiated(Date instanceTerminiated) {
+         this.instanceTerminiated = instanceTerminiated;
+         return self();
+      }
+
+      public T instanceStatus(Status instanceStatus) {
+         this.instanceStatus = instanceStatus;
+         return self();
+      }
+
+      public T uptime(long uptime) {
+         this.uptime = uptime;
+         return self();
+      }
+
+      public SimpleServerUsage build() {
+         return new SimpleServerUsage(this);
+      }
+
+
+      public T fromSimpleServerUsage(SimpleServerUsage in) {
+         return this
+               .instanceName(in.getInstanceName())
+               .flavorMemoryMb(in.getFlavorMemoryMb())
+               .flavorLocalGb(in.getFlavorLocalGb())
+               .flavorVcpus(in.getFlavorVcpus())
+               .tenantId(in.getTenantId())
+               .flavorName(in.getFlavorName())
+               .instanceCreated(in.getInstanceCreated())
+               .instanceTerminiated(in.getInstanceTerminiated())
+               .instanceStatus(in.getInstanceStatus())
+               .uptime(in.getUptime())
+               ;
+      }
+
+   }
+
+   private static class ConcreteBuilder extends Builder {
+      @Override
+      protected ConcreteBuilder self() {
+         return this;
+      }
+   }
+
+   @SerializedName("name")
+   private final String instanceName;
+   private final double hours;
+   @SerializedName("memory_mb")
+   private final double flavorMemoryMb;
+   @SerializedName("local_gb")
+   private final double flavorLocalGb;
+   @SerializedName("vcpus")
+   private final double flavorVcpus;
+   @SerializedName("tenant_id")
+   private final String tenantId;
+   @SerializedName("flavor")
+   private final String flavorName;
+   @SerializedName("started_at")
+   private final Date instanceCreated;
+   @SerializedName("ended_at")
+   private final Date instanceTerminiated;
+   @SerializedName("state")
+   private final Status instanceStatus;
+   private final long uptime;
+
+   private SimpleServerUsage(Builder builder) {
+      this.instanceName = checkNotNull(builder.instanceName, "instanceName");
+      this.hours = builder.hours;
+      this.flavorMemoryMb = builder.flavorMemoryMb;
+      this.flavorLocalGb = builder.flavorLocalGb;
+      this.flavorVcpus =  builder.flavorVcpus;
+      this.tenantId = checkNotNull(builder.tenantId, "tenantId");
+      this.flavorName = checkNotNull(builder.flavorName, "flavorName");
+      this.instanceCreated = builder.instanceCreated; //checkNotNull(builder.instanceCreated, "instanceCreated");
+      this.instanceTerminiated = builder.instanceTerminiated;
+      this.instanceStatus = checkNotNull(builder.instanceStatus, "instanceStatus");
+      this.uptime = checkNotNull(builder.uptime, "uptime");
+   }
+
+   /**
+    */
+   public String getInstanceName() {
+      return this.instanceName;
+   }
+
+   /**
+    */
+   public double getFlavorMemoryMb() {
+      return this.flavorMemoryMb;
+   }
+
+   /**
+    */
+   public double getFlavorLocalGb() {
+      return this.flavorLocalGb;
+   }
+
+   /**
+    */
+   public double getFlavorVcpus() {
+      return this.flavorVcpus;
+   }
+
+   /**
+    */
+   public String getTenantId() {
+      return this.tenantId;
+   }
+
+   /**
+    */
+   public String getFlavorName() {
+      return this.flavorName;
+   }
+
+   /**
+    */
+   public Date getInstanceCreated() {
+      return this.instanceCreated;
+   }
+
+   /**
+    */
+   @Nullable
+   public Date getInstanceTerminiated() {
+      return this.instanceTerminiated;
+   }
+
+   /**
+    */
+   public Status getInstanceStatus() {
+      return this.instanceStatus;
+   }
+
+   /**
+    */
+   public long getUptime() {
+      return this.uptime;
+   }
+
+   @Override
+   public int hashCode() {
+      return Objects.hashCode(instanceName, flavorMemoryMb, flavorLocalGb, flavorVcpus, tenantId, flavorName, instanceCreated, instanceTerminiated, instanceStatus, uptime);
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      if (this == obj) return true;
+      if (obj == null || getClass() != obj.getClass()) return false;
+      SimpleServerUsage that = SimpleServerUsage.class.cast(obj);
+      return Objects.equal(this.instanceName, that.instanceName)
+            && Objects.equal(this.flavorMemoryMb, that.flavorMemoryMb)
+            && Objects.equal(this.flavorLocalGb, that.flavorLocalGb)
+            && Objects.equal(this.flavorVcpus, that.flavorVcpus)
+            && Objects.equal(this.tenantId, that.tenantId)
+            && Objects.equal(this.flavorName, that.flavorName)
+            && Objects.equal(this.instanceCreated, that.instanceCreated)
+            && Objects.equal(this.instanceTerminiated, that.instanceTerminiated)
+            && Objects.equal(this.instanceStatus, that.instanceStatus)
+            && Objects.equal(this.uptime, that.uptime)
+            ;
+   }
+
+   protected ToStringHelper string() {
+      return Objects.toStringHelper("")
+            .add("instanceName", instanceName)
+            .add("flavorMemoryMb", flavorMemoryMb)
+            .add("flavorLocalGb", flavorLocalGb)
+            .add("flavorVcpus", flavorVcpus)
+            .add("tenantId", tenantId)
+            .add("flavorName", flavorName)
+            .add("instanceCreated", instanceCreated)
+            .add("instanceTerminiated", instanceTerminiated)
+            .add("instanceStatus", instanceStatus)
+            .add("uptime", uptime)
+            ;
+   }
+
+   @Override
+   public String toString() {
+      return string().toString();
+   }
+
+}
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/SimpleTenantUsage.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/SimpleTenantUsage.java
new file mode 100644
index 0000000000..b4b20ddd2e
--- /dev/null
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/domain/SimpleTenantUsage.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.domain;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.Set;
+
+import org.jclouds.javax.annotation.Nullable;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Objects.ToStringHelper;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.gson.annotations.SerializedName;
+
+/**
+ * Information the SimpleTenantUsage extension returns data about each tenant
+ *
+ * @author Adam Lowe
+ */
+public class SimpleTenantUsage {
+   public static Builder builder() {
+      return new ConcreteBuilder();
+   }
+
+   public Builder toBuilder() {
+      return new ConcreteBuilder().fromSimpleTenantUsage(this);
+   }
+
+   public static abstract class Builder> {
+      private String tenantId;
+      private double totalLocalGbUsage;
+      private double totalVcpusUsage;
+      private double totalMemoryMbUsage;
+      private double totalHours;
+      private Date start;
+      private Date stop;
+      private Set serverUsages = Sets.newLinkedHashSet();
+
+      protected abstract T self();
+
+      public T tenantId(String tenantId) {
+         this.tenantId = tenantId;
+         return self();
+      }
+      
+      public T totalLocalGbUsage(double total_local_gb_usage) {
+         this.totalLocalGbUsage = total_local_gb_usage;
+         return self();
+      }
+
+      public T totalVcpusUsage(double total_vcpus_usage) {
+         this.totalVcpusUsage = total_vcpus_usage;
+         return self();
+      }
+
+      public T totalMemoryMbUsage(double total_memory_mb_usage) {
+         this.totalMemoryMbUsage = total_memory_mb_usage;
+         return self();
+      }
+
+      public T totalHours(double total_hours) {
+         this.totalHours = total_hours;
+         return self();
+      }
+
+      public T start(Date start) {
+         this.start = start;
+         return self();
+      }
+
+      public T stop(Date stop) {
+         this.stop = stop;
+         return self();
+      }
+
+      public T serverUsages(Set serverUsages) {
+         this.serverUsages = serverUsages;
+         return self();
+      }
+
+      public SimpleTenantUsage build() {
+         return new SimpleTenantUsage(this);
+      }
+
+
+      public T fromSimpleTenantUsage(SimpleTenantUsage in) {
+         return this
+               .totalLocalGbUsage(in.getTotalLocalGbUsage())
+               .totalVcpusUsage(in.getTotalVcpusUsage())
+               .totalMemoryMbUsage(in.getTotalMemoryMbUsage())
+               .totalHours(in.getTotalHours())
+               .start(in.getStart())
+               .stop(in.getStop())
+               .serverUsages(in.getServerUsages())
+               ;
+      }
+
+   }
+
+   private static class ConcreteBuilder extends Builder {
+      @Override
+      protected ConcreteBuilder self() {
+         return this;
+      }
+   }
+
+   @SerializedName("tenant_id")
+   private final String tenantId;
+   @SerializedName("total_local_gb_usage")
+   private final double totalLocalGbUsage;
+   @SerializedName("total_vcpus_usage")
+   private final double totalVcpusUsage;
+   @SerializedName("total_memory_mb_usage")
+   private final double totalMemoryMbUsage;
+   @SerializedName("total_hours")
+   private final double totalHours;
+   private final Date start;
+   private final Date stop;
+   @SerializedName("server_usages")
+   private final Set serverUsages;
+
+   private SimpleTenantUsage(Builder builder) {
+      this.tenantId = builder.tenantId;
+      this.totalLocalGbUsage = builder.totalLocalGbUsage;
+      this.totalVcpusUsage = builder.totalVcpusUsage;
+      this.totalMemoryMbUsage = builder.totalMemoryMbUsage;
+      this.totalHours = builder.totalHours;
+      this.start = builder.start;
+      this.stop = builder.stop;
+      this.serverUsages = ImmutableSet.copyOf(checkNotNull(builder.serverUsages, "serverUsages"));
+   }
+
+   public String getTenantId() {
+      return tenantId;
+   }
+
+   /**
+    */
+   public double getTotalLocalGbUsage() {
+      return this.totalLocalGbUsage;
+   }
+
+   /**
+    */
+   public double getTotalVcpusUsage() {
+      return this.totalVcpusUsage;
+   }
+
+   /**
+    */
+   public double getTotalMemoryMbUsage() {
+      return this.totalMemoryMbUsage;
+   }
+
+   /**
+    */
+   public double getTotalHours() {
+      return this.totalHours;
+   }
+
+   /**
+    */
+   @Nullable
+   public Date getStart() {
+      return this.start;
+   }
+
+   /**
+    */
+   @Nullable
+   public Date getStop() {
+      return this.stop;
+   }
+
+   /**
+    */
+   @Nullable
+   public Set getServerUsages() {
+      return serverUsages == null ? ImmutableSet.of() : Collections.unmodifiableSet(this.serverUsages);
+   }
+
+   @Override
+   public int hashCode() {
+      return Objects.hashCode(totalLocalGbUsage, totalVcpusUsage, totalMemoryMbUsage, totalHours, start, stop, serverUsages);
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      if (this == obj) return true;
+      if (obj == null || getClass() != obj.getClass()) return false;
+      SimpleTenantUsage that = SimpleTenantUsage.class.cast(obj);
+      return Objects.equal(this.totalLocalGbUsage, that.totalLocalGbUsage)
+            && Objects.equal(this.totalVcpusUsage, that.totalVcpusUsage)
+            && Objects.equal(this.totalMemoryMbUsage, that.totalMemoryMbUsage)
+            && Objects.equal(this.totalHours, that.totalHours)
+            && Objects.equal(this.start, that.start)
+            && Objects.equal(this.stop, that.stop)
+            && Objects.equal(this.serverUsages, that.serverUsages)
+            ;
+   }
+
+   protected ToStringHelper string() {
+      return Objects.toStringHelper("")
+            .add("totalLocalGbUsage", totalLocalGbUsage)
+            .add("totalVcpusUsage", totalVcpusUsage)
+            .add("totalMemoryMbUsage", totalMemoryMbUsage)
+            .add("totalHours", totalHours)
+            .add("start", start)
+            .add("stop", stop)
+            .add("serverUsages", serverUsages)
+            ;
+   }
+
+   @Override
+   public String toString() {
+      return string().toString();
+   }
+
+}
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageAsyncClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageAsyncClient.java
new file mode 100644
index 0000000000..4e2e2b80cf
--- /dev/null
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageAsyncClient.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.extensions;
+
+import java.util.Set;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.openstack.filters.AuthenticateRequest;
+import org.jclouds.openstack.nova.v1_1.domain.SimpleTenantUsage;
+import org.jclouds.openstack.services.Extension;
+import org.jclouds.openstack.services.ServiceType;
+import org.jclouds.rest.annotations.ExceptionParser;
+import org.jclouds.rest.annotations.RequestFilters;
+import org.jclouds.rest.annotations.SelectJson;
+import org.jclouds.rest.annotations.SkipEncoding;
+import org.jclouds.rest.functions.ReturnEmptySetOnNotFoundOr404;
+import org.jclouds.rest.functions.ReturnNullOnNotFoundOr404;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+/**
+ * Provides asynchronous access to Simple Tenant Usage via the REST API.
+ * 

+ *
+ * @author Adam Lowe
+ * @see SimpleTenantUsageClient
+ * @see 
+ * @see 
+ * @see 
+ */
+@Extension(of = ServiceType.COMPUTE, namespace = ExtensionNamespaces.SIMPLE_TENANT_USAGE)
+@SkipEncoding({'/', '='})
+@RequestFilters(AuthenticateRequest.class)
+public interface SimpleTenantUsageAsyncClient {
+
+   /**
+    * @see SimpleTenantUsageClient#listTenantUsages()
+    */
+   @GET
+   @Path("/os-simple-tenant-usage")
+   @SelectJson("tenant_usages")
+   @Consumes(MediaType.APPLICATION_JSON)
+   @ExceptionParser(ReturnEmptySetOnNotFoundOr404.class)
+   ListenableFuture> listTenantUsages();
+
+   /**
+    * @see SimpleTenantUsageClient#getTenantUsage(String)
+    */
+   @GET
+   @Path("/os-simple-tenant-usage/{id}")
+   @SelectJson("tenant_usage")
+   @Consumes(MediaType.APPLICATION_JSON)
+   @ExceptionParser(ReturnNullOnNotFoundOr404.class)
+   ListenableFuture getTenantUsage(@PathParam("id") String tenantId);
+
+}
diff --git a/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClient.java b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClient.java
new file mode 100644
index 0000000000..dcef9aafde
--- /dev/null
+++ b/apis/openstack-nova/src/main/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClient.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.extensions;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.jclouds.concurrent.Timeout;
+import org.jclouds.openstack.nova.v1_1.domain.SimpleTenantUsage;
+import org.jclouds.openstack.services.Extension;
+import org.jclouds.openstack.services.ServiceType;
+
+/**
+ * Provides asynchronous access to Simple Tenant Usage via the REST API.
+ * 

+ *
+ * @author Adam Lowe
+ * @see SimpleTenantUsageAsyncClient
+ */
+@Extension(of = ServiceType.COMPUTE, namespace = ExtensionNamespaces.SIMPLE_TENANT_USAGE)
+@Timeout(duration = 180, timeUnit = TimeUnit.SECONDS)
+public interface SimpleTenantUsageClient {
+
+   /**
+    * Retrive tenant_usage for all tenants
+    *
+    * @return the set of TenantUsage reports
+    */
+   Set listTenantUsages();
+
+   /**
+    * Retrive tenant_usage for a specified tenant
+    *
+    * @return the requested tenant usage
+    */
+   SimpleTenantUsage getTenantUsage(String tenantId);
+}
diff --git a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClientExpectTest.java b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClientExpectTest.java
new file mode 100644
index 0000000000..9f9a01e508
--- /dev/null
+++ b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClientExpectTest.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.extensions;
+
+import static org.testng.Assert.*;
+import static org.testng.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.Set;
+
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.ContextBuilder;
+import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.date.DateService;
+import org.jclouds.date.internal.SimpleDateFormatDateService;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.nova.v1_1.NovaClient;
+import org.jclouds.openstack.nova.v1_1.domain.Ingress;
+import org.jclouds.openstack.nova.v1_1.domain.IpProtocol;
+import org.jclouds.openstack.nova.v1_1.domain.SecurityGroup;
+import org.jclouds.openstack.nova.v1_1.domain.SecurityGroupRule;
+import org.jclouds.openstack.nova.v1_1.domain.Server;
+import org.jclouds.openstack.nova.v1_1.domain.SimpleServerUsage;
+import org.jclouds.openstack.nova.v1_1.domain.SimpleTenantUsage;
+import org.jclouds.openstack.nova.v1_1.internal.BaseNovaClientExpectTest;
+import org.jclouds.openstack.nova.v1_1.parse.ParseSecurityGroupListTest;
+import org.jclouds.openstack.nova.v1_1.parse.ParseSecurityGroupTest;
+import org.jclouds.rest.internal.RestContextImpl;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+/**
+ * Tests SimpleTenantUsageClient guice wiring and parsing
+ * 
+ * @author Adam Lowe
+ */
+@Test(groups = "unit", testName = "SimpleTenantUsageClientExpectTest")
+public class SimpleTenantUsageClientExpectTest extends BaseNovaClientExpectTest {
+   private DateService dateService = new SimpleDateFormatDateService();
+
+   public void testList() throws Exception {
+      URI endpoint = URI.create("https://compute.north.host/v1.1/3456/os-simple-tenant-usage");
+      SimpleTenantUsageClient client = requestsSendResponses(keystoneAuthWithUsernameAndPassword,
+            responseWithKeystoneAccess, extensionsOfNovaRequest, extensionsOfNovaResponse,
+            HttpRequest.builder().method("GET").headers(ImmutableMultimap.of("Accept", MediaType.APPLICATION_JSON, "X-Auth-Token", authToken))
+            .endpoint(endpoint).build(),
+            HttpResponse.builder().statusCode(200)
+                  .payload(payloadFromResource("/simple_tenant_usages.json")).build())
+            .getSimpleTenantUsageExtensionForZone("az-1.region-a.geo-1").get();
+      
+      Set results = client.listTenantUsages();
+      
+      SimpleTenantUsage usage = Iterables.getOnlyElement(results);
+      assertEquals(usage.getTenantId(), "f8535069c3fb404cb61c873b1a0b4921");
+      assertEquals(usage.getTotalHours(), 4.888888888888889e-07);
+      assertEquals(usage.getTotalLocalGbUsage(), 1.9555555555555557e-05);
+      assertEquals(usage.getTotalMemoryMbUsage(), 0.0015018666666666667);
+      assertEquals(usage.getTotalVcpusUsage(), 7.333333333333333e-07);
+      assertEquals(usage.getStart(), dateService.iso8601DateParse("2012-04-18 12:18:39.702411"));
+      assertEquals(usage.getStop(), dateService.iso8601DateParse("2012-04-18 12:18:39.702499"));
+      assertNotNull(usage.getServerUsages());
+      assertTrue(usage.getServerUsages().isEmpty());
+   }
+
+   public void testGet() throws Exception {
+      URI endpoint = URI.create("https://compute.north.host/v1.1/3456/os-simple-tenant-usage/test-1234");
+      SimpleTenantUsageClient client = requestsSendResponses(keystoneAuthWithUsernameAndPassword,
+            responseWithKeystoneAccess, extensionsOfNovaRequest, extensionsOfNovaResponse,
+            HttpRequest.builder().method("GET").headers(ImmutableMultimap.of("Accept", MediaType.APPLICATION_JSON, "X-Auth-Token", authToken))
+                  .endpoint(endpoint).build(),
+            HttpResponse.builder().statusCode(200)
+                  .payload(payloadFromResource("/simple_tenant_usage.json")).build())
+            .getSimpleTenantUsageExtensionForZone("az-1.region-a.geo-1").get();
+
+      SimpleTenantUsage usage = client.getTenantUsage("test-1234");
+      assertEquals(usage.getTenantId(), "f8535069c3fb404cb61c873b1a0b4921");
+      
+      SimpleTenantUsage expected = SimpleTenantUsage.builder().totalHours(4.833333333333333E-7).totalLocalGbUsage(1.933333333333333E-05)
+            .start(dateService.iso8601DateParse("2012-04-18 13:32:07.255743")).stop(dateService.iso8601DateParse("2012-04-18 13:32:07.255743"))
+            .totalMemoryMbUsage(0.0014847999999999999).totalVcpusUsage(7.249999999999999E-07).serverUsages(
+            ImmutableSet.of(
+                  SimpleServerUsage.builder().hours(2.4166666666666665e-07).uptime(91149).flavorLocalGb(50).instanceName("test1").tenantId("f8535069c3fb404cb61c873b1a0b4921").flavorVcpus(2).flavorMemoryMb(4096).instanceStatus(SimpleServerUsage.Status.ACTIVE).flavorName("m1.medium").instanceCreated(this.dateService.iso8601SecondsDateParse("2012-04-17T12:12:58")).build(),
+                  SimpleServerUsage.builder().hours(2.4166666666666665e-07).uptime(84710).flavorLocalGb(30).instanceName("mish_test").tenantId("f8535069c3fb404cb61c873b1a0b4921").flavorVcpus(1).flavorMemoryMb(2048).instanceStatus(SimpleServerUsage.Status.ACTIVE).flavorName("m1.small").instanceCreated(this.dateService.iso8601SecondsDateParse("2012-04-17T14:00:17")).build()
+            )).build();
+      
+      assertEquals(usage, expected);
+   }
+
+}
diff --git a/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClientLiveTest.java b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClientLiveTest.java
new file mode 100644
index 0000000000..c3d7de7b14
--- /dev/null
+++ b/apis/openstack-nova/src/test/java/org/jclouds/openstack/nova/v1_1/extensions/SimpleTenantUsageClientLiveTest.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.nova.v1_1.extensions;
+
+import static org.testng.Assert.assertNotNull;
+
+import java.util.Set;
+
+import org.jclouds.openstack.nova.v1_1.domain.SecurityGroup;
+import org.jclouds.openstack.nova.v1_1.domain.SimpleTenantUsage;
+import org.jclouds.openstack.nova.v1_1.internal.BaseNovaClientExpectTest;
+import org.jclouds.openstack.nova.v1_1.internal.BaseNovaClientLiveTest;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Optional;
+
+/**
+ * Tests behavior of SimpleTenantUsageClient
+ *
+ * @author Adam Lowe
+ */
+@Test(groups = "live", testName = "SimpleTenantUsageClientLiveTest")
+public class SimpleTenantUsageClientLiveTest extends BaseNovaClientLiveTest {
+
+   public void testList() throws Exception {
+      for (String zoneId : novaContext.getApi().getConfiguredZones()) {
+         Optional optClient = novaContext.getApi().getSimpleTenantUsageExtensionForZone(zoneId);
+         if (optClient.isPresent() && identity.endsWith(":admin")) {
+            SimpleTenantUsageClient client = optClient.get();
+            Set usages = client.listTenantUsages();
+            assertNotNull(usages);
+            for (SimpleTenantUsage usage : usages) {
+               SimpleTenantUsage details = client.getTenantUsage(usage.getTenantId());
+               assertNotNull(details);
+            }
+         }
+      }
+   }
+}
diff --git a/apis/openstack-nova/src/test/resources/simple_tenant_usage.json b/apis/openstack-nova/src/test/resources/simple_tenant_usage.json
new file mode 100644
index 0000000000..de07dc4f7c
--- /dev/null
+++ b/apis/openstack-nova/src/test/resources/simple_tenant_usage.json
@@ -0,0 +1 @@
+{"tenant_usage": {"total_memory_mb_usage": 0.0014847999999999999, "total_vcpus_usage": 7.249999999999999e-07, "total_hours": 4.833333333333333e-07, "tenant_id": "f8535069c3fb404cb61c873b1a0b4921", "stop": "2012-04-18 13:32:07.255830", "server_usages": [{"hours": 2.4166666666666665e-07, "uptime": 91149, "local_gb": 50, "ended_at": null, "name": "test1", "tenant_id": "f8535069c3fb404cb61c873b1a0b4921", "vcpus": 2, "memory_mb": 4096, "state": "active", "flavor": "m1.medium", "started_at": "2012-04-17 12:12:58"}, {"hours": 2.4166666666666665e-07, "uptime": 84710, "local_gb": 30, "ended_at": null, "name": "mish_test", "tenant_id": "f8535069c3fb404cb61c873b1a0b4921", "vcpus": 1, "memory_mb": 2048, "state": "active", "flavor": "m1.small", "started_at": "2012-04-17 14:00:17"}], "start": "2012-04-18 13:32:07.255743", "total_local_gb_usage": 1.933333333333333e-05}}
\ No newline at end of file
diff --git a/apis/openstack-nova/src/test/resources/simple_tenant_usages.json b/apis/openstack-nova/src/test/resources/simple_tenant_usages.json
new file mode 100644
index 0000000000..6f3c0d74f6
--- /dev/null
+++ b/apis/openstack-nova/src/test/resources/simple_tenant_usages.json
@@ -0,0 +1 @@
+{"tenant_usages": [{"total_memory_mb_usage": 0.0015018666666666667, "total_vcpus_usage": 7.333333333333333e-07, "start": "2012-04-18 12:18:39.702411", "tenant_id": "f8535069c3fb404cb61c873b1a0b4921", "stop": "2012-04-18 12:18:39.702499", "total_hours": 4.888888888888889e-07, "total_local_gb_usage": 1.9555555555555557e-05}]
\ No newline at end of file

From e11dd37237616ba33bb8f8062caf82597371b549 Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Thu, 29 Mar 2012 10:22:46 +0400
Subject: [PATCH 27/50] Stubs for multipart upload support in swift.

---
 .../blobstore/CloudFilesBlobStore.java        |  3 +-
 .../swift/blobstore/SwiftBlobStore.java       | 12 ++++-
 .../blobstore/strategy/MultipartUpload.java   | 21 +++++++++
 .../internal/MultipartUploadStrategy.java     | 12 +++++
 .../SequentialMultipartUploadStrategy.java    | 46 +++++++++++++++++++
 .../HPCloudObjectStorageBlobStore.java        |  2 +-
 6 files changed, 92 insertions(+), 4 deletions(-)
 create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java
 create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java
 create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java

diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java
index 42be6d884c..40ba250abd 100644
--- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java
+++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java
@@ -42,6 +42,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
 
 import com.google.common.base.Supplier;
+import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy;
 
 /**
  * 
@@ -62,7 +63,7 @@ public class CloudFilesBlobStore extends SwiftBlobStore {
             Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) {
       super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions,
                container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions,
-               fetchBlobMetadataProvider);
+               fetchBlobMetadataProvider, null);
       this.enableCDNAndCache = enableCDNAndCache;
 
    }
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
index e96909ba18..6cc3329861 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
@@ -50,6 +50,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList;
 import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
+import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy;
 import org.jclouds.openstack.swift.domain.ContainerMetadata;
 
 import com.google.common.base.Function;
@@ -71,6 +72,7 @@ public class SwiftBlobStore extends BaseBlobStore {
    private final ObjectToBlobMetadata object2BlobMd;
    private final BlobToHttpGetOptions blob2ObjectGetOptions;
    private final Provider fetchBlobMetadataProvider;
+   private final Provider multipartUploadStrategy;
 
    @Inject
    protected SwiftBlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier defaultLocation,
@@ -79,7 +81,8 @@ public class SwiftBlobStore extends BaseBlobStore {
             BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
             ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object,
             ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions,
-            Provider fetchBlobMetadataProvider) {
+            Provider fetchBlobMetadataProvider,
+            Provider multipartUploadStrategy) {
       super(context, blobUtils, defaultLocation, locations);
       this.sync = sync;
       this.container2ResourceMd = container2ResourceMd;
@@ -90,6 +93,7 @@ public class SwiftBlobStore extends BaseBlobStore {
       this.object2BlobMd = object2BlobMd;
       this.blob2ObjectGetOptions = blob2ObjectGetOptions;
       this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider");
+      this.multipartUploadStrategy = multipartUploadStrategy;
    }
 
    /**
@@ -207,7 +211,11 @@ public class SwiftBlobStore extends BaseBlobStore {
    @Override
    public String putBlob(String container, Blob blob, PutOptions options) {
       // TODO implement options
-      return putBlob(container, blob);
+      if (options.isMultipart()) {
+        return multipartUploadStrategy.get().execute(container, blob, options);
+      } else {
+        return putBlob(container, blob);
+      }
    }
 
    /**
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java
new file mode 100644
index 0000000000..839f4774e2
--- /dev/null
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java
@@ -0,0 +1,21 @@
+package org.jclouds.openstack.swift.blobstore.strategy;
+
+/*
+@author Roman Bogorodskiy
+ */
+
+public interface MultipartUpload {
+
+    /* Maximum number of parts per upload */
+    public static final int MAX_NUMBER_OF_PARTS = 10000;
+    /* Maximum number of parts returned for a list parts request */
+    public static final int MAX_LIST_PARTS_RETURNED = 1000;
+    /* Maximum number of multipart uploads returned in a list multipart uploads request */
+    public static final int MAX_LIST_MPU_RETURNED = 1000;
+
+    /*
+    * part size 5 MB to 5 GB, last part can be < 5 MB
+    */
+    public static final long MIN_PART_SIZE = 5242880L;
+    public static final long MAX_PART_SIZE = 5368709120L;
+}
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java
new file mode 100644
index 0000000000..40a79ad45a
--- /dev/null
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java
@@ -0,0 +1,12 @@
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import com.google.inject.ImplementedBy;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
+
+@ImplementedBy(SequentialMultipartUploadStrategy.class)
+public interface MultipartUploadStrategy extends MultipartUpload {
+
+    String execute(String container, Blob blob, PutOptions options);
+}
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
new file mode 100644
index 0000000000..693cee51be
--- /dev/null
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
@@ -0,0 +1,46 @@
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import javax.annotation.Resource;
+import javax.inject.Named;
+
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.blobstore.reference.BlobStoreConstants;
+import org.jclouds.io.Payload;
+import org.jclouds.io.PayloadSlicer;
+import org.jclouds.logging.Logger;
+import org.jclouds.openstack.swift.blobstore.SwiftBlobStore;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+
+public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy {
+   @Resource
+   @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
+   protected Logger logger = Logger.NULL;
+
+   protected final SwiftBlobStore ablobstore;
+   protected final PayloadSlicer slicer;
+
+    public SequentialMultipartUploadStrategy(SwiftBlobStore ablobstore, PayloadSlicer slicer) {
+        this.ablobstore = checkNotNull(ablobstore, "ablobstore");
+        this.slicer = checkNotNull(slicer, "slicer");
+    }
+
+    @Override
+    public String execute(String container, Blob blob, PutOptions options) {
+        String key = blob.getMetadata().getName();
+        Payload payload = blob.getPayload();
+        /*MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm();
+        algorithm
+                .calculateChunkSize(checkNotNull(
+                        payload.getContentMetadata().getContentLength(),
+                        "contentLength required on all uploads to amazon s3; please invoke payload.getContentMetadata().setContentLength(length) first"));
+        int parts = algorithm.getParts();
+        long chunkSize = algorithm.getChunkSize();
+        if (parts > 0) {
+
+        } */
+        return "NOT IMPLEMENTED";
+    }
+}
diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java
index e05119ed8c..cd11feb720 100644
--- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java
+++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java
@@ -62,7 +62,7 @@ public class HPCloudObjectStorageBlobStore extends SwiftBlobStore {
             Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) {
       super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions,
                container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions,
-               fetchBlobMetadataProvider);
+               fetchBlobMetadataProvider, null);
       this.enableCDNAndCache = enableCDNAndCache;
 
    }

From 807d078c6f96cf98d8adaddc777ad6a872de603e Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Fri, 6 Apr 2012 17:26:19 +0400
Subject: [PATCH 28/50] First working implementation of swift multipart upload.
 Async client TDB.

---
 .../swift/CommonSwiftAsyncClient.java         |  16 +-
 .../openstack/swift/CommonSwiftClient.java    |   2 +
 .../swift/blobstore/SwiftAsyncBlobStore.java  |   9 +-
 .../swift/blobstore/SwiftBlobStore.java       |   2 +-
 .../MultipartUploadSlicingAlgorithm.java      | 152 ++++++++++++++++++
 .../internal/MultipartUploadStrategy.java     |   3 +-
 .../SequentialMultipartUploadStrategy.java    |  65 ++++++--
 .../swift/internal/StubSwiftAsyncClient.java  |  13 +-
 .../java/org/jclouds/aws/s3/AWSS3Client.java  |   1 +
 9 files changed, 242 insertions(+), 21 deletions(-)
 create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java

diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftAsyncClient.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftAsyncClient.java
index 2fff5ae24e..71aae2d781 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftAsyncClient.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftAsyncClient.java
@@ -52,14 +52,7 @@ import org.jclouds.openstack.swift.functions.ParseObjectInfoFromHeaders;
 import org.jclouds.openstack.swift.functions.ParseObjectInfoListFromJsonResponse;
 import org.jclouds.openstack.swift.functions.ReturnTrueOn404FalseOn409;
 import org.jclouds.openstack.swift.options.ListContainerOptions;
-import org.jclouds.rest.annotations.BinderParam;
-import org.jclouds.rest.annotations.Endpoint;
-import org.jclouds.rest.annotations.ExceptionParser;
-import org.jclouds.rest.annotations.ParamParser;
-import org.jclouds.rest.annotations.QueryParams;
-import org.jclouds.rest.annotations.RequestFilters;
-import org.jclouds.rest.annotations.ResponseParser;
-import org.jclouds.rest.annotations.SkipEncoding;
+import org.jclouds.rest.annotations.*;
 import org.jclouds.rest.functions.ReturnVoidOnNotFoundOr404;
 
 import com.google.common.util.concurrent.ListenableFuture;
@@ -183,4 +176,11 @@ public interface CommonSwiftAsyncClient {
    @Path("/{container}/{name}")
    ListenableFuture removeObject(@PathParam("container") String container, @PathParam("name") String name);
 
+   @PUT
+   @Path("/{container}/{name}")
+   @ResponseParser(ParseETagHeader.class)
+   @Headers(keys = "X-Object-Manifest", values="{container}/{name}")
+   ListenableFuture putObjectManifest(@PathParam("container") String container,
+                                            @PathParam("name") String name);
+
 }
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftClient.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftClient.java
index a8b5210a32..5ad663bacf 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftClient.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/CommonSwiftClient.java
@@ -26,6 +26,7 @@ import org.jclouds.blobstore.ContainerNotFoundException;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.concurrent.Timeout;
 import org.jclouds.http.options.GetOptions;
+import org.jclouds.io.Payload;
 import org.jclouds.openstack.swift.domain.AccountMetadata;
 import org.jclouds.openstack.swift.domain.ContainerMetadata;
 import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata;
@@ -113,4 +114,5 @@ public interface CommonSwiftClient {
     */
    boolean objectExists(String container, String name);
 
+   String putObjectManifest(String container, String name);
 }
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java
index b0873c9af6..e59b86f95e 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java
@@ -55,6 +55,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList;
 import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
+import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy;
 import org.jclouds.openstack.swift.domain.ContainerMetadata;
 import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata;
 import org.jclouds.openstack.swift.domain.ObjectInfo;
@@ -81,6 +82,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
    private final ObjectToBlobMetadata object2BlobMd;
    private final BlobToHttpGetOptions blob2ObjectGetOptions;
    private final Provider fetchBlobMetadataProvider;
+   //private final Provider multipartUploadStrategy;
 
    @Inject
    protected SwiftAsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
@@ -102,6 +104,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
       this.object2BlobMd = object2BlobMd;
       this.blob2ObjectGetOptions = blob2ObjectGetOptions;
       this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider");
+      //this.multipartUploadStrategy = multipartUploadStrategy;
    }
 
    /**
@@ -239,7 +242,11 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
    @Override
    public ListenableFuture putBlob(String container, Blob blob, PutOptions options) {
       // TODO implement options
-      return putBlob(container, blob);
+      //if (options.isMultipart()) {
+      //    return null; //Lis  multipartUploadStrategy.get().execute(container, blob, options);
+      //} else {
+        return putBlob(container, blob);
+      //}
    }
 
    @Override
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
index 6cc3329861..d19ab2ae44 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
@@ -212,7 +212,7 @@ public class SwiftBlobStore extends BaseBlobStore {
    public String putBlob(String container, Blob blob, PutOptions options) {
       // TODO implement options
       if (options.isMultipart()) {
-        return multipartUploadStrategy.get().execute(container, blob, options);
+        return multipartUploadStrategy.get().execute(container, blob, options, blob2Object);
       } else {
         return putBlob(container, blob);
       }
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
new file mode 100644
index 0000000000..26c5979dea
--- /dev/null
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *   MultipartUploadSlicingAlgorithm.java
+ *
+ * 
+ *   Created by: tibor
+ *
+ *   History
+ *
+ */
+
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.Inject;
+import org.jclouds.blobstore.reference.BlobStoreConstants;
+import org.jclouds.logging.Logger;
+import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
+
+import javax.annotation.Resource;
+import javax.inject.Named;
+
+public class MultipartUploadSlicingAlgorithm {
+
+   @Resource
+   @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
+   protected Logger logger = Logger.NULL;
+
+   @VisibleForTesting
+   static final long DEFAULT_PART_SIZE = 33554432; // 32MB
+   
+   @VisibleForTesting
+   static final int DEFAULT_MAGNITUDE_BASE = 100;
+   
+   @Inject(optional = true)
+   @Named("jclouds.mpu.parts.size")
+   @VisibleForTesting
+   long defaultPartSize = DEFAULT_PART_SIZE;
+   
+   @Inject(optional = true)
+   @Named("jclouds.mpu.parts.magnitude")
+   @VisibleForTesting
+   int magnitudeBase = DEFAULT_MAGNITUDE_BASE;
+
+   // calculated only once, but not from the constructor
+   private volatile int parts; // required number of parts with chunkSize
+   private volatile long chunkSize;
+   private volatile long remaining; // number of bytes remained for the last part
+
+   // sequentially updated values
+   private volatile int part;
+   private volatile long chunkOffset;
+   private volatile long copied;
+
+   @VisibleForTesting
+   protected long calculateChunkSize(long length) {
+      long unitPartSize = defaultPartSize; // first try with default part size
+      int parts = (int)(length / unitPartSize);
+      long partSize = unitPartSize;
+      int magnitude = (int) (parts / magnitudeBase);
+      if (magnitude > 0) {
+         partSize = magnitude * unitPartSize;
+         if (partSize > MultipartUpload.MAX_PART_SIZE) {
+            partSize = MultipartUpload.MAX_PART_SIZE;
+            unitPartSize = MultipartUpload.MAX_PART_SIZE;
+         }
+         parts = (int)(length / partSize);
+         if (parts * partSize < length) {
+            partSize = (magnitude + 1) * unitPartSize;
+            if (partSize > MultipartUpload.MAX_PART_SIZE) {
+               partSize = MultipartUpload.MAX_PART_SIZE;
+               unitPartSize = MultipartUpload.MAX_PART_SIZE;
+            }
+            parts = (int)(length / partSize);
+         }
+      }
+      if (parts > MultipartUpload.MAX_NUMBER_OF_PARTS) { // if splits in too many parts or
+                                         // cannot be split
+         unitPartSize = MultipartUpload.MIN_PART_SIZE; // take the minimum part size
+         parts = (int)(length / unitPartSize);
+      }
+      if (parts > MultipartUpload.MAX_NUMBER_OF_PARTS) { // if still splits in too many parts
+         parts = MultipartUpload.MAX_NUMBER_OF_PARTS - 1; // limit them. do not care about not
+                                          // covering
+      }
+      long remainder = length % unitPartSize;
+      if (remainder == 0 && parts > 0) {
+         parts -= 1;
+      }
+      this.chunkSize = partSize;
+      this.parts = parts;
+      this.remaining = length - partSize * parts;
+      logger.debug(" %d bytes partitioned in %d parts of part size: %d, remaining: %d%s", length, parts, chunkSize,
+            remaining, (remaining > MultipartUpload.MAX_PART_SIZE ? " overflow!" : ""));
+      return this.chunkSize;
+   }
+
+   public long getCopied() {
+      return copied;
+   }
+
+   public void setCopied(long copied) {
+      this.copied = copied;
+   }
+
+   @VisibleForTesting
+   protected int getParts() {
+      return parts;
+   }
+
+   protected int getNextPart() {
+      return ++part;
+   }
+
+   protected void addCopied(long copied) {
+      this.copied += copied;
+   }
+
+   protected long getNextChunkOffset() {
+      long next = chunkOffset;
+      chunkOffset += getChunkSize();
+      return next;
+   }
+
+   @VisibleForTesting
+   protected long getChunkSize() {
+      return chunkSize;
+   }
+
+   @VisibleForTesting
+   protected long getRemaining() {
+      return remaining;
+   }
+
+}
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java
index 40a79ad45a..c536a8e8c2 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartUploadStrategy.java
@@ -3,10 +3,11 @@ package org.jclouds.openstack.swift.blobstore.strategy.internal;
 import com.google.inject.ImplementedBy;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
 import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
 
 @ImplementedBy(SequentialMultipartUploadStrategy.class)
 public interface MultipartUploadStrategy extends MultipartUpload {
 
-    String execute(String container, Blob blob, PutOptions options);
+    String execute(String container, Blob blob, PutOptions options, BlobToObject blob2Object);
 }
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
index 693cee51be..e11bd41ce3 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
@@ -3,44 +3,91 @@ package org.jclouds.openstack.swift.blobstore.strategy.internal;
 import javax.annotation.Resource;
 import javax.inject.Named;
 
+import com.google.common.collect.Maps;
+import com.google.inject.Inject;
+import org.jclouds.blobstore.KeyNotFoundException;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.options.PutOptions;
 import org.jclouds.blobstore.reference.BlobStoreConstants;
 import org.jclouds.io.Payload;
 import org.jclouds.io.PayloadSlicer;
 import org.jclouds.logging.Logger;
+import org.jclouds.openstack.swift.SwiftClient;
 import org.jclouds.openstack.swift.blobstore.SwiftBlobStore;
+import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
+import org.jclouds.openstack.swift.domain.SwiftObject;
+import org.jclouds.util.Throwables2;
+
+import java.util.SortedMap;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
 
 public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy {
-   @Resource
-   @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
-   protected Logger logger = Logger.NULL;
+    public static final String PART_SEPARATOR = "/";
 
-   protected final SwiftBlobStore ablobstore;
-   protected final PayloadSlicer slicer;
+    @Resource
+    @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
+    protected Logger logger = Logger.NULL;
 
+    protected final SwiftBlobStore ablobstore;
+    protected final PayloadSlicer slicer;
+    
+    @Inject
     public SequentialMultipartUploadStrategy(SwiftBlobStore ablobstore, PayloadSlicer slicer) {
         this.ablobstore = checkNotNull(ablobstore, "ablobstore");
         this.slicer = checkNotNull(slicer, "slicer");
     }
 
     @Override
-    public String execute(String container, Blob blob, PutOptions options) {
+    public String execute(String container, Blob blob, PutOptions options, BlobToObject blob2Object) {
+        System.out.println("here we go");
         String key = blob.getMetadata().getName();
         Payload payload = blob.getPayload();
-        /*MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm();
+        MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm();
         algorithm
                 .calculateChunkSize(checkNotNull(
                         payload.getContentMetadata().getContentLength(),
-                        "contentLength required on all uploads to amazon s3; please invoke payload.getContentMetadata().setContentLength(length) first"));
+                        "contentLength required on all uploads to swift; please invoke payload.getContentMetadata().setContentLength(length) first"));
         int parts = algorithm.getParts();
         long chunkSize = algorithm.getChunkSize();
         if (parts > 0) {
+            SwiftClient client = (SwiftClient) ablobstore.getContext()
+                    .getProviderSpecificContext().getApi();
 
-        } */
+            try {
+                SortedMap etags = Maps.newTreeMap();
+                int part;
+                while ((part = algorithm.getNextPart()) <= parts) {
+                    System.out.println("Uploading part " + part);
+                    Payload chunkedPart = slicer.slice(payload,
+                            algorithm.getNextChunkOffset(), chunkSize);
+                    Blob blobPart = ablobstore.blobBuilder(blob.getMetadata().getName() + PART_SEPARATOR +
+                            String.valueOf(part)).payload(chunkedPart).contentDisposition(
+                            blob.getMetadata().getName() + PART_SEPARATOR + String.valueOf(part)).build();
+                    client.putObject(container, blob2Object.apply(blobPart));
+                }
+                long remaining = algorithm.getRemaining();
+                if (remaining > 0) {
+                    System.out.println("Uploading tail.");
+                    Payload chunkedPart = slicer.slice(payload,
+                            algorithm.getNextChunkOffset(), remaining);
+                    Blob blobPart = ablobstore.blobBuilder(blob.getMetadata().getName() + PART_SEPARATOR + 
+                    String.valueOf(part)).payload(chunkedPart).contentDisposition(
+                            blob.getMetadata().getName() + PART_SEPARATOR + String.valueOf(part)).build();
+                    client.putObject(container, blob2Object.apply(blobPart));
+                }
+                return client.putObjectManifest(container, key);
+            } catch (Exception ex) {
+                RuntimeException rtex = Throwables2.getFirstThrowableOfType(ex, RuntimeException.class);
+                if (rtex == null) {
+                    rtex = new RuntimeException(ex);
+                }
+                //client.abortMultipartUpload(container, key, uploadId);
+                throw rtex;
+            }
+
+        }
         return "NOT IMPLEMENTED";
     }
 }
diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java
index 01ccfe458b..9a3b189990 100644
--- a/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java
+++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java
@@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorService;
 import javax.inject.Inject;
 import javax.inject.Named;
 import javax.inject.Singleton;
+import javax.ws.rs.PathParam;
 
 import org.jclouds.Constants;
 import org.jclouds.blobstore.TransientAsyncBlobStore;
@@ -167,7 +168,12 @@ public class StubSwiftAsyncClient implements CommonSwiftAsyncClient {
       return blobStore.removeBlob(container, key);
    }
 
-   public ListenableFuture setObjectInfo(String container, String key, Map userMetadata) {
+    @Override
+    public ListenableFuture putObjectManifest(String container, String name) {
+        return null;
+    }
+
+    public ListenableFuture setObjectInfo(String container, String key, Map userMetadata) {
       throw new UnsupportedOperationException();
    }
 
@@ -179,6 +185,11 @@ public class StubSwiftAsyncClient implements CommonSwiftAsyncClient {
       return objectProvider.create(null);
    }
 
+
+   /*public String putObjectManifest(String container, String name) {
+       return "stub";
+   }                 */
+
    @Override
    public ListenableFuture objectExists(String bucketName, String key) {
       return immediateFuture(containerToBlobs.get(bucketName).containsKey(key));
diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java
index ea1ae6548c..5426847372 100644
--- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java
+++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/AWSS3Client.java
@@ -56,6 +56,7 @@ public interface AWSS3Client extends S3Client {
     */
    String initiateMultipartUpload(String bucketName, ObjectMetadata objectMetadata, PutObjectOptions... options);
 
+
    /**
     * This operation aborts a multipart upload. After a multipart upload is aborted, no additional
     * parts can be uploaded using that upload ID. The storage consumed by any previously uploaded

From c6b7d510b2238f3200239d73c798098615555024 Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Tue, 10 Apr 2012 14:07:26 +0400
Subject: [PATCH 29/50] Initial implementation of MPU for SwiftAsyncBlobStore.

---
 .../blobstore/CloudFilesAsyncBlobStore.java   |   2 +-
 .../swift/blobstore/SwiftAsyncBlobStore.java  |  17 +-
 .../swift/blobstore/SwiftBlobStore.java       |   1 -
 .../AsyncMultipartUploadStrategy.java         |  12 +
 .../ParallelMultipartUploadStrategy.java      | 268 ++++++++++++++++++
 .../HPCloudObjectStorageAsyncBlobStore.java   |   2 +-
 6 files changed, 291 insertions(+), 11 deletions(-)
 create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/AsyncMultipartUploadStrategy.java
 create mode 100644 apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java

diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java
index 34b61b03de..68ad22f587 100644
--- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java
+++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java
@@ -69,7 +69,7 @@ public class CloudFilesAsyncBlobStore extends SwiftAsyncBlobStore {
             Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) {
       super(context, blobUtils, service, defaultLocation, locations, sync, async, container2ResourceMd,
                container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd,
-               blob2ObjectGetOptions, fetchBlobMetadataProvider);
+               blob2ObjectGetOptions, fetchBlobMetadataProvider, null);
       this.enableCDNAndCache = enableCDNAndCache;
    }
 
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java
index e59b86f95e..1ffd600bf0 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftAsyncBlobStore.java
@@ -55,6 +55,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList;
 import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
+import org.jclouds.openstack.swift.blobstore.strategy.internal.AsyncMultipartUploadStrategy;
 import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy;
 import org.jclouds.openstack.swift.domain.ContainerMetadata;
 import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata;
@@ -82,7 +83,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
    private final ObjectToBlobMetadata object2BlobMd;
    private final BlobToHttpGetOptions blob2ObjectGetOptions;
    private final Provider fetchBlobMetadataProvider;
-   //private final Provider multipartUploadStrategy;
+   private final Provider multipartUploadStrategy;
 
    @Inject
    protected SwiftAsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils,
@@ -92,7 +93,8 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
             BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
             ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object,
             ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions,
-            Provider fetchBlobMetadataProvider) {
+            Provider fetchBlobMetadataProvider,
+            Provider multipartUploadStrategy) {
       super(context, blobUtils, service, defaultLocation, locations);
       this.sync = sync;
       this.async = async;
@@ -104,7 +106,7 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
       this.object2BlobMd = object2BlobMd;
       this.blob2ObjectGetOptions = blob2ObjectGetOptions;
       this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider");
-      //this.multipartUploadStrategy = multipartUploadStrategy;
+      this.multipartUploadStrategy = multipartUploadStrategy;
    }
 
    /**
@@ -241,12 +243,11 @@ public class SwiftAsyncBlobStore extends BaseAsyncBlobStore {
 
    @Override
    public ListenableFuture putBlob(String container, Blob blob, PutOptions options) {
-      // TODO implement options
-      //if (options.isMultipart()) {
-      //    return null; //Lis  multipartUploadStrategy.get().execute(container, blob, options);
-      //} else {
+      if (options.isMultipart()) {
+          return multipartUploadStrategy.get().execute(container, blob, options, blob2Object);
+      } else {
         return putBlob(container, blob);
-      //}
+      }
    }
 
    @Override
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
index d19ab2ae44..2afc789227 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java
@@ -210,7 +210,6 @@ public class SwiftBlobStore extends BaseBlobStore {
     */
    @Override
    public String putBlob(String container, Blob blob, PutOptions options) {
-      // TODO implement options
       if (options.isMultipart()) {
         return multipartUploadStrategy.get().execute(container, blob, options, blob2Object);
       } else {
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/AsyncMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/AsyncMultipartUploadStrategy.java
new file mode 100644
index 0000000000..017d1dac22
--- /dev/null
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/AsyncMultipartUploadStrategy.java
@@ -0,0 +1,12 @@
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.ImplementedBy;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
+
+@ImplementedBy(ParallelMultipartUploadStrategy.class)
+public interface AsyncMultipartUploadStrategy {
+    ListenableFuture execute(String container, Blob blob, PutOptions options, BlobToObject blob2Object);
+}
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java
new file mode 100644
index 0000000000..bdc86344c2
--- /dev/null
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java
@@ -0,0 +1,268 @@
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Inject;
+import org.jclouds.Constants;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.internal.BlobRuntimeException;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.blobstore.reference.BlobStoreConstants;
+import org.jclouds.concurrent.Futures;
+import org.jclouds.io.Payload;
+import org.jclouds.io.PayloadSlicer;
+import org.jclouds.logging.Logger;
+import org.jclouds.openstack.swift.SwiftAsyncClient;
+import org.jclouds.openstack.swift.SwiftClient;
+import org.jclouds.openstack.swift.blobstore.SwiftAsyncBlobStore;
+import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
+import org.jclouds.util.Throwables2;
+
+import javax.annotation.Resource;
+import javax.inject.Named;
+import java.util.Map;
+import java.util.Queue;
+import java.util.SortedMap;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStrategy {
+    @Resource
+    @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
+    protected Logger logger = Logger.NULL;
+
+    public static final String PART_SEPARATOR = "/";
+    @VisibleForTesting
+    static final int DEFAULT_PARALLEL_DEGREE = 4;
+    @VisibleForTesting
+    static final int DEFAULT_MIN_RETRIES = 5;
+    @VisibleForTesting
+    static final int DEFAULT_MAX_PERCENT_RETRIES = 10;
+
+    @Inject(optional = true)
+    @Named("jclouds.mpu.parallel.degree")
+    @VisibleForTesting
+    int parallelDegree = DEFAULT_PARALLEL_DEGREE;
+
+    @Inject(optional = true)
+    @Named("jclouds.mpu.parallel.retries.min")
+    @VisibleForTesting
+    int minRetries = DEFAULT_MIN_RETRIES;
+
+    @Inject(optional = true)
+    @Named("jclouds.mpu.parallel.retries.maxpercent")
+    @VisibleForTesting
+    int maxPercentRetries = DEFAULT_MAX_PERCENT_RETRIES;
+
+    /**
+     * maximum duration of an blob Request
+     */
+    @Inject(optional = true)
+    @Named(Constants.PROPERTY_REQUEST_TIMEOUT)
+    protected Long maxTime;
+
+    private final ExecutorService ioWorkerExecutor;
+
+    protected final SwiftAsyncBlobStore ablobstore;
+    protected final PayloadSlicer slicer;
+
+    @Inject
+    public ParallelMultipartUploadStrategy(SwiftAsyncBlobStore ablobstore, PayloadSlicer slicer,
+                                           @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor) {
+        this.ablobstore = checkNotNull(ablobstore, "ablobstore");
+        this.slicer = checkNotNull(slicer, "slicer");
+        this.ioWorkerExecutor = checkNotNull(ioWorkerExecutor, "ioWorkerExecutor");
+    }
+
+
+    protected void prepareUploadPart(final String container, final Blob blob, final String key,
+                                     final Integer part, final Payload payload,
+                                     final long offset, final long size, final SortedMap etags,
+                                     final BlockingQueue activeParts,
+                                     final Map> futureParts,
+                                     final AtomicInteger errors, final int maxRetries, final Map errorMap,
+                                     final Queue toRetry, final CountDownLatch latch,
+                                     BlobToObject blob2Object) {
+        if (errors.get() > maxRetries) {
+            activeParts.remove(part); // remove part from the bounded-queue without blocking
+            latch.countDown();
+            return;
+        }
+        final SwiftAsyncClient client = (SwiftAsyncClient) ablobstore.getContext()
+                .getProviderSpecificContext().getAsyncApi();
+        Payload chunkedPart = slicer.slice(payload, offset, size);
+        logger.debug(String.format("async uploading part %s of %s to container %s", part, key, container));
+        final long start = System.currentTimeMillis();
+        String blobPartName = blob.getMetadata().getName() + PART_SEPARATOR +
+                String.valueOf(part);
+
+        Blob blobPart = ablobstore.blobBuilder(blobPartName).payload(chunkedPart).
+                contentDisposition(blobPartName).build();
+        final ListenableFuture futureETag = client.putObject(container, blob2Object.apply(blobPart));
+        futureETag.addListener(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    etags.put(part, futureETag.get());
+                    logger.debug(String.format("async uploaded part %s of %s to container %s in %sms",
+                            part, key, container, (System.currentTimeMillis() - start)));
+                } catch (CancellationException e) {
+                    errorMap.put(part, e);
+                    String message = String.format("%s while uploading part %s - [%s,%s] to container %s with running since %dms",
+                            e.getMessage(), part, offset, size, container, (System.currentTimeMillis() - start));
+                    logger.debug(message);
+                } catch (Exception e) {
+                    errorMap.put(part, e);
+                    String message = String.format("%s while uploading part %s - [%s,%s] to container %s running since %dms",
+                            e.getMessage(), part, offset, size, container, (System.currentTimeMillis() - start));
+                    logger.error(message, e);
+                    if (errors.incrementAndGet() <= maxRetries)
+                        toRetry.add(new Part(part, offset, size));
+                } finally {
+                    activeParts.remove(part); // remove part from the bounded-queue without blocking
+                    futureParts.remove(part);
+                    latch.countDown();
+                }
+            }
+        }, ioWorkerExecutor);
+        futureParts.put(part, futureETag);
+    }
+
+    @Override
+    public ListenableFuture execute(final String container, final Blob blob, final PutOptions options, final BlobToObject blob2Object) {
+        return Futures.makeListenable(
+                ioWorkerExecutor.submit(new Callable() {
+                    @Override
+                    public String call() throws Exception {
+                        String key = blob.getMetadata().getName();
+                        Payload payload = blob.getPayload();
+                        MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm();
+                        algorithm.calculateChunkSize(payload.getContentMetadata()
+                                .getContentLength());
+                        int parts = algorithm.getParts();
+                        long chunkSize = algorithm.getChunkSize();
+                        long remaining = algorithm.getRemaining();
+                        if (parts > 0) {
+                            SwiftClient client = (SwiftClient) ablobstore
+                                    .getContext().getProviderSpecificContext().getApi();
+                            final Map> futureParts =
+                                    new ConcurrentHashMap>();
+                            final Map errorMap = Maps.newHashMap();
+                            AtomicInteger errors = new AtomicInteger(0);
+                            int maxRetries = Math.max(minRetries, parts * maxPercentRetries / 100);
+                            int effectiveParts = remaining > 0 ? parts + 1 : parts;
+                            try {
+                                logger.debug(String.format("initiated multipart upload of %s to container %s" +
+                                        " consisting from %s part (possible max. retries: %d)",
+                                        key, container, effectiveParts, maxRetries));
+                                // we need a bounded-blocking queue to control the amount of parallel jobs
+                                ArrayBlockingQueue activeParts = new ArrayBlockingQueue(parallelDegree);
+                                Queue toRetry = new ConcurrentLinkedQueue();
+                                SortedMap etags = new ConcurrentSkipListMap();
+                                CountDownLatch latch = new CountDownLatch(effectiveParts);
+                                int part;
+                                while ((part = algorithm.getNextPart()) <= parts) {
+                                    Integer partKey = new Integer(part);
+                                    activeParts.put(partKey);
+
+                                    prepareUploadPart(container, blob, key, partKey, payload,
+                                            algorithm.getNextChunkOffset(), chunkSize, etags,
+                                            activeParts, futureParts, errors, maxRetries, errorMap, toRetry, latch,
+                                            blob2Object);
+                                }
+                                if (remaining > 0) {
+                                    Integer partKey = new Integer(part);
+                                    activeParts.put(partKey);
+                                    prepareUploadPart(container, blob, key, partKey, payload,
+                                            algorithm.getNextChunkOffset(), remaining, etags,
+                                            activeParts, futureParts, errors, maxRetries, errorMap, toRetry, latch,
+                                            blob2Object);
+                                }
+                                latch.await();
+                                // handling retries
+                                while (errors.get() <= maxRetries && toRetry.size() > 0) {
+                                    int atOnce = Math.min(Math.min(toRetry.size(), errors.get()), parallelDegree);
+                                    CountDownLatch retryLatch = new CountDownLatch(atOnce);
+                                    for (int i = 0; i < atOnce; i++) {
+                                        Part failedPart = toRetry.poll();
+                                        Integer partKey = new Integer(failedPart.getPart());
+                                        activeParts.put(partKey);
+                                        prepareUploadPart(container, blob, key, partKey, payload,
+                                                failedPart.getOffset(), failedPart.getSize(), etags,
+                                                activeParts, futureParts, errors, maxRetries, errorMap, toRetry, retryLatch,
+                                                blob2Object);
+                                    }
+                                    retryLatch.await();
+                                }
+                                if (errors.get() > maxRetries) {
+                                    throw new BlobRuntimeException(String.format(
+                                            "Too many failed parts: %s while multipart upload of %s to container %s",
+                                            errors.get(), key, container));
+                                }
+
+                                String eTag = client.putObjectManifest(container, key);
+                                logger.debug(String.format("multipart upload of %s to container %s" +
+                                        " succeffully finished with %s retries", key, container, errors.get()));
+                                return eTag;
+                            } catch (Exception ex) {
+                                RuntimeException rtex = Throwables2.getFirstThrowableOfType(ex, RuntimeException.class);
+                                if (rtex == null) {
+                                    rtex = new RuntimeException(ex);
+                                }
+                                for (Map.Entry> entry : futureParts.entrySet()) {
+                                    entry.getValue().cancel(false);
+                                }
+                                /*
+                                if (uploadId != null) {
+                                    client.abortMultipartUpload(container, key, uploadId);
+                                } */
+                                throw rtex;
+                            }
+                        } else {
+                            ListenableFuture futureETag = ablobstore.putBlob(container, blob, options);
+                            return maxTime != null ?
+                                    futureETag.get(maxTime, TimeUnit.SECONDS) : futureETag.get();
+                        }
+                    }
+                }), ioWorkerExecutor);
+    }
+
+    class Part {
+        private int part;
+        private long offset;
+        private long size;
+
+        Part(int part, long offset, long size) {
+            this.part = part;
+            this.offset = offset;
+            this.size = size;
+        }
+
+        public int getPart() {
+            return part;
+        }
+
+        public void setPart(int part) {
+            this.part = part;
+        }
+
+        public long getOffset() {
+            return offset;
+        }
+
+        public void setOffset(long offset) {
+            this.offset = offset;
+        }
+
+        public long getSize() {
+            return size;
+        }
+
+        public void setSize(long size) {
+            this.size = size;
+        }
+    }
+}
diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java
index ed305371de..fdaaa3ec4d 100644
--- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java
+++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java
@@ -69,7 +69,7 @@ public class HPCloudObjectStorageAsyncBlobStore extends SwiftAsyncBlobStore {
             Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) {
       super(context, blobUtils, service, defaultLocation, locations, sync, async, container2ResourceMd,
                container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd,
-               blob2ObjectGetOptions, fetchBlobMetadataProvider);
+               blob2ObjectGetOptions, fetchBlobMetadataProvider, null);
       this.enableCDNAndCache = enableCDNAndCache;
    }
 

From 338d28325c7f520b53d2c968175aa4460d09e7b3 Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Tue, 10 Apr 2012 14:28:24 +0400
Subject: [PATCH 30/50] Clean up commented out useless function.

---
 .../openstack/swift/internal/StubSwiftAsyncClient.java       | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java
index 9a3b189990..7178760509 100644
--- a/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java
+++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/internal/StubSwiftAsyncClient.java
@@ -185,11 +185,6 @@ public class StubSwiftAsyncClient implements CommonSwiftAsyncClient {
       return objectProvider.create(null);
    }
 
-
-   /*public String putObjectManifest(String container, String name) {
-       return "stub";
-   }                 */
-
    @Override
    public ListenableFuture objectExists(String bucketName, String key) {
       return immediateFuture(containerToBlobs.get(bucketName).containsKey(key));

From a3c8023e84cc0841f39440fdaabba5cec5712675 Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Tue, 17 Apr 2012 15:21:26 +0400
Subject: [PATCH 31/50] Use CommonSwiftClient instead of SwiftClient in
 multipart code.

---
 .../cloudfiles/blobstore/CloudFilesAsyncBlobStore.java      | 6 ++++--
 .../strategy/internal/ParallelMultipartUploadStrategy.java  | 3 ++-
 .../internal/SequentialMultipartUploadStrategy.java         | 3 ++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java
index 68ad22f587..4d1cbb63e0 100644
--- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java
+++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesAsyncBlobStore.java
@@ -49,6 +49,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
 import com.google.common.base.Function;
 import com.google.common.base.Supplier;
 import com.google.common.util.concurrent.ListenableFuture;
+import org.jclouds.openstack.swift.blobstore.strategy.internal.AsyncMultipartUploadStrategy;
 
 /**
  * 
@@ -66,10 +67,11 @@ public class CloudFilesAsyncBlobStore extends SwiftAsyncBlobStore {
             BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
             ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object,
             ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions,
-            Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) {
+            Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache,
+            Provider multipartUploadStrategy) {
       super(context, blobUtils, service, defaultLocation, locations, sync, async, container2ResourceMd,
                container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd,
-               blob2ObjectGetOptions, fetchBlobMetadataProvider, null);
+               blob2ObjectGetOptions, fetchBlobMetadataProvider, multipartUploadStrategy);
       this.enableCDNAndCache = enableCDNAndCache;
    }
 
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java
index bdc86344c2..95e4fc8983 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java
@@ -13,6 +13,7 @@ import org.jclouds.concurrent.Futures;
 import org.jclouds.io.Payload;
 import org.jclouds.io.PayloadSlicer;
 import org.jclouds.logging.Logger;
+import org.jclouds.openstack.swift.CommonSwiftAsyncClient;
 import org.jclouds.openstack.swift.SwiftAsyncClient;
 import org.jclouds.openstack.swift.SwiftClient;
 import org.jclouds.openstack.swift.blobstore.SwiftAsyncBlobStore;
@@ -91,7 +92,7 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra
             latch.countDown();
             return;
         }
-        final SwiftAsyncClient client = (SwiftAsyncClient) ablobstore.getContext()
+        final CommonSwiftAsyncClient client = (CommonSwiftAsyncClient) ablobstore.getContext()
                 .getProviderSpecificContext().getAsyncApi();
         Payload chunkedPart = slicer.slice(payload, offset, size);
         logger.debug(String.format("async uploading part %s of %s to container %s", part, key, container));
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
index e11bd41ce3..d77012661b 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
@@ -12,6 +12,7 @@ import org.jclouds.blobstore.reference.BlobStoreConstants;
 import org.jclouds.io.Payload;
 import org.jclouds.io.PayloadSlicer;
 import org.jclouds.logging.Logger;
+import org.jclouds.openstack.swift.CommonSwiftClient;
 import org.jclouds.openstack.swift.SwiftClient;
 import org.jclouds.openstack.swift.blobstore.SwiftBlobStore;
 import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
@@ -52,7 +53,7 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
         int parts = algorithm.getParts();
         long chunkSize = algorithm.getChunkSize();
         if (parts > 0) {
-            SwiftClient client = (SwiftClient) ablobstore.getContext()
+            CommonSwiftClient client = (CommonSwiftClient) ablobstore.getContext()
                     .getProviderSpecificContext().getApi();
 
             try {

From 19e56692e396d563e160369be7ae7438dd42d8f6 Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Tue, 17 Apr 2012 18:22:34 +0400
Subject: [PATCH 32/50] Enable multipart for HPCloud.

---
 .../jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java   | 5 +++--
 .../strategy/internal/ParallelMultipartUploadStrategy.java  | 5 ++---
 .../blobstore/HPCloudObjectStorageAsyncBlobStore.java       | 6 ++++--
 .../blobstore/HPCloudObjectStorageBlobStore.java            | 6 ++++--
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java
index 40ba250abd..c0d8f2d099 100644
--- a/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java
+++ b/apis/cloudfiles/src/main/java/org/jclouds/cloudfiles/blobstore/CloudFilesBlobStore.java
@@ -60,10 +60,11 @@ public class CloudFilesBlobStore extends SwiftBlobStore {
             BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
             ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object,
             ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions,
-            Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) {
+            Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache,
+            Provider multipartUploadStrategy) {
       super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions,
                container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions,
-               fetchBlobMetadataProvider, null);
+               fetchBlobMetadataProvider, multipartUploadStrategy);
       this.enableCDNAndCache = enableCDNAndCache;
 
    }
diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java
index 95e4fc8983..d4331caaa5 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/ParallelMultipartUploadStrategy.java
@@ -14,8 +14,7 @@ import org.jclouds.io.Payload;
 import org.jclouds.io.PayloadSlicer;
 import org.jclouds.logging.Logger;
 import org.jclouds.openstack.swift.CommonSwiftAsyncClient;
-import org.jclouds.openstack.swift.SwiftAsyncClient;
-import org.jclouds.openstack.swift.SwiftClient;
+import org.jclouds.openstack.swift.CommonSwiftClient;
 import org.jclouds.openstack.swift.blobstore.SwiftAsyncBlobStore;
 import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
 import org.jclouds.util.Throwables2;
@@ -147,7 +146,7 @@ public class ParallelMultipartUploadStrategy implements AsyncMultipartUploadStra
                         long chunkSize = algorithm.getChunkSize();
                         long remaining = algorithm.getRemaining();
                         if (parts > 0) {
-                            SwiftClient client = (SwiftClient) ablobstore
+                            CommonSwiftClient client = (CommonSwiftClient) ablobstore
                                     .getContext().getProviderSpecificContext().getApi();
                             final Map> futureParts =
                                     new ConcurrentHashMap>();
diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java
index fdaaa3ec4d..847eb686ea 100644
--- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java
+++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageAsyncBlobStore.java
@@ -45,6 +45,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList;
 import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
+import org.jclouds.openstack.swift.blobstore.strategy.internal.AsyncMultipartUploadStrategy;
 
 import com.google.common.base.Function;
 import com.google.common.base.Supplier;
@@ -66,10 +67,11 @@ public class HPCloudObjectStorageAsyncBlobStore extends SwiftAsyncBlobStore {
             BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
             ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object,
             ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions,
-            Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) {
+            Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache,
+            Provider multipartUploadStrategy) {
       super(context, blobUtils, service, defaultLocation, locations, sync, async, container2ResourceMd,
                container2ContainerListOptions, container2ResourceList, object2Blob, blob2Object, object2BlobMd,
-               blob2ObjectGetOptions, fetchBlobMetadataProvider, null);
+               blob2ObjectGetOptions, fetchBlobMetadataProvider, multipartUploadStrategy);
       this.enableCDNAndCache = enableCDNAndCache;
    }
 
diff --git a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java
index cd11feb720..365cb294d0 100644
--- a/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java
+++ b/providers/hpcloud-objectstorage/src/main/java/org/jclouds/hpcloud/objectstorage/blobstore/HPCloudObjectStorageBlobStore.java
@@ -40,6 +40,7 @@ import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceList;
 import org.jclouds.openstack.swift.blobstore.functions.ContainerToResourceMetadata;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob;
 import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata;
+import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy;
 
 import com.google.common.base.Supplier;
 
@@ -59,10 +60,11 @@ public class HPCloudObjectStorageBlobStore extends SwiftBlobStore {
             BlobStoreListContainerOptionsToListContainerOptions container2ContainerListOptions,
             ContainerToResourceList container2ResourceList, ObjectToBlob object2Blob, BlobToObject blob2Object,
             ObjectToBlobMetadata object2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions,
-            Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache) {
+            Provider fetchBlobMetadataProvider, EnableCDNAndCache enableCDNAndCache,
+            Provider multipartUploadStrategy) {
       super(context, blobUtils, defaultLocation, locations, sync, container2ResourceMd, container2ContainerListOptions,
                container2ResourceList, object2Blob, blob2Object, object2BlobMd, blob2ObjectGetOptions,
-               fetchBlobMetadataProvider, null);
+               fetchBlobMetadataProvider, multipartUploadStrategy);
       this.enableCDNAndCache = enableCDNAndCache;
 
    }

From 52c144fce5f3f504d51672f62c1d80a179c48737 Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Wed, 18 Apr 2012 12:41:09 +0400
Subject: [PATCH 33/50] Remove deub prints and fall back to traditional upload
 if file is not large enough.

---
 .../internal/SequentialMultipartUploadStrategy.java      | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
index d77012661b..c723a51742 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
@@ -42,7 +42,6 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
 
     @Override
     public String execute(String container, Blob blob, PutOptions options, BlobToObject blob2Object) {
-        System.out.println("here we go");
         String key = blob.getMetadata().getName();
         Payload payload = blob.getPayload();
         MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm();
@@ -57,10 +56,8 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
                     .getProviderSpecificContext().getApi();
 
             try {
-                SortedMap etags = Maps.newTreeMap();
                 int part;
                 while ((part = algorithm.getNextPart()) <= parts) {
-                    System.out.println("Uploading part " + part);
                     Payload chunkedPart = slicer.slice(payload,
                             algorithm.getNextChunkOffset(), chunkSize);
                     Blob blobPart = ablobstore.blobBuilder(blob.getMetadata().getName() + PART_SEPARATOR +
@@ -70,7 +67,6 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
                 }
                 long remaining = algorithm.getRemaining();
                 if (remaining > 0) {
-                    System.out.println("Uploading tail.");
                     Payload chunkedPart = slicer.slice(payload,
                             algorithm.getNextChunkOffset(), remaining);
                     Blob blobPart = ablobstore.blobBuilder(blob.getMetadata().getName() + PART_SEPARATOR + 
@@ -84,11 +80,10 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
                 if (rtex == null) {
                     rtex = new RuntimeException(ex);
                 }
-                //client.abortMultipartUpload(container, key, uploadId);
                 throw rtex;
             }
-
+        } else {
+            return ablobstore.putBlob(container, blob, PutOptions.NONE);
         }
-        return "NOT IMPLEMENTED";
     }
 }

From 2fa96a9cb38e988f036eb2ef9969a45fba2c00c1 Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Wed, 18 Apr 2012 14:28:47 +0400
Subject: [PATCH 34/50] Remove useless comment.

---
 .../openstack/swift/blobstore/strategy/MultipartUpload.java   | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java
index 839f4774e2..9c2ead8253 100644
--- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java
+++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/MultipartUpload.java
@@ -1,9 +1,5 @@
 package org.jclouds.openstack.swift.blobstore.strategy;
 
-/*
-@author Roman Bogorodskiy
- */
-
 public interface MultipartUpload {
 
     /* Maximum number of parts per upload */

From 1b5462346b321720aeb6b110d70e14345825c31d Mon Sep 17 00:00:00 2001
From: Roman Bogorodskiy 
Date: Thu, 19 Apr 2012 17:18:27 +0400
Subject: [PATCH 35/50] Add a live test for swift multipart upload.

---
 .../SwiftBlobIntegrationLiveTest.java         | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
index 8268a4a339..dc6a9555d8 100644
--- a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
+++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
@@ -18,11 +18,20 @@
  */
 package org.jclouds.openstack.swift.blobstore.integration;
 
+import com.google.common.io.ByteStreams;
+import com.google.common.io.InputSupplier;
+import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.crypto.CryptoStreams;
+import org.testng.ITestContext;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import java.io.*;
+
 /**
  * 
  * @author James Murty
@@ -30,6 +39,9 @@ import org.testng.annotations.Test;
  */
 @Test(groups = "live")
 public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
+   private InputSupplier oneHundredOneConstitutions;
+   private byte[] oneHundredOneConstitutionsMD5;
+
    public SwiftBlobIntegrationLiveTest() {
       provider = "swift";
    }
@@ -39,6 +51,13 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
       // not supported in swift
    }
 
+    @BeforeClass(groups = { "integration", "live" }, dependsOnMethods = "setupContext")
+    @Override
+    public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
+        super.setUpResourcesOnThisThread(testContext);
+        oneHundredOneConstitutions = getTestDataSupplier();
+        oneHundredOneConstitutionsMD5 = CryptoStreams.md5(oneHundredOneConstitutions);
+    }
 
    @Override
    protected void checkContentDisposition(Blob blob, String contentDisposition) {
@@ -61,4 +80,22 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
       return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, { "colon:" },
                { "asteri*k" }, { "{greaten" }, { "p|pe" } };
    }
+    
+   public void testMultipartChunkedFileStream() throws IOException, InterruptedException {
+       FileOutputStream fous = new FileOutputStream(new File("target/const.txt"));
+       ByteStreams.copy(oneHundredOneConstitutions.getInput(), fous);
+       fous.flush();
+       fous.close();
+       String containerName = getContainerName();
+
+       try {
+           BlobStore blobStore = context.getBlobStore();
+           blobStore.createContainerInLocation(null, containerName);
+           Blob blob = blobStore.blobBuilder("const.txt")
+                   .payload(new File("target/const.txt")).build();
+           blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
+       } finally {
+           returnContainer(containerName);
+       }
+   }
 }

From d89526212d4b340c2b51e8d809c13c3195242f03 Mon Sep 17 00:00:00 2001
From: danikov 
Date: Wed, 18 Apr 2012 16:59:01 +0100
Subject: [PATCH 36/50] bugfix for self-referential loop + test

---
 .../org/jclouds/util/ConcreteFunction.java    | 45 +++++++++++++++++++
 .../test/java/org/jclouds/util/Maps2Test.java | 15 +++++++
 2 files changed, 60 insertions(+)
 create mode 100644 core/src/main/java/org/jclouds/util/ConcreteFunction.java

diff --git a/core/src/main/java/org/jclouds/util/ConcreteFunction.java b/core/src/main/java/org/jclouds/util/ConcreteFunction.java
new file mode 100644
index 0000000000..27ac98c67d
--- /dev/null
+++ b/core/src/main/java/org/jclouds/util/ConcreteFunction.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.util;
+
+import com.google.common.base.Function;
+
+
+/**
+ * For wrapping covariant functions for passing to non-covariant methods
+ * 
+ * @author danikov
+ */
+public class ConcreteFunction implements Function {
+   private final Function delegate;
+   
+   public static  ConcreteFunction wrap(Function delegate) {
+      return new ConcreteFunction(delegate);
+   }
+
+   public ConcreteFunction(Function delegate) {
+      this.delegate = delegate;
+   }
+
+   @Override
+   public T apply(F input) {
+      return delegate.apply(input);
+   }
+
+}
diff --git a/core/src/test/java/org/jclouds/util/Maps2Test.java b/core/src/test/java/org/jclouds/util/Maps2Test.java
index d0f3cbf1fd..8c52536fa7 100644
--- a/core/src/test/java/org/jclouds/util/Maps2Test.java
+++ b/core/src/test/java/org/jclouds/util/Maps2Test.java
@@ -31,6 +31,7 @@ import org.testng.annotations.Test;
 import com.google.common.base.Function;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 /**
@@ -86,4 +87,18 @@ public class Maps2Test {
             }), expected);
    }
    
+   @Test
+   public void testCovariantUniqueIndex() {
+      Iterable values = Lists.newArrayList(1, 2, 3, 4, 5);
+      Map map = Maps2.uniqueIndex(values, new Function() {
+
+         @Override
+         public Double apply(Object input) {
+            return (Integer)input + 0.1;
+         }
+      });
+      
+      assertEquals(map.get(1.1), 1);
+   }
+   
 }

From 4dc5e0f781982e932647a0bb0f031d0740c0a777 Mon Sep 17 00:00:00 2001
From: danikov 
Date: Thu, 19 Apr 2012 10:20:07 +0100
Subject: [PATCH 37/50] functions and endpoints for vCloud Director

---
 .../director/v1_5/endpoints/Catalog.java      |  39 +++++++
 .../director/v1_5/endpoints/Network.java      |  39 +++++++
 .../vcloud/director/v1_5/endpoints/Org.java   |  39 +++++++
 .../director/v1_5/endpoints/OrgList.java      |  39 +++++++
 .../director/v1_5/endpoints/TasksList.java    |  39 +++++++
 .../director/v1_5/endpoints/VCloudLogin.java  |  40 +++++++
 .../vcloud/director/v1_5/endpoints/Vdc.java   |  39 +++++++
 .../functions/AllCatalogItemsInCatalog.java   |  81 ++++++++++++++
 .../v1_5/functions/AllCatalogItemsInOrg.java  |  59 +++++++++++
 .../v1_5/functions/AllCatalogsInOrg.java      |  68 ++++++++++++
 .../director/v1_5/functions/AllVdcsInOrg.java |  71 +++++++++++++
 .../DefaultNetworkNameInTemplate.java         |  54 ++++++++++
 .../OrgNameAndCatalogNameToEndpoint.java      |  78 ++++++++++++++
 .../OrgNameAndVdcNameToEndpoint.java          |  77 ++++++++++++++
 .../OrgNameCatalogNameItemNameToEndpoint.java |  78 ++++++++++++++
 ...CatalogNameVAppTemplateNameToEndpoint.java |  87 +++++++++++++++
 .../v1_5/functions/OrgNameToEndpoint.java     |  59 +++++++++++
 .../functions/OrgNameToTasksListEndpoint.java |  64 +++++++++++
 .../OrgNameVdcNameNetworkNameToEndpoint.java  |  59 +++++++++++
 ...meVdcNameResourceEntityNameToEndpoint.java |  59 +++++++++++
 .../OrgNameVdcNameResourceNameToEndpoint.java |  79 ++++++++++++++
 .../v1_5/functions/OrgsForLocations.java      |  92 ++++++++++++++++
 .../director/v1_5/functions/OrgsForNames.java |  77 ++++++++++++++
 .../v1_5/functions/ReferenceToEndpoint.java   |  39 +++++++
 .../v1_5/functions/ReferenceToName.java       |  37 +++++++
 .../functions/SectionForVAppTemplate.java     |  56 ++++++++++
 .../VAppTemplatesForCatalogItems.java         | 100 ++++++++++++++++++
 .../v1_5/functions/VAppTemplatesInOrg.java    |  70 ++++++++++++
 28 files changed, 1718 insertions(+)
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Catalog.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Network.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Org.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/OrgList.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/TasksList.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/VCloudLogin.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Vdc.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogItemsInCatalog.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogItemsInOrg.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogsInOrg.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllVdcsInOrg.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/DefaultNetworkNameInTemplate.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameAndCatalogNameToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameAndVdcNameToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameCatalogNameItemNameToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameCatalogNameVAppTemplateNameToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameToTasksListEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameNetworkNameToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameResourceEntityNameToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameResourceNameToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgsForLocations.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgsForNames.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/ReferenceToEndpoint.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/ReferenceToName.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/SectionForVAppTemplate.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/VAppTemplatesForCatalogItems.java
 create mode 100644 labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/VAppTemplatesInOrg.java

diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Catalog.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Catalog.java
new file mode 100644
index 0000000000..ed25c38a9b
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Catalog.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.endpoints;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.inject.Qualifier;
+
+/**
+ * Related to a VCloud Catalog.
+ * 
+ * @author Adrian Cole
+ * 
+ */
+@Retention(value = RetentionPolicy.RUNTIME)
+@Target(value = { ElementType.TYPE, ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
+@Qualifier
+public @interface Catalog {
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Network.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Network.java
new file mode 100644
index 0000000000..d140afa967
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Network.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.endpoints;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.inject.Qualifier;
+
+/**
+ * Related to a VCloud Network.
+ * 
+ * @author Adrian Cole
+ * 
+ */
+@Retention(value = RetentionPolicy.RUNTIME)
+@Target(value = { ElementType.TYPE, ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
+@Qualifier
+public @interface Network {
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Org.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Org.java
new file mode 100644
index 0000000000..d774605ffd
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Org.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.endpoints;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.inject.Qualifier;
+
+/**
+ * Related to a VCloud express Org.
+ * 
+ * @author Adrian Cole
+ * 
+ */
+@Retention(value = RetentionPolicy.RUNTIME)
+@Target(value = { ElementType.TYPE, ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
+@Qualifier
+public @interface Org {
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/OrgList.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/OrgList.java
new file mode 100644
index 0000000000..041ed0fb28
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/OrgList.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.endpoints;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.inject.Qualifier;
+
+/**
+ * Related to a VCloud express Org List.
+ * 
+ * @author Adrian Cole
+ * 
+ */
+@Retention(value = RetentionPolicy.RUNTIME)
+@Target(value = { ElementType.TYPE, ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
+@Qualifier
+public @interface OrgList {
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/TasksList.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/TasksList.java
new file mode 100644
index 0000000000..8f97c5ffb2
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/TasksList.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.endpoints;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.inject.Qualifier;
+
+/**
+ * Related to a VCloud express Task List.
+ * 
+ * @author Adrian Cole
+ * 
+ */
+@Retention(value = RetentionPolicy.RUNTIME)
+@Target(value = { ElementType.TYPE, ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
+@Qualifier
+public @interface TasksList {
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/VCloudLogin.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/VCloudLogin.java
new file mode 100644
index 0000000000..a97b295bd0
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/VCloudLogin.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.endpoints;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.inject.Qualifier;
+
+/**
+ * Represents a component related to vCloud.
+ * 
+ * @see 
+ * @author Adrian Cole
+ * 
+ */
+@Retention(value = RetentionPolicy.RUNTIME)
+@Target(value = { ElementType.TYPE, ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
+@Qualifier
+public @interface VCloudLogin {
+
+}
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Vdc.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Vdc.java
new file mode 100644
index 0000000000..a53f515f62
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/endpoints/Vdc.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.endpoints;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.inject.Qualifier;
+
+/**
+ * Related to a VCloud express Catalog.
+ * 
+ * @author Adrian Cole
+ * 
+ */
+@Retention(value = RetentionPolicy.RUNTIME)
+@Target(value = {ElementType.TYPE, ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
+@Qualifier
+public @interface Vdc {
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogItemsInCatalog.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogItemsInCatalog.java
new file mode 100644
index 0000000000..d9d408d75b
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogItemsInCatalog.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.collect.Iterables.filter;
+import static org.jclouds.concurrent.FutureIterables.transformParallel;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import org.jclouds.Constants;
+import org.jclouds.logging.Logger;
+import org.jclouds.vcloud.director.v1_5.VCloudDirectorMediaType;
+import org.jclouds.vcloud.director.v1_5.domain.Catalog;
+import org.jclouds.vcloud.director.v1_5.domain.CatalogItem;
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorAsyncClient;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class AllCatalogItemsInCatalog implements Function> {
+   @Resource
+   public Logger logger = Logger.NULL;
+
+   private final VCloudDirectorAsyncClient aclient;
+   private final ExecutorService executor;
+
+   @Inject
+   AllCatalogItemsInCatalog(VCloudDirectorAsyncClient aclient, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
+      this.aclient = aclient;
+      this.executor = executor;
+   }
+
+   @Override
+   public Iterable apply(Catalog from) {
+
+      Iterable catalogItems = transformParallel(filter(from.getCatalogItems(), new Predicate() {
+
+         @Override
+         public boolean apply(Reference input) {
+            return input.getType().equals(VCloudDirectorMediaType.CATALOG_ITEM);
+         }
+
+      }), new Function>() {
+
+         @Override
+         public Future apply(Reference from) {
+            return aclient.getCatalogClient().getCatalogItem(from.getHref());
+         }
+
+      }, executor, null, logger, "catalogItems in " + from.getHref());
+      return catalogItems;
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogItemsInOrg.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogItemsInOrg.java
new file mode 100644
index 0000000000..d72568246b
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogItemsInOrg.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Catalog;
+import org.jclouds.vcloud.director.v1_5.domain.CatalogItem;
+import org.jclouds.vcloud.director.v1_5.domain.org.Org;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class AllCatalogItemsInOrg implements Function> {
+
+   private final Function> allCatalogsInOrg;
+
+   private final Function> allCatalogItemsInCatalog;
+
+   @Inject
+   AllCatalogItemsInOrg(Function> allCatalogsInOrg,
+            Function> allCatalogItemsInCatalog) {
+      this.allCatalogsInOrg = allCatalogsInOrg;
+      this.allCatalogItemsInCatalog = allCatalogItemsInCatalog;
+   }
+
+   @Override
+   public Iterable apply(Org from) {
+      return Iterables.concat(Iterables.transform(allCatalogsInOrg.apply(from),
+               new Function>() {
+                  @Override
+                  public Iterable apply(Catalog from) {
+                     return allCatalogItemsInCatalog.apply(from);
+                  }
+
+               }));
+   }
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogsInOrg.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogsInOrg.java
new file mode 100644
index 0000000000..5b2c5d6b84
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllCatalogsInOrg.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import org.jclouds.Constants;
+import org.jclouds.concurrent.FutureIterables;
+import org.jclouds.logging.Logger;
+import org.jclouds.vcloud.director.v1_5.domain.Catalog;
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.domain.org.AdminOrg;
+import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorAsyncClient;
+
+import com.google.common.base.Function;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class AllCatalogsInOrg implements Function> {
+   @Resource
+   public Logger logger = Logger.NULL;
+
+   private final VCloudDirectorAsyncClient aclient;
+   private final ExecutorService executor;
+
+   @Inject
+   AllCatalogsInOrg(VCloudDirectorAsyncClient aclient, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
+      this.aclient = aclient;
+      this.executor = executor;
+   }
+
+   @Override
+   public Iterable apply(final AdminOrg org) {
+      Iterable catalogs = FutureIterables.transformParallel(org.getCatalogs(),
+            new Function>() {
+               @Override
+               public Future apply(Reference from) {
+                  return aclient.getCatalogClient().getCatalog(from.getHref());
+               }
+
+            }, executor, null, logger, "catalogs in " + org.getName());
+      return catalogs;
+   }
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllVdcsInOrg.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllVdcsInOrg.java
new file mode 100644
index 0000000000..a17407e386
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/AllVdcsInOrg.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static org.jclouds.concurrent.FutureIterables.transformParallel;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import org.jclouds.Constants;
+import org.jclouds.logging.Logger;
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.domain.Vdc;
+import org.jclouds.vcloud.director.v1_5.domain.org.AdminOrg;
+import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorAsyncClient;
+
+import com.google.common.base.Function;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class AllVdcsInOrg implements Function> {
+   @Resource
+   public Logger logger = Logger.NULL;
+
+   private final VCloudDirectorAsyncClient aclient;
+   private final ExecutorService executor;
+
+   @Inject
+   AllVdcsInOrg(VCloudDirectorAsyncClient aclient, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
+      this.aclient = aclient;
+      this.executor = executor;
+   }
+
+   @Override
+   public Iterable apply(final AdminOrg org) {
+
+      Iterable catalogItems = transformParallel(org.getVdcs(),
+            new Function>() {
+               @Override
+               public Future apply(Reference from) {
+                  return aclient.getVdcClient().getVdc(from.getHref());
+               }
+
+            }, executor, null, logger, "vdcs in org " + org.getName());
+      return catalogItems;
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/DefaultNetworkNameInTemplate.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/DefaultNetworkNameInTemplate.java
new file mode 100644
index 0000000000..5760a2f0af
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/DefaultNetworkNameInTemplate.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Iterables.get;
+
+import java.util.Set;
+
+import javax.annotation.Resource;
+import javax.inject.Singleton;
+
+import org.jclouds.dmtf.ovf.Network;
+import org.jclouds.dmtf.ovf.NetworkSection;
+import org.jclouds.logging.Logger;
+import org.jclouds.vcloud.director.v1_5.domain.VAppTemplate;
+
+import com.google.common.base.Function;
+
+@Singleton
+public class DefaultNetworkNameInTemplate implements Function {
+   @Resource
+   protected Logger logger = Logger.NULL;
+   
+   private SectionForVAppTemplate networkSelector = 
+         new SectionForVAppTemplate(NetworkSection.class);
+
+   @Override
+   public String apply(VAppTemplate vAppTemplate) {
+      checkArgument(vAppTemplate != null, "vAppTemplate was null!");
+      vAppTemplate.getSections();
+      Set networks = networkSelector.apply(vAppTemplate).getNetworks();
+      checkArgument(networks.size() > 0, "no networks found in vAppTemplate %s", vAppTemplate);
+      if (networks.size() > 1)
+         logger.warn("multiple networks found for %s, choosing first from: %s", vAppTemplate.getName(), networks);
+      return get(networks, 0).getName();
+   }
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameAndCatalogNameToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameAndCatalogNameToEndpoint.java
new file mode 100644
index 0000000000..62b571d136
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameAndCatalogNameToEndpoint.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.jclouds.vcloud.director.v1_5.predicates.ReferencePredicates.nameEquals;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.domain.org.AdminOrg;
+import org.jclouds.vcloud.director.v1_5.endpoints.Catalog;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author danikov
+ */
+@Singleton
+public class OrgNameAndCatalogNameToEndpoint implements Function {
+   private final Supplier> orgMap;
+   private final Supplier defaultOrg;
+   private final Supplier defaultCatalog;
+
+   @Inject
+   public OrgNameAndCatalogNameToEndpoint(Supplier> orgMap,
+         @org.jclouds.vcloud.director.v1_5.endpoints.Org Supplier defaultOrg,
+         @Catalog Supplier defaultCatalog) {
+      this.orgMap = orgMap;
+      this.defaultOrg = defaultOrg;
+      this.defaultCatalog = defaultCatalog;
+   }
+
+   @SuppressWarnings("unchecked")
+   public URI apply(Object from) {
+      Iterable orgCatalog = (Iterable) checkNotNull(from, "args");
+      Object org = Iterables.get(orgCatalog, 0);
+      Object catalog = Iterables.get(orgCatalog, 1);
+      if (org == null && catalog == null)
+         return defaultCatalog.get().getHref();
+      else if (org == null)
+         org = defaultOrg.get().getName();
+
+      try {
+         Set catalogs = checkNotNull(orgMap.get().get(org)).getCatalogs();
+         return catalog == null ? Iterables.getLast(catalogs).getHref() : 
+            Iterables.find(catalogs, nameEquals((String)catalog)).getHref();
+      } catch (NullPointerException e) {
+         throw new NoSuchElementException(org + "/" + catalog + " not found in " + orgMap.get());
+      }
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameAndVdcNameToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameAndVdcNameToEndpoint.java
new file mode 100644
index 0000000000..cfcbc34d58
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameAndVdcNameToEndpoint.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.jclouds.vcloud.director.v1_5.predicates.ReferencePredicates.nameEquals;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.domain.org.AdminOrg;
+import org.jclouds.vcloud.director.v1_5.endpoints.Vdc;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author danikov
+ */
+@Singleton
+public class OrgNameAndVdcNameToEndpoint implements Function {
+   private final Supplier> orgNameToVdcEndpoint;
+   private final Supplier defaultOrg;
+   private final Supplier defaultVdc;
+
+   @Inject
+   public OrgNameAndVdcNameToEndpoint(Supplier> orgNameToVDCEndpoint,
+         @org.jclouds.vcloud.director.v1_5.endpoints.Org Supplier defaultOrg, @Vdc Supplier defaultVdc) {
+      this.orgNameToVdcEndpoint = orgNameToVDCEndpoint;
+      this.defaultOrg = defaultOrg;
+      this.defaultVdc = defaultVdc;
+   }
+
+   @SuppressWarnings("unchecked")
+   public URI apply(Object from) {
+      Iterable orgVdc = (Iterable) checkNotNull(from, "args");
+      Object org = Iterables.get(orgVdc, 0);
+      Object vdc = Iterables.get(orgVdc, 1);
+      if (org == null && vdc == null)
+         return defaultVdc.get().getHref();
+      else if (org == null)
+         org = defaultOrg.get().getName();
+
+      try {
+        Set vdcs = checkNotNull(orgNameToVdcEndpoint.get().get(org)).getVdcs();
+         return vdc == null ? Iterables.getLast(vdcs).getHref() : 
+            Iterables.find(vdcs, nameEquals((String)vdc)).getHref();
+      } catch (NullPointerException e) {
+         throw new NoSuchElementException(org + "/" + vdc + " not found in " + orgNameToVdcEndpoint.get());
+      }
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameCatalogNameItemNameToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameCatalogNameItemNameToEndpoint.java
new file mode 100644
index 0000000000..5f327c9d21
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameCatalogNameItemNameToEndpoint.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.jclouds.vcloud.director.v1_5.predicates.ReferencePredicates.nameEquals;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.domain.Catalog;
+import org.jclouds.vcloud.director.v1_5.endpoints.Org;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author danikov
+ */
+@Singleton
+public class OrgNameCatalogNameItemNameToEndpoint implements Function {
+   private final Supplier>> orgCatalogMap;
+   private final Supplier defaultOrg;
+   private final Supplier defaultCatalog;
+
+   @Inject
+   public OrgNameCatalogNameItemNameToEndpoint(
+         Supplier>> orgCatalogMap,
+         @Org Supplier defaultOrg, 
+         @org.jclouds.vcloud.director.v1_5.endpoints.Catalog Supplier defaultCatalog) {
+      this.orgCatalogMap = orgCatalogMap;
+      this.defaultOrg = defaultOrg;
+      this.defaultCatalog = defaultCatalog;
+   }
+
+   @SuppressWarnings("unchecked")
+   public URI apply(Object from) {
+      Iterable orgCatalog = (Iterable) checkNotNull(from, "args");
+      Object org = Iterables.get(orgCatalog, 0);
+      Object catalog = Iterables.get(orgCatalog, 1);
+      Object catalogItem = Iterables.get(orgCatalog, 2);
+      if (org == null)
+         org = defaultOrg.get().getName();
+      if (catalog == null)
+         catalog = defaultCatalog.get().getName();
+      try {
+         Map catalogs = checkNotNull(orgCatalogMap.get().get(org));
+         return Iterables.find(catalogs.get(catalog).getCatalogItems(), nameEquals((String)catalogItem)).getHref();
+      } catch (NullPointerException e) {
+         throw new NoSuchElementException(org + "/" + catalog + "/" + catalogItem + " not found in "
+               + orgCatalogMap.get());
+      }
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameCatalogNameVAppTemplateNameToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameCatalogNameVAppTemplateNameToEndpoint.java
new file mode 100644
index 0000000000..d49e2fe787
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameCatalogNameVAppTemplateNameToEndpoint.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.CatalogItem;
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.endpoints.Catalog;
+import org.jclouds.vcloud.director.v1_5.endpoints.Org;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author danikov
+ */
+@Singleton
+public class OrgNameCatalogNameVAppTemplateNameToEndpoint implements Function {
+   private final Supplier>>> orgCatalogItemMap;
+   private final Supplier defaultOrg;
+   private final Supplier defaultCatalog;
+
+   @Inject
+   public OrgNameCatalogNameVAppTemplateNameToEndpoint(
+            Supplier>>> orgCatalogItemMap,
+            @Org Supplier defaultOrg, @Catalog Supplier defaultCatalog) {
+      this.orgCatalogItemMap = orgCatalogItemMap;
+      this.defaultOrg = defaultOrg;
+      this.defaultCatalog = defaultCatalog;
+   }
+
+   @SuppressWarnings("unchecked")
+   public URI apply(Object from) {
+      Iterable orgCatalog = (Iterable) checkNotNull(from, "args");
+      Object org = Iterables.get(orgCatalog, 0);
+      Object catalog = Iterables.get(orgCatalog, 1);
+      Object catalogItem = Iterables.get(orgCatalog, 2);
+      if (org == null)
+         org = defaultOrg.get().getName();
+      if (catalog == null)
+         catalog = defaultCatalog.get().getName();
+      Map>> orgCatalogItemMap = this.orgCatalogItemMap.get();
+
+      if (!orgCatalogItemMap.containsKey(org))
+         throw new NoSuchElementException("org: " + org + " not found in " + orgCatalogItemMap.keySet());
+      Map> catalogs = orgCatalogItemMap.get(org);
+
+      if (!catalogs.containsKey(catalog))
+         throw new NoSuchElementException("catalog: " + org + "/" + catalog + " not found in " + catalogs.keySet());
+      Map catalogMap = catalogs.get(catalog);
+
+      if (!catalogMap.containsKey(catalogItem))
+         throw new NoSuchElementException("item: " + org + "/" + catalog + "/" + catalogItem + " not found in "
+                  + catalogMap.keySet());
+      CatalogItem item = catalogMap.get(catalogItem);
+
+      return checkNotNull(item.getEntity(), "item: " + org + "/" + catalog + "/" + catalogItem + " has no entity")
+               .getHref();
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameToEndpoint.java
new file mode 100644
index 0000000000..f385aee72e
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameToEndpoint.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.endpoints.Org;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+
+/**
+ * 
+ * @author danikov
+ */
+@Singleton
+public class OrgNameToEndpoint implements Function {
+   private final Supplier> orgNameToEndpointSupplier;
+   private final Supplier defaultOrg;
+
+   @Inject
+   public OrgNameToEndpoint(@Org Supplier> orgNameToEndpointSupplier,
+         @Org Supplier defaultOrg) {
+      this.orgNameToEndpointSupplier = orgNameToEndpointSupplier;
+      this.defaultOrg = defaultOrg;
+   }
+
+   public URI apply(Object from) {
+      try {
+         Map orgNameToEndpoint = orgNameToEndpointSupplier.get();
+         return from == null ? defaultOrg.get().getHref() : orgNameToEndpoint.get(from).getHref();
+      } catch (NullPointerException e) {
+         throw new NoSuchElementException("org " + from + " not found in " + orgNameToEndpointSupplier.get().keySet());
+      }
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameToTasksListEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameToTasksListEndpoint.java
new file mode 100644
index 0000000000..9e124fa8f7
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameToTasksListEndpoint.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.domain.org.Org;
+import org.jclouds.vcloud.director.v1_5.endpoints.TasksList;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+
+/**
+ * 
+ * @author danikov
+ */
+@Singleton
+public class OrgNameToTasksListEndpoint implements Function {
+   private final Supplier> orgMap;
+   private final Supplier defaultTasksList;
+
+   @Inject
+   public OrgNameToTasksListEndpoint(Supplier> orgMap,
+         @TasksList Supplier defaultTasksList) {
+      this.orgMap = orgMap;
+      this.defaultTasksList = defaultTasksList;
+   }
+
+   public URI apply(Object from) {
+      Object org = from;
+      if (org == null)
+         return defaultTasksList.get().getHref();
+      try {
+         return checkNotNull(orgMap.get().get(org)).getHref();
+      } catch (NullPointerException e) {
+         throw new NoSuchElementException(org + " not found in " + orgMap.get());
+      }
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameNetworkNameToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameNetworkNameToEndpoint.java
new file mode 100644
index 0000000000..c11bf1d534
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameNetworkNameToEndpoint.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static org.jclouds.vcloud.director.v1_5.predicates.ReferencePredicates.nameEquals;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.endpoints.Org;
+import org.jclouds.vcloud.director.v1_5.endpoints.Vdc;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author danikov
+ */
+@Singleton
+public class OrgNameVdcNameNetworkNameToEndpoint extends OrgNameVdcNameResourceNameToEndpoint {
+   @Inject
+   public OrgNameVdcNameNetworkNameToEndpoint(
+         Supplier>> orgVdcMap,
+         @Org Supplier defaultOrg, @Vdc Supplier defaultVdc) {
+      super(orgVdcMap, defaultOrg, defaultVdc);
+   }
+
+   protected URI getEndpointOfResourceInVdc(Object org, Object Vdc, Object resource,
+         org.jclouds.vcloud.director.v1_5.domain.Vdc VdcObject) {
+      Reference resourceEntity = Iterables.find(VdcObject.getAvailableNetworks(), nameEquals((String)Vdc));
+      if (resourceEntity == null)
+         throw new NoSuchElementException("network " + resource + " in Vdc " + Vdc + ", org " + org + " not found in "
+               + VdcObject.getAvailableNetworks());
+      return resourceEntity.getHref();
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameResourceEntityNameToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameResourceEntityNameToEndpoint.java
new file mode 100644
index 0000000000..20e25663bf
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameResourceEntityNameToEndpoint.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static org.jclouds.vcloud.director.v1_5.predicates.ReferencePredicates.nameEquals;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.endpoints.Org;
+import org.jclouds.vcloud.director.v1_5.endpoints.Vdc;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author danikov
+ */
+@Singleton
+public class OrgNameVdcNameResourceEntityNameToEndpoint extends OrgNameVdcNameResourceNameToEndpoint {
+   @Inject
+   public OrgNameVdcNameResourceEntityNameToEndpoint(
+         Supplier>> orgVdcMap,
+         @Org Supplier defaultOrg, @Vdc Supplier defaultVdc) {
+      super(orgVdcMap, defaultOrg, defaultVdc);
+   }
+
+   protected URI getEndpointOfResourceInVdc(Object org, Object Vdc, Object resource,
+         org.jclouds.vcloud.director.v1_5.domain.Vdc VdcObject) {
+      Reference resourceEntity = Iterables.find(VdcObject.getResourceEntities(), nameEquals((String)resource));
+      if (resourceEntity == null)
+         throw new NoSuchElementException("entity " + resource + " in Vdc " + Vdc + ", org " + org + " not found in "
+               + VdcObject.getResourceEntities());
+      return resourceEntity.getHref();
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameResourceNameToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameResourceNameToEndpoint.java
new file mode 100644
index 0000000000..7ff831e3e2
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgNameVdcNameResourceNameToEndpoint.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import javax.inject.Inject;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.domain.Vdc;
+import org.jclouds.vcloud.director.v1_5.endpoints.Org;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+/**
+ * 
+ * @author danikov
+ */
+public abstract class OrgNameVdcNameResourceNameToEndpoint  implements Function{
+
+   protected final Supplier>> orgVdcMap;
+   protected final Supplier defaultOrg;
+   protected final Supplier defaultVdc;
+
+   @Inject
+   public OrgNameVdcNameResourceNameToEndpoint(
+         Supplier>> orgVdcMap,
+         @Org Supplier defaultOrg, @org.jclouds.vcloud.director.v1_5.endpoints.Vdc Supplier defaultVdc) {
+      this.orgVdcMap = orgVdcMap;
+      this.defaultOrg = defaultOrg;
+      this.defaultVdc = defaultVdc;
+   }
+
+   @SuppressWarnings("unchecked")
+   public URI apply(Object from) {
+      Iterable orgVdc = (Iterable) checkNotNull(from, "args");
+      Object org = Iterables.get(orgVdc, 0);
+      Object Vdc = Iterables.get(orgVdc, 1);
+      Object resource = Iterables.get(orgVdc, 2);
+      if (org == null)
+         org = defaultOrg.get().getName();
+      if (Vdc == null)
+         Vdc = defaultVdc.get().getName();
+      Map> orgToVdcs = orgVdcMap.get();
+      checkState(orgToVdcs != null, "could not get map of org name to Vdcs!");
+      Map Vdcs = orgToVdcs.get(org);
+      if (Vdcs == null)
+         throw new NoSuchElementException("org " + org + " not found in " + orgToVdcs.keySet());
+      org.jclouds.vcloud.director.v1_5.domain.Vdc VdcObject = Vdcs.get(Vdc);
+      if (VdcObject == null)
+         throw new NoSuchElementException("Vdc " + Vdc + " in org " + org + " not found in " + Vdcs.keySet());
+      return getEndpointOfResourceInVdc(org, Vdc, resource, VdcObject);
+   }
+
+   protected abstract URI getEndpointOfResourceInVdc(Object org, Object Vdc, Object resource, Vdc VdcObject);
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgsForLocations.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgsForLocations.java
new file mode 100644
index 0000000000..c34a039a06
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgsForLocations.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.collect.Iterables.filter;
+import static com.google.common.collect.Iterables.transform;
+import static org.jclouds.concurrent.FutureIterables.transformParallel;
+
+import java.net.URI;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import org.jclouds.Constants;
+import org.jclouds.domain.Location;
+import org.jclouds.domain.LocationScope;
+import org.jclouds.logging.Logger;
+import org.jclouds.vcloud.director.v1_5.domain.org.Org;
+import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorAsyncClient;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Sets;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class OrgsForLocations implements Function, Iterable> {
+   @Resource
+   public Logger logger = Logger.NULL;
+   private final VCloudDirectorAsyncClient aclient;
+   private final ExecutorService executor;
+
+   @Inject
+   OrgsForLocations(VCloudDirectorAsyncClient aclient, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
+      this.aclient = aclient;
+      this.executor = executor;
+   }
+
+   /**
+    * Zones are assignable, but we want regions. so we look for zones, whose
+    * parent is region. then, we use a set to extract the unique set.
+    */
+   @Override
+   public Iterable apply(Iterable from) {
+
+      return transformParallel(Sets.newLinkedHashSet(transform(filter(from, new Predicate() {
+
+         @Override
+         public boolean apply(Location input) {
+            return input.getScope() == LocationScope.ZONE;
+         }
+
+      }), new Function() {
+
+         @Override
+         public URI apply(Location from) {
+            return URI.create(from.getParent().getId());
+         }
+
+      })), new Function>() {
+
+         @Override
+         public Future apply(URI from) {
+            return aclient.getOrgClient().getOrg(from);
+         }
+
+      }, executor, null, logger, "organizations for uris");
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgsForNames.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgsForNames.java
new file mode 100644
index 0000000000..c1ae4bebc1
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/OrgsForNames.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static org.jclouds.concurrent.FutureIterables.transformParallel;
+import static org.jclouds.vcloud.director.v1_5.predicates.ReferencePredicates.nameEquals;
+
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import org.jclouds.Constants;
+import org.jclouds.logging.Logger;
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+import org.jclouds.vcloud.director.v1_5.domain.org.Org;
+import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorAsyncClient;
+import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorClient;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class OrgsForNames implements Function, Iterable> {
+   @Resource
+   public Logger logger = Logger.NULL;
+   private final VCloudDirectorAsyncClient aclient;
+   private final VCloudDirectorClient sclient;
+   private final ExecutorService executor;
+
+   @Inject
+   OrgsForNames(VCloudDirectorAsyncClient aclient, 
+         VCloudDirectorClient sclient, 
+         @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
+      this.aclient = aclient;
+      this.sclient = sclient;
+      this.executor = executor;
+   }
+
+   @Override
+   public Iterable apply(Iterable from) {
+      final Set orgs = sclient.getOrgClient().getOrgList().getOrgs();
+      
+      return transformParallel(from, new Function>() {
+
+         @Override
+         public Future apply(String from) {
+            return aclient.getOrgClient().getOrg(Iterables.find(orgs, nameEquals(from)).getHref());
+         }
+
+      }, executor, null, logger, "organizations for names");
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/ReferenceToEndpoint.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/ReferenceToEndpoint.java
new file mode 100644
index 0000000000..79829e5af9
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/ReferenceToEndpoint.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import java.net.URI;
+
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+
+import com.google.common.base.Function;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class ReferenceToEndpoint implements Function {
+   
+   @Override
+   public URI apply(Reference from) {
+      return from.getHref();
+   }
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/ReferenceToName.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/ReferenceToName.java
new file mode 100644
index 0000000000..2af735f471
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/ReferenceToName.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.Reference;
+
+import com.google.common.base.Function;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class ReferenceToName implements Function {
+   
+   @Override
+   public String apply(Reference from) {
+      return from.getName();
+   }
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/SectionForVAppTemplate.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/SectionForVAppTemplate.java
new file mode 100644
index 0000000000..2efcd27ee4
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/SectionForVAppTemplate.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.dmtf.ovf.SectionType;
+import org.jclouds.vcloud.director.v1_5.domain.VAppTemplate;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Sets;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class SectionForVAppTemplate implements Function {
+   
+   private final Class sectionType;
+
+   @Inject
+   SectionForVAppTemplate(Class sectionType) {
+      this.sectionType = sectionType;
+   }
+
+   @SuppressWarnings("unchecked")
+   @Override
+   public S apply(VAppTemplate from) {
+      Set sections = Sets.newLinkedHashSet();
+      for (SectionType section : from.getSections()) {
+         if (sectionType.isAssignableFrom(section.getClass())) {
+            return (S)section;
+         }
+      }
+      return null;
+   }
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/VAppTemplatesForCatalogItems.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/VAppTemplatesForCatalogItems.java
new file mode 100644
index 0000000000..6e674c72f9
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/VAppTemplatesForCatalogItems.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.collect.Iterables.filter;
+import static org.jclouds.concurrent.FutureIterables.transformParallel;
+import static org.jclouds.util.Throwables2.propagateOrNull;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import org.jclouds.Constants;
+import org.jclouds.compute.reference.ComputeServiceConstants;
+import org.jclouds.concurrent.ExceptionParsingListenableFuture;
+import org.jclouds.concurrent.Futures;
+import org.jclouds.logging.Logger;
+import org.jclouds.rest.AuthorizationException;
+import org.jclouds.vcloud.director.v1_5.VCloudDirectorMediaType;
+import org.jclouds.vcloud.director.v1_5.domain.CatalogItem;
+import org.jclouds.vcloud.director.v1_5.domain.VAppTemplate;
+import org.jclouds.vcloud.director.v1_5.user.VCloudDirectorAsyncClient;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class VAppTemplatesForCatalogItems implements Function, Iterable> {
+   @Resource
+   @Named(ComputeServiceConstants.COMPUTE_LOGGER)
+   public Logger logger = Logger.NULL;
+   private final VCloudDirectorAsyncClient aclient;
+   private final ExecutorService executor;
+   private final ReturnNullOnAuthorizationException returnNullOnAuthorizationException;
+
+   @Singleton
+   static class ReturnNullOnAuthorizationException implements Function {
+
+      public VAppTemplate apply(Exception from) {
+         if (from instanceof AuthorizationException) {
+            return null;
+         }
+         return propagateOrNull(from);
+      }
+   }
+
+   @Inject
+   VAppTemplatesForCatalogItems(VCloudDirectorAsyncClient aclient,
+            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor,
+            ReturnNullOnAuthorizationException returnNullOnAuthorizationException) {
+      this.aclient = aclient;
+      this.executor = executor;
+      this.returnNullOnAuthorizationException = returnNullOnAuthorizationException;
+   }
+
+   @Override
+   public Iterable apply(Iterable from) {
+      return transformParallel(filter(from, new Predicate() {
+
+         @Override
+         public boolean apply(CatalogItem input) {
+            return input.getEntity().getType().equals(VCloudDirectorMediaType.VAPP_TEMPLATE);
+         }
+
+      }), new Function>() {
+
+         @Override
+         public Future apply(CatalogItem from) {
+            return new ExceptionParsingListenableFuture(Futures.makeListenable(VCloudDirectorAsyncClient.class
+                     .cast(aclient).getVAppTemplateClient().getVAppTemplate(from.getEntity().getHref()), executor),
+                     returnNullOnAuthorizationException);
+         }
+
+      }, executor, null, logger, "vappTemplates in");
+   }
+
+}
\ No newline at end of file
diff --git a/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/VAppTemplatesInOrg.java b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/VAppTemplatesInOrg.java
new file mode 100644
index 0000000000..9d114a5b39
--- /dev/null
+++ b/labs/vcloud-director/src/main/java/org/jclouds/vcloud/director/v1_5/functions/VAppTemplatesInOrg.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.vcloud.director.v1_5.functions;
+
+import static com.google.common.base.Predicates.and;
+import static com.google.common.base.Predicates.notNull;
+import static com.google.common.collect.Iterables.filter;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.vcloud.director.v1_5.domain.CatalogItem;
+import org.jclouds.vcloud.director.v1_5.domain.ResourceEntity.Status;
+import org.jclouds.vcloud.director.v1_5.domain.VAppTemplate;
+import org.jclouds.vcloud.director.v1_5.domain.org.Org;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * @author danikov
+ */
+@Singleton
+public class VAppTemplatesInOrg implements Function> {
+
+   private final Function> allCatalogItemsInOrg;
+   private final Function, Iterable> vAppTemplatesForCatalogItems;
+
+   @Inject
+   VAppTemplatesInOrg(Function> allCatalogItemsInOrg,
+            Function, Iterable> vAppTemplatesForCatalogItems) {
+      this.allCatalogItemsInOrg = allCatalogItemsInOrg;
+      this.vAppTemplatesForCatalogItems = vAppTemplatesForCatalogItems;
+   }
+
+   @Override
+   public Iterable apply(Org from) {
+      Iterable catalogs = allCatalogItemsInOrg.apply(from);
+      Iterable vAppTemplates = vAppTemplatesForCatalogItems.apply(catalogs);
+      return filter(vAppTemplates, and(notNull(), new Predicate(){
+
+         //TODO: test this
+         @Override
+         public boolean apply(VAppTemplate input) {
+            if (input == null)
+               return false;
+            return ImmutableSet.of(Status.RESOLVED, Status.POWERED_OFF).contains(input.getStatus());
+         }
+         
+      }));
+   }
+
+}
\ No newline at end of file

From 66dc8787ae935c0ab2de96ac84ae9867b2dfee68 Mon Sep 17 00:00:00 2001
From: Adrian Cole 
Date: Thu, 19 Apr 2012 11:55:56 -0700
Subject: [PATCH 38/50] added javadoc

---
 core/src/main/java/org/jclouds/Wrapper.java | 25 +++++++++++++--------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/jclouds/Wrapper.java b/core/src/main/java/org/jclouds/Wrapper.java
index d80839f894..085cc9ab9a 100644
--- a/core/src/main/java/org/jclouds/Wrapper.java
+++ b/core/src/main/java/org/jclouds/Wrapper.java
@@ -24,7 +24,12 @@ import com.google.common.annotations.Beta;
 import com.google.common.reflect.TypeToken;
 
 /**
- * 
+ * {@link Wrapper} allows access to the provider-specific, or library-driven api
+ * behind an abstraction. One wrapped context can support multiple wrappers.
+ * 

+ * For example, the {@code CloudStackContext} can be wrapped by both
+ * {@code ComputeServiceContext} and {@code LoadBalancerServiceContext}, as the
+ * api covers these features.
  * 
  * @author Adrian Cole
  * 
@@ -39,26 +44,28 @@ public interface Wrapper {
    TypeToken getWrappedType();
 
    /**
-    * Return an object of the specified type to allow access to the wrapped context. If the wrapped
-    * context is not assignable from the supplied type, an {@link IllegalArgumentException} is
-    * thrown.
+    * Return an object of the specified type to allow access to the wrapped
+    * context. If the wrapped context is not assignable from the supplied type,
+    * an {@link IllegalArgumentException} is thrown.
     * 
     * @param type
-    *           the type of the context to be returned. The wrapped context must be assignable from
-    *           this type.
+    *           the type of the context to be returned. The wrapped context must
+    *           be assignable from this type.
     * @return an instance of the specified type
     * @throws IllegalArgumentException
-    *            if the wrapped context is not assignable from the specified class.
+    *            if the wrapped context is not assignable from the specified
+    *            class.
     * @see #getWrappedType()
     */
     C unwrap(TypeToken type) throws IllegalArgumentException;
-   
+
    /**
     * shortcut for {@code unwrap(TypeToken.of(clazz))}
+    * 
     * @see #unwrap(TypeToken)
     */
     C unwrap(Class clazz) throws IllegalArgumentException;
-   
+
    /**
     * shortcut for {@code unwrap(getWrappedType())}
     * 

From 9b80b984a47dae9fc7ba570353466653a9fba54e Mon Sep 17 00:00:00 2001
From: Andrew Bayer 
Date: Thu, 19 Apr 2012 13:02:47 -0700
Subject: [PATCH 39/50] Switch to calling apt-get update before every apt-get
 install.

I found that the previous "apt-get install || (apt-get update; apt-get
install) didn't work in all cases, but switching to always calling
apt-get update got things working properly.
---
 compute/src/test/resources/initscript_with_java.sh            | 4 ++--
 compute/src/test/resources/initscript_with_jboss.sh           | 4 ++--
 compute/src/test/resources/runscript.sh                       | 4 ++--
 .../src/test/resources/test_guest_additions_installer_init.sh | 2 +-
 scriptbuilder/src/main/resources/functions/installOpenJDK.sh  | 2 +-
 scriptbuilder/src/main/resources/functions/setupPublicCurl.sh | 2 +-
 .../src/test/resources/test_install_jdk_scriptbuilder.sh      | 4 ++--
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/compute/src/test/resources/initscript_with_java.sh b/compute/src/test/resources/initscript_with_java.sh
index d7bbeed3ea..093c4c2c2f 100644
--- a/compute/src/test/resources/initscript_with_java.sh
+++ b/compute/src/test/resources/initscript_with_java.sh
@@ -90,7 +90,7 @@ function ensure_cmd_or_install_package_apt(){
   local cmd=$1
   local pkg=$2
   
-  hash $cmd 2>/dev/null || apt-get-install $pkg || ( apt-get-update && apt-get-install $pkg )
+  hash $cmd 2>/dev/null || ( apt-get-update && apt-get-install $pkg )
 }
 
 function ensure_cmd_or_install_package_yum(){
@@ -155,7 +155,7 @@ END_OF_JCLOUDS_FILE
 function installOpenJDK() {
   if hash apt-get 2>/dev/null; then
     export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-6-openjdk}
-    test -d $JAVA_HOME || apt-get-install openjdk-6-jdk || ( apt-get-update && apt-get-install openjdk-6-jdk )
+    test -d $JAVA_HOME || ( apt-get-update && apt-get-install openjdk-6-jdk )
   elif hash yum 2>/dev/null; then
     export pkg=java-1.6.0-openjdk-devel
     yum --nogpgcheck -y install $pkg &&
diff --git a/compute/src/test/resources/initscript_with_jboss.sh b/compute/src/test/resources/initscript_with_jboss.sh
index edec50effd..f9bd79504b 100644
--- a/compute/src/test/resources/initscript_with_jboss.sh
+++ b/compute/src/test/resources/initscript_with_jboss.sh
@@ -90,7 +90,7 @@ function ensure_cmd_or_install_package_apt(){
   local cmd=$1
   local pkg=$2
   
-  hash $cmd 2>/dev/null || apt-get-install $pkg || ( apt-get-update && apt-get-install $pkg )
+  hash $cmd 2>/dev/null || ( apt-get-update && apt-get-install $pkg )
 }
 
 function ensure_cmd_or_install_package_yum(){
@@ -155,7 +155,7 @@ END_OF_JCLOUDS_FILE
 function installOpenJDK() {
   if hash apt-get 2>/dev/null; then
     export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-6-openjdk}
-    test -d $JAVA_HOME || apt-get-install openjdk-6-jdk || ( apt-get-update && apt-get-install openjdk-6-jdk )
+    test -d $JAVA_HOME || ( apt-get-update && apt-get-install openjdk-6-jdk )
   elif hash yum 2>/dev/null; then
     export pkg=java-1.6.0-openjdk-devel
     yum --nogpgcheck -y install $pkg &&
diff --git a/compute/src/test/resources/runscript.sh b/compute/src/test/resources/runscript.sh
index 1c45816838..44d5880cde 100644
--- a/compute/src/test/resources/runscript.sh
+++ b/compute/src/test/resources/runscript.sh
@@ -90,7 +90,7 @@ function ensure_cmd_or_install_package_apt(){
   local cmd=$1
   local pkg=$2
   
-  hash $cmd 2>/dev/null || apt-get-install $pkg || ( apt-get-update && apt-get-install $pkg )
+  hash $cmd 2>/dev/null || ( apt-get-update && apt-get-install $pkg )
 }
 
 function ensure_cmd_or_install_package_yum(){
@@ -155,7 +155,7 @@ END_OF_JCLOUDS_FILE
 function installOpenJDK() {
   if hash apt-get 2>/dev/null; then
     export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-6-openjdk}
-    test -d $JAVA_HOME || apt-get-install openjdk-6-jdk || ( apt-get-update && apt-get-install openjdk-6-jdk )
+    test -d $JAVA_HOME || ( apt-get-update && apt-get-install openjdk-6-jdk )
   elif hash yum 2>/dev/null; then
     export pkg=java-1.6.0-openjdk-devel
     yum --nogpgcheck -y install $pkg &&
diff --git a/labs/virtualbox/src/test/resources/test_guest_additions_installer_init.sh b/labs/virtualbox/src/test/resources/test_guest_additions_installer_init.sh
index cccddd00ef..53bae7b1cd 100644
--- a/labs/virtualbox/src/test/resources/test_guest_additions_installer_init.sh
+++ b/labs/virtualbox/src/test/resources/test_guest_additions_installer_init.sh
@@ -90,7 +90,7 @@ function ensure_cmd_or_install_package_apt(){
   local cmd=$1
   local pkg=$2
   
-  hash $cmd 2>/dev/null || apt-get-install $pkg || ( apt-get-update && apt-get-install $pkg )
+  hash $cmd 2>/dev/null || ( apt-get-update && apt-get-install $pkg )
 }
 
 function ensure_cmd_or_install_package_yum(){
diff --git a/scriptbuilder/src/main/resources/functions/installOpenJDK.sh b/scriptbuilder/src/main/resources/functions/installOpenJDK.sh
index 71539db08c..5634d5cd50 100644
--- a/scriptbuilder/src/main/resources/functions/installOpenJDK.sh
+++ b/scriptbuilder/src/main/resources/functions/installOpenJDK.sh
@@ -20,7 +20,7 @@ END_OF_JCLOUDS_FILE
 function installOpenJDK() {
   if hash apt-get 2>/dev/null; then
     export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-6-openjdk}
-    test -d $JAVA_HOME || apt-get-install openjdk-6-jdk || ( apt-get-update && apt-get-install openjdk-6-jdk )
+    test -d $JAVA_HOME || ( apt-get-update && apt-get-install openjdk-6-jdk )
   elif hash yum 2>/dev/null; then
     export pkg=java-1.6.0-openjdk-devel
     yum --nogpgcheck -y install $pkg &&
diff --git a/scriptbuilder/src/main/resources/functions/setupPublicCurl.sh b/scriptbuilder/src/main/resources/functions/setupPublicCurl.sh
index b0854d7813..25a6c6dff4 100644
--- a/scriptbuilder/src/main/resources/functions/setupPublicCurl.sh
+++ b/scriptbuilder/src/main/resources/functions/setupPublicCurl.sh
@@ -5,7 +5,7 @@ function ensure_cmd_or_install_package_apt(){
   local cmd=$1
   local pkg=$2
   
-  hash $cmd 2>/dev/null || apt-get-install $pkg || ( apt-get-update && apt-get-install $pkg )
+  hash $cmd 2>/dev/null || ( apt-get-update && apt-get-install $pkg )
 }
 
 function ensure_cmd_or_install_package_yum(){
diff --git a/scriptbuilder/src/test/resources/test_install_jdk_scriptbuilder.sh b/scriptbuilder/src/test/resources/test_install_jdk_scriptbuilder.sh
index 52c107410f..334cc4c868 100644
--- a/scriptbuilder/src/test/resources/test_install_jdk_scriptbuilder.sh
+++ b/scriptbuilder/src/test/resources/test_install_jdk_scriptbuilder.sh
@@ -90,7 +90,7 @@ function ensure_cmd_or_install_package_apt(){
   local cmd=$1
   local pkg=$2
   
-  hash $cmd 2>/dev/null || apt-get-install $pkg || ( apt-get-update && apt-get-install $pkg )
+  hash $cmd 2>/dev/null || ( apt-get-update && apt-get-install $pkg )
 }
 
 function ensure_cmd_or_install_package_yum(){
@@ -155,7 +155,7 @@ END_OF_JCLOUDS_FILE
 function installOpenJDK() {
   if hash apt-get 2>/dev/null; then
     export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-6-openjdk}
-    test -d $JAVA_HOME || apt-get-install openjdk-6-jdk || ( apt-get-update && apt-get-install openjdk-6-jdk )
+    test -d $JAVA_HOME || ( apt-get-update && apt-get-install openjdk-6-jdk )
   elif hash yum 2>/dev/null; then
     export pkg=java-1.6.0-openjdk-devel
     yum --nogpgcheck -y install $pkg &&

From e4abeecc4d270c20750321e8d3f4d83257642250 Mon Sep 17 00:00:00 2001
From: Gustavo Morozowski 
Date: Thu, 19 Apr 2012 22:09:18 -0300
Subject: [PATCH 40/50] base implemenation of nodepool

---
 labs/nodepool/pom.xml                         |  93 ++++++
 .../nodepool/PooledComputeService.java        |  28 ++
 .../internal/BasePooledComputeService.java    | 265 ++++++++++++++++
 .../java/org/jclouds/nodepool/AppTest.java    | 284 ++++++++++++++++++
 4 files changed, 670 insertions(+)
 create mode 100644 labs/nodepool/pom.xml
 create mode 100644 labs/nodepool/src/main/java/org/jclouds/nodepool/PooledComputeService.java
 create mode 100644 labs/nodepool/src/main/java/org/jclouds/nodepool/internal/BasePooledComputeService.java
 create mode 100644 labs/nodepool/src/test/java/org/jclouds/nodepool/AppTest.java

diff --git a/labs/nodepool/pom.xml b/labs/nodepool/pom.xml
new file mode 100644
index 0000000000..403658b301
--- /dev/null
+++ b/labs/nodepool/pom.xml
@@ -0,0 +1,93 @@
+
+
+  4.0.0
+  
+    jclouds-project
+    org.jclouds
+    1.5.0-SNAPSHOT
+        ../../project/pom.xml
+  
+  org.jclouds.labs
+  nodepool
+  jclouds nodepool
+  bundle
+  
+    UTF-8
+    0.1.46
+    ${test.aws.identity}
+    ${test.aws.credential}
+  
+  
+	
+		org.jclouds
+		jclouds-compute
+		${project.version}
+	
+   
+      org.jclouds
+      jclouds-core
+      ${project.version}
+      test-jar
+      test
+   
+    
+      org.jclouds
+      jclouds-compute
+      ${project.version}
+      test-jar
+      test
+    
+	
+		org.jclouds.provider
+		aws-ec2
+		${project.version}
+      test
+	
+	
+		org.jclouds.driver
+		jclouds-enterprise
+		${project.version}
+      test
+	
+	
+		org.jclouds.driver
+		jclouds-sshj
+		${project.version}
+      test
+	
+	
+		org.jclouds
+		jclouds-scriptbuilder
+		${project.version}
+      test
+	
+	
+		org.jclouds.driver
+		jclouds-jsch
+		${project.version}
+      test
+	
+	
+		com.jcraft
+		jsch
+      test
+     
+
+    
+        
+            
+                org.apache.felix
+                maven-bundle-plugin
+                
+                    
+                        ${project.artifactId}
+                        org.jclouds.nodepool*;version="${project.version}"
+                        org.jclouds*;version="${project.version}",*
+                    
+                
+            
+        
+    
+    
+
diff --git a/labs/nodepool/src/main/java/org/jclouds/nodepool/PooledComputeService.java b/labs/nodepool/src/main/java/org/jclouds/nodepool/PooledComputeService.java
new file mode 100644
index 0000000000..771a1bef9e
--- /dev/null
+++ b/labs/nodepool/src/main/java/org/jclouds/nodepool/PooledComputeService.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.nodepool;
+
+import org.jclouds.compute.ComputeService;
+import org.jclouds.compute.RunNodesException;
+
+public interface PooledComputeService extends ComputeService {
+
+   void startPool() throws RunNodesException;
+
+}
diff --git a/labs/nodepool/src/main/java/org/jclouds/nodepool/internal/BasePooledComputeService.java b/labs/nodepool/src/main/java/org/jclouds/nodepool/internal/BasePooledComputeService.java
new file mode 100644
index 0000000000..9bc399077c
--- /dev/null
+++ b/labs/nodepool/src/main/java/org/jclouds/nodepool/internal/BasePooledComputeService.java
@@ -0,0 +1,265 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.nodepool.internal;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.jclouds.compute.ComputeService;
+import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.compute.RunNodesException;
+import org.jclouds.compute.RunScriptOnNodesException;
+import org.jclouds.compute.domain.ComputeMetadata;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.compute.domain.Hardware;
+import org.jclouds.compute.domain.Image;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.NodeMetadataBuilder;
+import org.jclouds.compute.domain.Template;
+import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.compute.options.RunScriptOptions;
+import org.jclouds.compute.options.TemplateOptions;
+import org.jclouds.compute.predicates.NodePredicates;
+import org.jclouds.domain.Location;
+import org.jclouds.nodepool.PooledComputeService;
+import org.jclouds.scriptbuilder.domain.Statement;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ListenableFuture;
+
+public class BasePooledComputeService implements PooledComputeService {
+
+   private final ComputeService backingComputeService;
+   private final String backingGroup;
+   private final Template backingTemplate;
+   private final int minPoolSize;
+   private Map groupMapping;
+
+   public BasePooledComputeService(ComputeService backingComputeService, String backingGroup, Template backingTemplate, int minPoolSize) {
+      this.backingComputeService = backingComputeService;
+      this.backingGroup = backingGroup;
+      this.backingTemplate = backingTemplate;
+      this.minPoolSize = minPoolSize;
+   }
+
+   @Override
+   public void startPool() throws RunNodesException {
+      Set backingNodes =
+            backingComputeService.createNodesInGroup(backingGroup, minPoolSize, backingTemplate);
+      groupMapping = Maps.newHashMap();
+      for (NodeMetadata node : backingNodes) {
+         groupMapping.put(node, "unassigned");
+      }
+   }
+
+   @Override
+   public ComputeServiceContext getContext() {
+      return backingComputeService.getContext();
+   }
+
+   @Override
+   public TemplateBuilder templateBuilder() {
+      return backingComputeService.templateBuilder();
+   }
+
+   @Override
+   public TemplateOptions templateOptions() {
+      return backingComputeService.templateOptions();
+   }
+
+   @Override
+   public Set listHardwareProfiles() {
+      return ImmutableSet.of(backingTemplate.getHardware());
+   }
+
+   @Override
+   public Set listImages() {
+      return ImmutableSet.of(backingTemplate.getImage());
+   }
+
+   @Override
+   public Set listNodes() {
+      Set allocatedNodes = Sets.newLinkedHashSet();
+      for (ComputeMetadata node : backingComputeService.listNodes()) {
+         NodeMetadata metadata = backingComputeService.getNodeMetadata(node.getId());
+         String group = groupMapping.get(node);
+         if ("unassigned".equals(group))
+            continue;
+         NodeMetadata nodeWithUpdatedGroup =
+               NodeMetadataBuilder.fromNodeMetadata(metadata).group(group).build();
+         allocatedNodes.add(nodeWithUpdatedGroup);
+      }
+      return allocatedNodes;
+   }
+
+   @Override
+   public Set listAssignableLocations() {
+      return ImmutableSet.of(backingTemplate.getLocation());
+   }
+
+   @Override
+   public Set createNodesInGroup(String group,
+         int count, Template template) throws RunNodesException {
+      throw new RuntimeException("not implemented");
+   }
+
+   @Override
+   public Set createNodesInGroup(String group,
+         int count, TemplateOptions templateOptions)
+               throws RunNodesException {
+      throw new RuntimeException("not implemented");
+   }
+
+   @Override
+   public Set createNodesInGroup(String group,
+         int count) throws RunNodesException {
+      int allocatedCount = 0;
+      Set allocatedNodes = Sets.newLinkedHashSet();
+      for (NodeMetadata metadata : groupMapping.keySet()) {
+         if (groupMapping.get(metadata).equals("unassigned")) {
+            groupMapping.put(metadata, "group");
+            NodeMetadata nodeWithUpdatedGroup =
+                  NodeMetadataBuilder.fromNodeMetadata(metadata).group(group).build();
+            allocatedNodes.add(nodeWithUpdatedGroup);
+            allocatedCount += 1;
+            if (allocatedCount == count) break;
+         }
+      }
+      return allocatedNodes;
+   }
+
+   @Override
+   public void resumeNode(String id) {
+      backingComputeService.resumeNode(id);
+
+   }
+
+   @Override
+   public void resumeNodesMatching(Predicate filter) {
+      backingComputeService.resumeNodesMatching(filter);
+
+   }
+
+   @Override
+   public void suspendNode(String id) {
+      backingComputeService.suspendNode(id);
+   }
+
+   @Override
+   public void suspendNodesMatching(Predicate filter) {
+      backingComputeService.suspendNodesMatching(filter);
+   }
+
+   @Override
+   public void destroyNode(String id) {
+
+      backingComputeService.destroyNode(id);
+   }
+
+   @Override
+   public Set destroyNodesMatching(
+         Predicate filter) {
+      return backingComputeService.destroyNodesMatching(filter);
+   }
+
+   @Override
+   public void rebootNode(String id) {
+      backingComputeService.rebootNode(id);
+
+   }
+
+   @Override
+   public void rebootNodesMatching(Predicate filter) {
+      backingComputeService.rebootNodesMatching(filter);
+
+   }
+
+   @Override
+   public NodeMetadata getNodeMetadata(String id) {
+      return backingComputeService.getNodeMetadata(id);
+   }
+
+   @Override
+   public Set listNodesDetailsMatching(
+         Predicate filter) {
+      return backingComputeService.listNodesDetailsMatching(filter);
+   }
+
+   @Override
+   public Map runScriptOnNodesMatching(
+         Predicate filter, String runScript)
+               throws RunScriptOnNodesException {
+      // TODO Auto-generated method stub
+      return backingComputeService.runScriptOnNodesMatching(filter, runScript);
+   }
+
+   @Override
+   public Map runScriptOnNodesMatching(
+         Predicate filter, Statement runScript)
+               throws RunScriptOnNodesException {
+      return backingComputeService.runScriptOnNodesMatching(filter, runScript);
+   }
+
+   @Override
+   public Map runScriptOnNodesMatching(
+         Predicate filter, String runScript,
+         RunScriptOptions options) throws RunScriptOnNodesException {
+      return backingComputeService.runScriptOnNodesMatching(filter, runScript, options);
+   }
+
+   @Override
+   public Map runScriptOnNodesMatching(
+         Predicate filter, Statement runScript,
+         RunScriptOptions options) throws RunScriptOnNodesException {
+      return backingComputeService.runScriptOnNodesMatching(filter, runScript, options);
+   }
+
+   @Override
+   public ExecResponse runScriptOnNode(String id, Statement runScript,
+         RunScriptOptions options) {
+      return backingComputeService.runScriptOnNode(id, runScript, options);
+   }
+
+   @Override
+   public ListenableFuture submitScriptOnNode(String id,
+         Statement runScript, RunScriptOptions options) {
+      return backingComputeService.submitScriptOnNode(id, runScript, options);
+   }
+
+   @Override
+   public ExecResponse runScriptOnNode(String id, Statement runScript) {
+      return backingComputeService.runScriptOnNode(id, runScript);
+   }
+
+   @Override
+   public ExecResponse runScriptOnNode(String id, String runScript,
+         RunScriptOptions options) {
+      return backingComputeService.runScriptOnNode(id, runScript, options);
+   }
+
+   @Override
+   public ExecResponse runScriptOnNode(String id, String runScript) {
+      return backingComputeService.runScriptOnNode(id, runScript);
+   }
+
+}
diff --git a/labs/nodepool/src/test/java/org/jclouds/nodepool/AppTest.java b/labs/nodepool/src/test/java/org/jclouds/nodepool/AppTest.java
new file mode 100644
index 0000000000..1b07aafaf8
--- /dev/null
+++ b/labs/nodepool/src/test/java/org/jclouds/nodepool/AppTest.java
@@ -0,0 +1,284 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.nodepool;
+
+import static com.google.common.base.Throwables.propagate;
+import static com.google.common.collect.Iterables.getOnlyElement;
+import static org.jclouds.scriptbuilder.domain.Statements.newStatementList;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Properties;
+import java.util.logging.Logger;
+
+import junit.framework.TestCase;
+
+import org.jclouds.Constants;
+import org.jclouds.compute.ComputeService;
+import org.jclouds.compute.ComputeServiceContextFactory;
+import org.jclouds.compute.RunNodesException;
+import org.jclouds.compute.domain.ComputeMetadata;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.NodeState;
+import org.jclouds.compute.domain.OsFamily;
+import org.jclouds.compute.domain.Template;
+import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
+import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
+import org.jclouds.nodepool.internal.BasePooledComputeService;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+import org.jclouds.scriptbuilder.statements.java.InstallJDK;
+import org.jclouds.scriptbuilder.statements.login.AdminAccess;
+import org.jclouds.sshj.config.SshjSshClientModule;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Module;
+
+/**
+ * Unit test for simple App.
+ */
+public class AppTest extends TestCase {
+   private String identity;
+   private String credential;
+   private String providerName;
+   private File privateKey;
+   private File publicKey;
+   private String endPointUrl;
+   public String profile;
+   private int retentionTime;
+   public int instanceCap;
+
+   private String imageId;
+   private String osFamily;
+   private String osVersion;
+   private String hardwareId;
+   private int ram;
+   private int cores;
+   private String initScript;
+   private String name;
+   private boolean stopOnTerminate;
+   private ComputeService compute;
+   private PooledComputeService pooledCompute;
+   private Collection nodes = new LinkedList();
+   private static Logger LOGGER = Logger.getLogger("AppTest");
+   
+   private long time;
+
+   @Override
+   protected void setUp() throws Exception {
+      // setup JCloudsCloud
+      identity = "insert-your-identity-here";
+      credential = "insert-your-credential-here";
+      providerName = "aws-ec2";
+      privateKey = new File("private-key");
+      publicKey = new File("public-key");
+      endPointUrl = "";
+      profile = "aws-slave-profile";
+      retentionTime = -1;
+      instanceCap = 3;
+
+      // cloud instance template
+      name = "aws-jenkins-slave";
+      // numExecutors = 1;
+      // description = ""
+      imageId = "us-east-1/ami-4dad7424";
+      osFamily = "";
+      osVersion = "";
+      hardwareId = "t1.micro";
+      ram = -1;
+      cores = -1;
+      // labels = "whii";
+      initScript = "touch /tmp/hellothere";
+      stopOnTerminate = true;
+   }
+
+   /**
+    * Rigourous Test :-)
+    */
+   public void testApp() {
+      createCompute();
+      assertNotNull(compute);
+      createAndStartPool();
+      assertNotNull(pooledCompute);
+      for (int i = 0; i < 3; i++) {
+         startCounter();
+         provision("pool-1");
+         stopCounter();
+      }
+      for (int i = 0; i < 3; i++) {
+         startCounter();
+         provision("pool-2");
+         stopCounter();
+      }
+      for (int i = 0; i < 3; i++) {
+         startCounter();
+         provision("pool-3");
+         stopCounter();
+      }
+      assertEquals(9, getRunningNodesCount());
+      for (NodeMetadata slave : nodes) {
+         assertNotNull(slave);
+         LOGGER.info(slave.getId() + "-" + slave.getGroup());
+         terminate(slave);
+      }
+      assertEquals(0, getRunningNodesCount());
+   }
+
+   private void stopCounter() {
+      LOGGER.info("Elapsed time: " + (System.currentTimeMillis() - time));
+   }
+
+   private void startCounter() {
+      time = System.currentTimeMillis();      
+   }
+
+   public AppTest getCloud() {
+      return this;
+   }
+
+   public ComputeService getCompute() {
+      return pooledCompute;
+   }
+   
+   public NodeMetadata provision(String groupName) {
+      LOGGER.info("Provisioning new node");
+      NodeMetadata nodeMetadata = createNodeWithJdk(groupName);
+      nodes.add(nodeMetadata);
+      return nodeMetadata;
+   }
+
+   public int getRunningNodesCount() {
+      int nodeCount = 0;
+
+      for (ComputeMetadata cm : pooledCompute.listNodes()) {
+         if (NodeMetadata.class.isInstance(cm)) {
+            String nodeGroup = ((NodeMetadata) cm).getGroup();
+
+            if (!((NodeMetadata) cm).getState().equals(NodeState.SUSPENDED)
+                  && !((NodeMetadata) cm).getState().equals(NodeState.TERMINATED)) {
+               nodeCount++;
+            }
+         }
+      }
+      return nodeCount;
+   }
+
+   private void createCompute() {
+      Properties overrides = new Properties();
+      if (!Strings.isNullOrEmpty(this.endPointUrl)) {
+         overrides.setProperty(Constants.PROPERTY_ENDPOINT, this.endPointUrl);
+      }
+      Iterable modules = ImmutableSet. of(new SshjSshClientModule(), new SLF4JLoggingModule(),
+            new EnterpriseConfigurationModule());
+      this.compute = new ComputeServiceContextFactory().createContext(this.providerName, this.identity,
+            this.credential, modules, overrides).getComputeService();
+   }
+
+   private void createAndStartPool() {
+      LOGGER.info("creating jclouds nodepool");
+      ImmutableMap userMetadata = ImmutableMap.of("Name", name);
+      TemplateBuilder templateBuilder = compute.templateBuilder();
+      if (!Strings.isNullOrEmpty(imageId)) {
+         LOGGER.info("Setting image id to " + imageId);
+         templateBuilder.imageId(imageId);
+      } else {
+         if (!Strings.isNullOrEmpty(osFamily)) {
+            LOGGER.info("Setting osFamily to " + osFamily);
+            templateBuilder.osFamily(OsFamily.valueOf(osFamily));
+         }
+         if (!Strings.isNullOrEmpty(osVersion)) {
+            LOGGER.info("Setting osVersion to " + osVersion);
+            templateBuilder.osVersionMatches(osVersion);
+         }
+      }
+      if (!Strings.isNullOrEmpty((hardwareId))) {
+         LOGGER.info("Setting hardware Id to " + hardwareId);
+      } else {
+         LOGGER.info("Setting minRam " + ram + " and minCores " + cores);
+         templateBuilder.minCores(cores).minRam(ram);
+      }
+
+      Template template = templateBuilder.build();
+
+      // setup the jcloudTemplate to customize the nodeMetadata with jdk, etc.
+      // also opening ports
+      AdminAccess adminAccess = AdminAccess.builder().adminUsername("jenkins").installAdminPrivateKey(false) // no
+                                                                                                             // need
+            .grantSudoToAdminUser(false) // no need
+            .adminPrivateKey(getCloud().privateKey) // temporary due to jclouds
+                                                    // bug
+            .authorizeAdminPublicKey(true).adminPublicKey(getCloud().publicKey).build();
+
+      // Jenkins needs /jenkins dir.
+      Statement jenkinsDirStatement = Statements.newStatementList(Statements.exec("mkdir /jenkins"),
+            Statements.exec("chown jenkins /jenkins"));
+
+      Statement bootstrap = newStatementList(adminAccess, jenkinsDirStatement, Statements.exec(this.initScript),
+            InstallJDK.fromOpenJDK());
+
+      template.getOptions().inboundPorts(22).userMetadata(userMetadata).runScript(bootstrap);
+
+      pooledCompute = new BasePooledComputeService(compute, "jenkins-pool", template, 10);
+
+      try {
+         pooledCompute.startPool();
+      } catch (RunNodesException e) {
+         destroyBadNodesAndPropagate(e);
+      }
+   }
+
+   private NodeMetadata createNodeWithJdk(String groupName) {
+      LOGGER.info("creating jclouds node");
+
+      NodeMetadata nodeMetadata = null;
+
+      try {
+         nodeMetadata = getOnlyElement(pooledCompute.createNodesInGroup(groupName, 1));
+      } catch (RunNodesException e) {
+         throw destroyBadNodesAndPropagate(e);
+      }
+
+      // Check if nodeMetadata is null and throw
+      return nodeMetadata;
+   }
+
+   private RuntimeException destroyBadNodesAndPropagate(RunNodesException e) {
+      for (Map.Entry nodeError : e.getNodeErrors().entrySet())
+         getCloud().getCompute().destroyNode(nodeError.getKey().getId());
+      throw propagate(e);
+   }
+
+   public void terminate(NodeMetadata nodeMetaData) {
+      if (stopOnTerminate) {
+         LOGGER.info("Suspending the Slave : " + nodeMetaData.getName());
+         final ComputeService compute = getCloud().getCompute();
+         compute.suspendNode(nodeMetaData.getId());
+      } else {
+         LOGGER.info("Terminating the Slave : " + nodeMetaData.getName());
+         final ComputeService compute = getCloud().getCompute();
+         compute.destroyNode(nodeMetaData.getId());
+      }
+   }
+
+}

From b87e1397eefcabda6b28f33f5eb159cd629aeb41 Mon Sep 17 00:00:00 2001
From: Adrian Cole 
Date: Thu, 19 Apr 2012 23:14:13 -0700
Subject: [PATCH 41/50] functions for regions

---
 .../location/functions/RegionToEndpoint.java  | 60 ++++++++++++++++
 .../functions/RegionToEndpointTest.java       | 68 +++++++++++++++++++
 2 files changed, 128 insertions(+)
 create mode 100644 core/src/main/java/org/jclouds/location/functions/RegionToEndpoint.java
 create mode 100644 core/src/test/java/org/jclouds/location/functions/RegionToEndpointTest.java

diff --git a/core/src/main/java/org/jclouds/location/functions/RegionToEndpoint.java b/core/src/main/java/org/jclouds/location/functions/RegionToEndpoint.java
new file mode 100644
index 0000000000..f873e2e9e9
--- /dev/null
+++ b/core/src/main/java/org/jclouds/location/functions/RegionToEndpoint.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.location.functions;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import java.net.URI;
+import java.util.Map;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.javax.annotation.Nullable;
+import org.jclouds.location.Region;
+
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class RegionToEndpoint implements Function {
+
+   private final Supplier>> regionToEndpointSupplier;
+
+   @Inject
+   public RegionToEndpoint(@Region Supplier>> regionToEndpointSupplier) {
+      this.regionToEndpointSupplier = checkNotNull(regionToEndpointSupplier, "regionToEndpointSupplier");
+   }
+
+   @Override
+   public URI apply(@Nullable Object from) {
+      checkArgument(from != null && from instanceof String, "you must specify a region, as a String argument");
+      Map> regionToEndpoint = regionToEndpointSupplier.get();
+      checkState(regionToEndpoint.size() > 0, "no region name to endpoint mappings configured!");
+      checkArgument(regionToEndpoint.containsKey(from),
+               "requested location %s, which is not in the configured locations: %s", from, regionToEndpoint);
+      return regionToEndpoint.get(from).get();
+   }
+}
\ No newline at end of file
diff --git a/core/src/test/java/org/jclouds/location/functions/RegionToEndpointTest.java b/core/src/test/java/org/jclouds/location/functions/RegionToEndpointTest.java
new file mode 100644
index 0000000000..7c0e015a7f
--- /dev/null
+++ b/core/src/test/java/org/jclouds/location/functions/RegionToEndpointTest.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.location.functions;
+
+import static org.testng.Assert.assertEquals;
+
+import java.io.File;
+import java.net.URI;
+import java.util.Map;
+
+import org.testng.annotations.Test;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Tests behavior of {@code RegionToEndpoint}
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "RegionToEndpointTest")
+public class RegionToEndpointTest {
+
+   @Test
+   public void testCorrect() {
+      RegionToEndpoint fn = new RegionToEndpoint(Suppliers.>> ofInstance(ImmutableMap.of("1",
+               Suppliers.ofInstance(URI.create("http://1")))));
+      assertEquals(fn.apply("1"), URI.create("http://1"));
+   }
+
+   @Test(expectedExceptions = IllegalArgumentException.class)
+   public void testMustBeString() {
+      RegionToEndpoint fn = new RegionToEndpoint(Suppliers.>> ofInstance(ImmutableMap.of("1",
+               Suppliers.ofInstance(URI.create("http://1")))));
+      fn.apply(new File("foo"));
+   }
+
+   @Test(expectedExceptions = IllegalStateException.class)
+   public void testMustHaveEndpoints() {
+      RegionToEndpoint fn = new RegionToEndpoint(Suppliers.>> ofInstance(ImmutableMap
+               .> of()));
+      fn.apply("1");
+   }
+
+   @Test(expectedExceptions = IllegalArgumentException.class)
+   public void testNullIsIllegal() {
+      RegionToEndpoint fn = new RegionToEndpoint(Suppliers.>> ofInstance(ImmutableMap.of("1",
+               Suppliers.ofInstance(URI.create("http://1")))));
+      fn.apply(null);
+   }
+}

From eda3f05c8e006c30f86a7aa4f9efc315433b4554 Mon Sep 17 00:00:00 2001
From: Adrian Cole 
Date: Thu, 19 Apr 2012 23:18:36 -0700
Subject: [PATCH 42/50] Issue 905:create openstack-swift api

---
 labs/openstack-swift/pom.xml                  | 151 ++++++++++++++++++
 .../openstack/swift/v1/SwiftApiMetadata.java  | 100 ++++++++++++
 .../openstack/swift/v1/SwiftAsyncClient.java  |  69 ++++++++
 .../openstack/swift/v1/SwiftClient.java       |  71 ++++++++
 .../swift/v1/config/SwiftProperties.java      |  28 ++++
 .../v1/config/SwiftRestClientModule.java      |  85 ++++++++++
 .../swift/v1/domain/AccountMetadata.java      | 101 ++++++++++++
 .../swift/v1/domain/ContainerMetadata.java    | 138 ++++++++++++++++
 .../swift/v1/features/AccountAsyncClient.java |  83 ++++++++++
 .../swift/v1/features/AccountClient.java      |  62 +++++++
 .../v1/features/ContainerAsyncClient.java     |  38 +++++
 .../swift/v1/features/ContainerClient.java    |  37 +++++
 .../swift/v1/features/ObjectAsyncClient.java  |  38 +++++
 .../swift/v1/features/ObjectClient.java       |  43 +++++
 ...rseAccountMetadataResponseFromHeaders.java |  42 +++++
 .../swift/v1/handlers/SwiftErrorHandler.java  |  82 ++++++++++
 .../v1/options/ListContainersOptions.java     |  79 +++++++++
 .../services/org.jclouds.apis.ApiMetadata     |   1 +
 .../v1/PasswordAuthenticationExpectTest.java  |  75 +++++++++
 .../swift/v1/SwiftApiMetadataTest.java        |  38 +++++
 .../swift/v1/SwiftErrorHandlerTest.java       | 120 ++++++++++++++
 .../v1/features/AccountClientExpectTest.java  | 104 ++++++++++++
 .../v1/features/AccountClientLiveTest.java    |  61 +++++++
 .../features/ContainerClientExpectTest.java   |  31 ++++
 .../v1/features/ContainerClientLiveTest.java  |  31 ++++
 .../v1/features/ObjectClientExpectTest.java   |  30 ++++
 .../v1/features/ObjectClientLiveTest.java     |  30 ++++
 .../BaseSwiftAsyncClientExpectTest.java       |  39 +++++
 .../internal/BaseSwiftClientExpectTest.java   |  30 ++++
 .../v1/internal/BaseSwiftClientLiveTest.java  |  74 +++++++++
 .../v1/internal/BaseSwiftExpectTest.java      |  52 ++++++
 .../v1/options/ListContainersOptionsTest.java |  96 +++++++++++
 .../v1/parse/ParseContainerListTest.java      |  59 +++++++
 .../src/test/resources/container_list.json    |   4 +
 .../src/test/resources/logback.xml            |  51 ++++++
 labs/pom.xml                                  |   1 +
 36 files changed, 2174 insertions(+)
 create mode 100644 labs/openstack-swift/pom.xml
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftAsyncClient.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftClient.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftProperties.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftRestClientModule.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/AccountMetadata.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/ContainerMetadata.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountAsyncClient.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountClient.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ContainerAsyncClient.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ContainerClient.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ObjectAsyncClient.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ObjectClient.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/functions/ParseAccountMetadataResponseFromHeaders.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/handlers/SwiftErrorHandler.java
 create mode 100644 labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/options/ListContainersOptions.java
 create mode 100644 labs/openstack-swift/src/main/resources/META-INF/services/org.jclouds.apis.ApiMetadata
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/PasswordAuthenticationExpectTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/SwiftApiMetadataTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/SwiftErrorHandlerTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/AccountClientExpectTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/AccountClientLiveTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ContainerClientExpectTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ContainerClientLiveTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectClientExpectTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectClientLiveTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftAsyncClientExpectTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftClientExpectTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftClientLiveTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftExpectTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/options/ListContainersOptionsTest.java
 create mode 100644 labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/parse/ParseContainerListTest.java
 create mode 100644 labs/openstack-swift/src/test/resources/container_list.json
 create mode 100644 labs/openstack-swift/src/test/resources/logback.xml

diff --git a/labs/openstack-swift/pom.xml b/labs/openstack-swift/pom.xml
new file mode 100644
index 0000000000..d1480dd579
--- /dev/null
+++ b/labs/openstack-swift/pom.xml
@@ -0,0 +1,151 @@
+
+
+
+  4.0.0
+  
+    org.jclouds
+    jclouds-project
+    1.5.0-SNAPSHOT
+    ../../project/pom.xml
+  
+  org.jclouds.labs
+  openstack-swift
+  jcloud openstack-swift api
+  jclouds components to access an implementation of OpenStack Swift
+  bundle
+
+  
+    
+    http://localhost:5000
+    
+    1.0
+    
+    FIXME_IDENTITY
+    FIXME_CREDENTIALS
+    passwordCredentials
+    FIXME_HTTPURL
+    FIXME_HTTPMD5
+  
+
+  
+    
+      org.jclouds.common
+      openstack-common
+      ${project.version}
+    
+    
+      org.jclouds
+      jclouds-blobstore
+      ${project.version}
+    
+    
+      org.jclouds
+      jclouds-core
+      ${project.version}
+      test-jar
+      test
+    
+    
+      org.jclouds.common
+      openstack-common
+      ${project.version}
+      test-jar
+      test
+    
+    
+      org.jclouds
+      jclouds-blobstore
+      ${project.version}
+      test-jar
+      test
+    
+    
+      org.jclouds.driver
+      jclouds-slf4j
+      ${project.version}
+      test
+    
+    
+      ch.qos.logback
+      logback-classic
+      1.0.0
+      test
+    
+  
+  
+  
+    
+      live
+      
+        
+          
+            org.apache.maven.plugins
+            maven-surefire-plugin
+            
+              
+                integration
+                integration-test
+                
+                  test
+                
+                
+                  
+                    ${test.openstack-swift.endpoint}
+                    ${test.openstack-swift.api-version}
+                    ${test.openstack-swift.build-version}
+                    ${test.openstack-swift.identity}
+                    ${test.openstack-swift.credential}
+                    ${test.jclouds.keystone.credential-type}
+                    ${jclouds.blobstore.httpstream.url}
+                    ${jclouds.blobstore.httpstream.md5}
+                  
+                
+              
+            
+          
+        
+      
+    
+  
+
+  
+    
+        
+        org.apache.felix
+        maven-bundle-plugin
+        
+          
+            ${project.artifactId}
+            org.jclouds.openstack.swift.v1*;version="${project.version}"
+                        
+                        org.jclouds.blobstore.internal;version="${project.version}",
+                        org.jclouds.rest.internal;version="${project.version}",
+                        org.jclouds*;version="${project.version}",
+                                *
+                        
+          
+        
+      
+    
+  
+
+
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java
new file mode 100644
index 0000000000..60a20eea8b
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1;
+
+import static org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties.SERVICE_TYPE;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.jclouds.apis.ApiMetadata;
+import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
+import org.jclouds.openstack.services.ServiceType;
+import org.jclouds.openstack.swift.v1.config.SwiftRestClientModule;
+import org.jclouds.rest.RestContext;
+import org.jclouds.rest.internal.BaseRestApiMetadata;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.reflect.TypeToken;
+import com.google.inject.Module;
+
+/**
+ * Implementation of {@link ApiMetadata} for Swift 1.0 API
+ * 
+ * @author Adrian Cole
+ */
+public class SwiftApiMetadata extends BaseRestApiMetadata {
+   
+   /** The serialVersionUID */
+   private static final long serialVersionUID = 6725672099385580694L;
+
+   public static final TypeToken> CONTEXT_TOKEN = new TypeToken>() {
+      private static final long serialVersionUID = -5070937833892503232L;
+   };
+
+   @Override
+   public Builder toBuilder() {
+      return new Builder().fromApiMetadata(this);
+   }
+
+   public SwiftApiMetadata() {
+      this(new Builder());
+   }
+
+   protected SwiftApiMetadata(Builder builder) {
+      super(builder);
+   }
+
+   public static Properties defaultProperties() {
+      Properties properties = BaseRestApiMetadata.defaultProperties();
+      properties.setProperty(SERVICE_TYPE, ServiceType.OBJECT_STORE);
+      // TODO: this doesn't actually do anything yet.
+      properties.setProperty(KeystoneProperties.VERSION, "2.0");
+      return properties;
+   }
+
+   public static class Builder extends BaseRestApiMetadata.Builder {
+
+      protected Builder() {
+         super(SwiftClient.class, SwiftAsyncClient.class);
+          id("openstack-swift")
+         .name("OpenStack Swift Diablo+ API")
+         .identityName("tenantId:user")
+         .credentialName("password")
+         .documentation(URI.create("http://docs.openstack.org/api/openstack-object-storage/1.0/content/ch_object-storage-dev-overview.html"))
+         .version("1.0")
+         .defaultEndpoint("http://localhost:5000")
+         .defaultProperties(SwiftApiMetadata.defaultProperties())
+         .defaultModules(ImmutableSet.>of(SwiftRestClientModule.class));
+      }
+      
+      @Override
+      public SwiftApiMetadata build() {
+         return new SwiftApiMetadata(this);
+      }
+
+      @Override
+      public Builder fromApiMetadata(ApiMetadata in) {
+         super.fromApiMetadata(in);
+         return this;
+      }
+
+   }
+
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftAsyncClient.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftAsyncClient.java
new file mode 100644
index 0000000000..5f1fc80d59
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftAsyncClient.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1;
+
+import java.util.Set;
+
+import org.jclouds.javax.annotation.Nullable;
+import org.jclouds.location.Region;
+import org.jclouds.location.functions.RegionToEndpoint;
+import org.jclouds.openstack.swift.v1.features.AccountAsyncClient;
+import org.jclouds.openstack.swift.v1.features.ContainerAsyncClient;
+import org.jclouds.openstack.swift.v1.features.ObjectAsyncClient;
+import org.jclouds.rest.annotations.Delegate;
+import org.jclouds.rest.annotations.EndpointParam;
+
+import com.google.inject.Provides;
+
+/**
+ * Provides asynchronous access to Swift via their REST API.
+ * 

+ * 
+ * @see SwiftClient
+ * @see api doc
+ * @author Adrian Cole
+ */
+public interface SwiftAsyncClient {
+   /**
+    * 
+    * @return the Region codes configured
+    */
+   @Provides
+   @Region
+   Set getConfiguredRegions();
+
+   /**
+    * Provides asynchronous access to Account features.
+    */
+   @Delegate
+   AccountAsyncClient getAccountClientForRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region);
+
+   /**
+    * Provides asynchronous access to Container features.
+    */
+   @Delegate
+   ContainerAsyncClient getContainerClientForRegion(
+            @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region);
+
+   /**
+    * Provides asynchronous access to Object features.
+    */
+   @Delegate
+   ObjectAsyncClient getObjectClientForRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region);
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftClient.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftClient.java
new file mode 100644
index 0000000000..4f1e30e70c
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftClient.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.jclouds.concurrent.Timeout;
+import org.jclouds.javax.annotation.Nullable;
+import org.jclouds.location.Region;
+import org.jclouds.location.functions.RegionToEndpoint;
+import org.jclouds.openstack.swift.v1.features.AccountClient;
+import org.jclouds.openstack.swift.v1.features.ContainerClient;
+import org.jclouds.openstack.swift.v1.features.ObjectClient;
+import org.jclouds.rest.annotations.Delegate;
+import org.jclouds.rest.annotations.EndpointParam;
+
+import com.google.inject.Provides;
+
+/**
+ * Provides synchronous access to Swift.
+ * 

+ * 
+ * @see SwiftAsyncClient
+ * @see api doc
+ * @author Adrian Cole
+ */
+@Timeout(duration = 60, timeUnit = TimeUnit.SECONDS)
+public interface SwiftClient {
+   /**
+    * 
+    * @return the Region codes configured
+    */
+   @Provides
+   @Region
+   Set getConfiguredRegions();
+
+   /**
+    * Provides synchronous access to Account features.
+    */
+   @Delegate
+   AccountClient getAccountClientForRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region);
+
+   /**
+    * Provides synchronous access to Container features.
+    */
+   @Delegate
+   ContainerClient getContainerClientForRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region);
+
+   /**
+    * Provides synchronous access to Object features.
+    */
+   @Delegate
+   ObjectClient getObjectClientForRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region);
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftProperties.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftProperties.java
new file mode 100644
index 0000000000..57216017cf
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftProperties.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.config;
+
+/**
+ * Configuration properties and constants used in openstack Swift connections.
+ *
+ * @author Adam Lowe
+ */
+public class SwiftProperties {
+
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftRestClientModule.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftRestClientModule.java
new file mode 100644
index 0000000000..54dfd01498
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftRestClientModule.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.config;
+
+import java.util.Map;
+
+import org.jclouds.http.HttpErrorHandler;
+import org.jclouds.http.annotation.ClientError;
+import org.jclouds.http.annotation.Redirection;
+import org.jclouds.http.annotation.ServerError;
+import org.jclouds.json.config.GsonModule.DateAdapter;
+import org.jclouds.json.config.GsonModule.Iso8601DateAdapter;
+import org.jclouds.openstack.keystone.v2_0.config.KeystoneAuthenticationModule;
+import org.jclouds.openstack.swift.v1.SwiftAsyncClient;
+import org.jclouds.openstack.swift.v1.SwiftClient;
+import org.jclouds.openstack.swift.v1.features.AccountAsyncClient;
+import org.jclouds.openstack.swift.v1.features.AccountClient;
+import org.jclouds.openstack.swift.v1.features.ContainerAsyncClient;
+import org.jclouds.openstack.swift.v1.features.ContainerClient;
+import org.jclouds.openstack.swift.v1.features.ObjectAsyncClient;
+import org.jclouds.openstack.swift.v1.features.ObjectClient;
+import org.jclouds.openstack.swift.v1.handlers.SwiftErrorHandler;
+import org.jclouds.rest.ConfiguresRestClient;
+import org.jclouds.rest.config.RestClientModule;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Configures the Swift connection.
+ * 
+ * @author Adrian Cole
+ */
+@ConfiguresRestClient
+public class SwiftRestClientModule extends RestClientModule {
+
+   public static final Map, Class> DELEGATE_MAP = ImmutableMap., Class> builder()
+         .put(AccountClient.class, AccountAsyncClient.class)
+         .put(ContainerClient.class, ContainerAsyncClient.class)
+         .put(ObjectClient.class, ObjectAsyncClient.class)
+         .build();
+
+   public SwiftRestClientModule() {
+      super(DELEGATE_MAP);
+   }
+   
+   @Override
+   protected void configure() {
+      bind(DateAdapter.class).to(Iso8601DateAdapter.class);
+      super.configure();
+   }
+
+   @Override
+   protected void installLocations() {
+      // TODO: select this from KeystoneProperties.VERSION; note you select from
+      // a guice provided
+      // property, so it will have to come from somewhere else, maybe we move
+      // this to the the
+      // ContextBuilder
+      install(KeystoneAuthenticationModule.forRegions());
+      super.installLocations();
+   }
+
+   @Override
+   protected void bindErrorHandlers() {
+      bind(HttpErrorHandler.class).annotatedWith(Redirection.class).to(SwiftErrorHandler.class);
+      bind(HttpErrorHandler.class).annotatedWith(ClientError.class).to(SwiftErrorHandler.class);
+      bind(HttpErrorHandler.class).annotatedWith(ServerError.class).to(SwiftErrorHandler.class);
+   }
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/AccountMetadata.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/AccountMetadata.java
new file mode 100644
index 0000000000..61a53d5c4a
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/AccountMetadata.java
@@ -0,0 +1,101 @@
+package org.jclouds.openstack.swift.v1.domain;
+
+import static com.google.common.base.Objects.equal;
+import static com.google.common.base.Objects.toStringHelper;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Objects.ToStringHelper;
+
+/**
+ * @author Adrian Cole
+ * @see api doc
+ */
+public class AccountMetadata {
+
+   public static Builder builder() {
+      return new Builder();
+   }
+
+   public Builder toBuilder() {
+      return builder().fromAccountMetadata(this);
+   }
+
+   public static class Builder {
+      protected int containerCount;
+      protected long bytesUsed;
+
+      /**
+       * @see AccountMetadata#getContainerCount()
+       */
+      public Builder containerCount(int containerCount) {
+         this.containerCount = containerCount;
+         return this;
+      }
+
+      /**
+       * @see AccountMetadata#getBytesUsed()
+       */
+      public Builder bytesUsed(long bytesUsed) {
+         this.bytesUsed = bytesUsed;
+         return this;
+      }
+
+      public AccountMetadata build() {
+         return new AccountMetadata(containerCount, bytesUsed);
+      }
+
+      public Builder fromAccountMetadata(AccountMetadata from) {
+         return containerCount(from.getContainerCount()).bytesUsed(from.getBytesUsed());
+      }
+   }
+
+   protected final int containerCount;
+   protected final long bytesUsed;
+
+   public AccountMetadata(int containerCount, long bytesUsed) {
+      this.containerCount = containerCount;
+      this.bytesUsed = bytesUsed;
+   }
+
+   /**
+    * 
+    * @return the number of containers in OpenStack Object Storage for the account
+    */
+   public int getContainerCount() {
+      return containerCount;
+   }
+
+   /**
+    * @return the total bytes stored in OpenStack Object Storage for the account
+    */
+   public long getBytesUsed() {
+      return bytesUsed;
+   }
+
+   @Override
+   public boolean equals(Object object) {
+      if (this == object) {
+         return true;
+      }
+      if (object instanceof AccountMetadata) {
+         final AccountMetadata other = AccountMetadata.class.cast(object);
+         return equal(getContainerCount(), other.getContainerCount()) && equal(getBytesUsed(), other.getBytesUsed());
+      } else {
+         return false;
+      }
+   }
+
+   @Override
+   public int hashCode() {
+      return Objects.hashCode(getContainerCount(), getBytesUsed());
+   }
+
+   @Override
+   public String toString() {
+      return string().toString();
+   }
+
+   protected ToStringHelper string() {
+      return toStringHelper("").add("containerCount", getContainerCount()).add("bytesUsed", getBytesUsed());
+   }
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/ContainerMetadata.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/ContainerMetadata.java
new file mode 100644
index 0000000000..2852022535
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/ContainerMetadata.java
@@ -0,0 +1,138 @@
+package org.jclouds.openstack.swift.v1.domain;
+
+import static com.google.common.base.Objects.equal;
+import static com.google.common.base.Objects.toStringHelper;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Objects.ToStringHelper;
+
+/**
+ * retrieve a list of existing storage containers ordered by name. The sort order for the name is
+ * based on a binary comparison, a single built-in collating sequence that compares string data
+ * using SQLite's memcmp() function, regardless of text encoding.
+ * 
+ * @author Adrian Cole
+ * @see api
+ *      doc
+ */
+public class ContainerMetadata implements Comparable {
+
+   public static Builder builder() {
+      return new Builder();
+   }
+
+   public Builder toBuilder() {
+      return builder().fromAccountMetadata(this);
+   }
+
+   public static class Builder {
+      protected String name;
+      protected int count;
+      protected int bytes;
+
+      /**
+       * @see ContainerMetadata#getName()
+       */
+      public Builder name(String name) {
+         this.name = checkNotNull(name, "name");
+         return this;
+      }
+
+      /**
+       * @see ContainerMetadata#getCount()
+       */
+      public Builder count(int count) {
+         this.count = count;
+         return this;
+      }
+
+      /**
+       * @see ContainerMetadata#getBytes()
+       */
+      public Builder bytes(int bytes) {
+         this.bytes = bytes;
+         return this;
+      }
+
+      public ContainerMetadata build() {
+         return new ContainerMetadata(name, count, bytes);
+      }
+
+      public Builder fromAccountMetadata(ContainerMetadata from) {
+         return name(from.getName()).count(from.getCount()).bytes(from.getBytes());
+      }
+   }
+
+   protected final String name;
+   protected final int count;
+   protected final int bytes;
+
+   public ContainerMetadata(String name, int count, int bytes) {
+      this.name = checkNotNull(name, "name");
+      this.count = count;
+      this.bytes = bytes;
+   }
+
+   /**
+    * 
+    * @return the name of the container
+    */
+   public String getName() {
+      return name;
+   }
+
+   /**
+    * 
+    * @return the number of objects in the container
+    */
+   public int getCount() {
+      return count;
+   }
+
+   /**
+    * @return the total bytes stored in this container
+    */
+   public int getBytes() {
+      return bytes;
+   }
+
+   @Override
+   public boolean equals(Object object) {
+      if (this == object) {
+         return true;
+      }
+      if (object instanceof ContainerMetadata) {
+         final ContainerMetadata other = ContainerMetadata.class.cast(object);
+         return equal(getName(), other.getName()) && equal(getCount(), other.getCount())
+                  && equal(getBytes(), other.getBytes());
+      } else {
+         return false;
+      }
+   }
+
+   @Override
+   public int hashCode() {
+      return Objects.hashCode(getName(), getCount(), getBytes());
+   }
+
+   @Override
+   public String toString() {
+      return string().toString();
+   }
+
+   protected ToStringHelper string() {
+      return toStringHelper("").add("name", getName()).add("count", getCount()).add("bytes", getBytes());
+   }
+
+   @Override
+   public int compareTo(ContainerMetadata that) {
+      if (that == null)
+         return 1;
+      if (this == that)
+         return 0;
+      return this.getName().compareTo(that.getName());
+   }
+
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountAsyncClient.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountAsyncClient.java
new file mode 100644
index 0000000000..c159b221cd
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountAsyncClient.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import java.util.Set;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.HEAD;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.openstack.filters.AuthenticateRequest;
+import org.jclouds.openstack.swift.v1.domain.AccountMetadata;
+import org.jclouds.openstack.swift.v1.domain.ContainerMetadata;
+import org.jclouds.openstack.swift.v1.functions.ParseAccountMetadataResponseFromHeaders;
+import org.jclouds.openstack.swift.v1.options.ListContainersOptions;
+import org.jclouds.rest.annotations.ExceptionParser;
+import org.jclouds.rest.annotations.QueryParams;
+import org.jclouds.rest.annotations.RequestFilters;
+import org.jclouds.rest.annotations.ResponseParser;
+import org.jclouds.rest.annotations.SkipEncoding;
+import org.jclouds.rest.functions.ReturnEmptySetOnNotFoundOr404;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+/**
+ * Storage Account Services
+ * 
+ * @see AccountClient
+ * @author Adrian Cole
+ * @see api doc
+ */
+@SkipEncoding( { '/', '=' })
+@RequestFilters(AuthenticateRequest.class)
+public interface AccountAsyncClient {
+
+   /**
+    * @see AccountClient#getAccountMetadata
+    */
+   @HEAD
+   @ResponseParser(ParseAccountMetadataResponseFromHeaders.class)
+   @Path("/")
+   ListenableFuture getAccountMetadata();
+
+   /**
+    * @see AccountClient#listContainers()
+    */
+   @GET
+   @Consumes(MediaType.APPLICATION_JSON)
+   @QueryParams(keys = "format", values = "json")
+   @ExceptionParser(ReturnEmptySetOnNotFoundOr404.class)
+   @Path("/")
+   ListenableFuture> listContainers();
+
+   /**
+    * @see AccountClient#listContainers(ListContainersOptions)
+    */
+   @GET
+   @Consumes(MediaType.APPLICATION_JSON)
+   @QueryParams(keys = "format", values = "json")
+   @ExceptionParser(ReturnEmptySetOnNotFoundOr404.class)
+   @Path("/")
+   ListenableFuture> listContainers(ListContainersOptions options);
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountClient.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountClient.java
new file mode 100644
index 0000000000..a00414c61a
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountClient.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.jclouds.concurrent.Timeout;
+import org.jclouds.openstack.swift.v1.domain.AccountMetadata;
+import org.jclouds.openstack.swift.v1.domain.ContainerMetadata;
+import org.jclouds.openstack.swift.v1.options.ListContainersOptions;
+
+/**
+ * Storage Account Services
+ * 
+ * @see AccountAsyncClient
+ * @author Adrian Cole
+ * @see api doc
+ */
+@Timeout(duration = 180, timeUnit = TimeUnit.SECONDS)
+public interface AccountClient {
+   /**
+    * Retrieve Account Metadata
+    * 
+    * @return account metadata including container count and bytes used
+    */
+   AccountMetadata getAccountMetadata();
+
+   /**
+    * @see #listContainers(ListContainersOptions)
+    */
+   Set listContainers();
+
+   /**
+    * retrieve a list of existing storage containers ordered by name. The sort order for the name is
+    * based on a binary comparison, a single built-in collating sequence that compares string data
+    * using SQLite's memcmp() function, regardless of text encoding.
+    * 
+    * @param options
+    * @return a list of existing storage containers ordered by name.
+    */
+   Set listContainers(ListContainersOptions options);
+
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ContainerAsyncClient.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ContainerAsyncClient.java
new file mode 100644
index 0000000000..68fda93deb
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ContainerAsyncClient.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import org.jclouds.openstack.filters.AuthenticateRequest;
+import org.jclouds.rest.annotations.RequestFilters;
+import org.jclouds.rest.annotations.SkipEncoding;
+
+/**
+ * Storage Container Services
+ * 
+ * @see ContainerClient
+ * @author Adrian Cole
+ * @see api doc
+ */
+@SkipEncoding({ '/', '=' })
+@RequestFilters(AuthenticateRequest.class)
+public interface ContainerAsyncClient {
+
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ContainerClient.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ContainerClient.java
new file mode 100644
index 0000000000..fc91e8e253
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ContainerClient.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import java.util.concurrent.TimeUnit;
+
+import org.jclouds.concurrent.Timeout;
+
+/**
+ * Storage Container Services
+ * 
+ * @see ContainerAsyncClient
+ * @author Adrian Cole
+ * @see api doc
+ */
+@Timeout(duration = 180, timeUnit = TimeUnit.SECONDS)
+public interface ContainerClient {
+
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ObjectAsyncClient.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ObjectAsyncClient.java
new file mode 100644
index 0000000000..597a5e2e17
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ObjectAsyncClient.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import org.jclouds.openstack.filters.AuthenticateRequest;
+import org.jclouds.rest.annotations.RequestFilters;
+import org.jclouds.rest.annotations.SkipEncoding;
+
+/**
+ * Storage Object Services
+ * 
+ * @see ObjectClient
+ * @author Adrian Cole
+ * @see api doc
+ */
+@SkipEncoding( { '/', '=' })
+@RequestFilters(AuthenticateRequest.class)
+public interface ObjectAsyncClient {
+
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ObjectClient.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ObjectClient.java
new file mode 100644
index 0000000000..18a8145401
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/ObjectClient.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import java.util.concurrent.TimeUnit;
+
+import org.jclouds.concurrent.Timeout;
+
+/**
+ * Storage Object Services An object represents the data and any metadata for the files stored in
+ * the system. Through the ReST interface, metadata for an object can be included by adding custom
+ * HTTP headers to the request and the data payload as the request body. Objects cannot exceed 5GB
+ * and must have names that do not exceed 1024 bytes after URL encoding. However, objects larger
+ * than 5GB can be segmented and then concatenated together so that you can upload 5 GB segments and
+ * download a single concatenated object. You can work with the segments and manifests directly with
+ * HTTP requests.
+ * 
+ * @see ObjectAsyncClient
+ * @author Adrian Cole
+ * @see api doc
+ */
+@Timeout(duration = 180, timeUnit = TimeUnit.SECONDS)
+public interface ObjectClient {
+
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/functions/ParseAccountMetadataResponseFromHeaders.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/functions/ParseAccountMetadataResponseFromHeaders.java
new file mode 100644
index 0000000000..e01ccadfed
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/functions/ParseAccountMetadataResponseFromHeaders.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.functions;
+
+import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.swift.v1.domain.AccountMetadata;
+
+import com.google.common.base.Function;
+
+/**
+ * This parses {@link AccountMetadata} from HTTP headers.
+ * 
+ * @author James Murty
+ */
+public class ParseAccountMetadataResponseFromHeaders implements Function {
+
+   /**
+    * parses the http response headers to create a new {@link AccountMetadata} object.
+    */
+   public AccountMetadata apply(HttpResponse from) {
+      return AccountMetadata.builder()
+               .bytesUsed(Long.parseLong(from.getFirstHeaderOrNull("X-Account-Bytes-Used")))
+               .containerCount(Integer.parseInt(from.getFirstHeaderOrNull("X-Account-Container-Count")))
+               .build();
+   }
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/handlers/SwiftErrorHandler.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/handlers/SwiftErrorHandler.java
new file mode 100644
index 0000000000..0437dd48e3
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/handlers/SwiftErrorHandler.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.handlers;
+
+import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.inject.Singleton;
+
+import org.jclouds.blobstore.ContainerNotFoundException;
+import org.jclouds.blobstore.KeyNotFoundException;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpErrorHandler;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.HttpResponseException;
+import org.jclouds.rest.AuthorizationException;
+
+/**
+ * This will parse and set an appropriate exception on the command object.
+ * 
+ * @author Adrian Cole
+ * 
+ */
+// TODO: is there error spec someplace? let's type errors, etc.
+@Singleton
+public class SwiftErrorHandler implements HttpErrorHandler {
+   public static final String PREFIX = "^/v[0-9][^/]*/[a-zA-Z]+_[^/]+/";
+   public static final Pattern CONTAINER_PATH = Pattern.compile(PREFIX + "([^/]+)$");
+   public static final Pattern CONTAINER_KEY_PATH = Pattern.compile(PREFIX + "([^/]+)/(.*)");
+
+   public void handleError(HttpCommand command, HttpResponse response) {
+      // it is important to always read fully and close streams
+      byte[] data = closeClientButKeepContentStream(response);
+      String message = data != null ? new String(data) : null;
+
+      Exception exception = message != null ? new HttpResponseException(command, response, message)
+               : new HttpResponseException(command, response);
+      message = message != null ? message : String.format("%s -> %s", command.getCurrentRequest().getRequestLine(),
+               response.getStatusLine());
+      switch (response.getStatusCode()) {
+         case 401:
+            exception = new AuthorizationException(exception.getMessage(), exception);
+            break;
+         case 404:
+            if (!command.getCurrentRequest().getMethod().equals("DELETE")) {
+               String path = command.getCurrentRequest().getEndpoint().getPath();
+               Matcher matcher = CONTAINER_PATH.matcher(path);
+               Exception oldException = exception;
+               if (matcher.find()) {
+                  exception = new ContainerNotFoundException(matcher.group(1), message);
+                  exception.initCause(oldException);
+               } else {
+                  matcher = CONTAINER_KEY_PATH.matcher(path);
+                  if (matcher.find()) {
+                     exception = new KeyNotFoundException(matcher.group(1), matcher.group(2), message);
+                     exception.initCause(oldException);
+                  }
+               }
+            }
+            break;
+      }
+      command.setException(exception);
+   }
+}
diff --git a/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/options/ListContainersOptions.java b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/options/ListContainersOptions.java
new file mode 100644
index 0000000000..e2afee6159
--- /dev/null
+++ b/labs/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/options/ListContainersOptions.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.options;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import org.jclouds.http.options.BaseHttpRequestOptions;
+
+/**
+ * Contains options supported in the REST API for the GET container operation. 

+ */
+public class ListContainersOptions extends BaseHttpRequestOptions {
+   public static final ListContainersOptions NONE = new ListContainersOptions();
+
+   /**
+    * Given a string value x, return object names greater in value than the specified marker.
+    */
+   public ListContainersOptions marker(String marker) {
+      queryParameters.put("marker", checkNotNull(marker, "marker"));
+      return this;
+   }
+
+   public String getMarker() {
+      return getFirstQueryOrNull("marker");
+   }
+
+   /**
+    * For an integer value n, limits the number of results to n values.
+    */
+   public ListContainersOptions limit(int limit) {
+      checkState(limit >= 0, "limit must be >= 0");
+      checkState(limit <= 10000, "limit must be <= 10000");
+      queryParameters.put("limit", Integer.toString(limit));
+      return this;
+   }
+
+   public int getLimit() {
+      String val = getFirstQueryOrNull("limit");
+      return val != null ? new Integer(val) : 10000;
+   }
+
+
+   public static class Builder {
+
+      /**
+       * @see ListContainersOptions#marker(String)
+       */
+      public static ListContainersOptions marker(String marker) {
+         ListContainersOptions options = new ListContainersOptions();
+         return options.marker(marker);
+      }
+
+      /**
+       * @see ListContainersOptions#limit(int)
+       */
+      public static ListContainersOptions limit(int limit) {
+         ListContainersOptions options = new ListContainersOptions();
+         return options.limit(limit);
+      }
+
+   }
+}
diff --git a/labs/openstack-swift/src/main/resources/META-INF/services/org.jclouds.apis.ApiMetadata b/labs/openstack-swift/src/main/resources/META-INF/services/org.jclouds.apis.ApiMetadata
new file mode 100644
index 0000000000..84f1200b16
--- /dev/null
+++ b/labs/openstack-swift/src/main/resources/META-INF/services/org.jclouds.apis.ApiMetadata
@@ -0,0 +1 @@
+org.jclouds.openstack.swift.v1.SwiftApiMetadata
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/PasswordAuthenticationExpectTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/PasswordAuthenticationExpectTest.java
new file mode 100644
index 0000000000..8ec1e81980
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/PasswordAuthenticationExpectTest.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1;
+
+import static org.testng.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftClientExpectTest;
+import org.jclouds.openstack.swift.v1.parse.ParseContainerListTest;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * 
+ * @see KeystoneProperties#CREDENTIAL_TYPE
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "PasswordAuthenticationExpectTest")
+public class PasswordAuthenticationExpectTest extends BaseSwiftClientExpectTest {
+
+   /**
+    * this reflects the properties that a user would pass to createContext
+    */
+   @Override
+   protected Properties setupProperties() {
+      Properties contextProperties = super.setupProperties();
+      contextProperties.setProperty("jclouds.keystone.credential-type", "passwordCredentials");
+      return contextProperties;
+   }
+
+   public void testListContainersWhenResponseIs2xx() throws Exception {
+      HttpRequest listContainers = HttpRequest
+            .builder()
+            .method("GET")
+            .endpoint(URI.create("https://objects.jclouds.org/v1.0/40806637803162/?format=json"))
+            .headers(
+                  ImmutableMultimap. builder().put("Accept", "application/json")
+                        .put("X-Auth-Token", authToken).build()).build();
+
+      HttpResponse listContainersResponse = HttpResponse.builder().statusCode(200)
+            .payload(payloadFromResource("/container_list.json")).build();
+
+      SwiftClient clientWhenContainersExist = requestsSendResponses(keystoneAuthWithUsernameAndPassword,
+            responseWithKeystoneAccess, listContainers, listContainersResponse);
+
+      assertEquals(clientWhenContainersExist.getConfiguredRegions(), ImmutableSet.of("region-a.geo-1"));
+
+      assertEquals(clientWhenContainersExist.getAccountClientForRegion("region-a.geo-1").listContainers().toString(),
+            new ParseContainerListTest().expected().toString());
+   }
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/SwiftApiMetadataTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/SwiftApiMetadataTest.java
new file mode 100644
index 0000000000..2cd22db3c8
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/SwiftApiMetadataTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1;
+
+import org.jclouds.Wrapper;
+import org.jclouds.apis.internal.BaseApiMetadataTest;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.reflect.TypeToken;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "SwiftApiMetadataTest")
+// public class SwiftApiMetadataTest extends BaseBlobStoreApiMetadataTest {
+public class SwiftApiMetadataTest extends BaseApiMetadataTest {
+   public SwiftApiMetadataTest() {
+      super(new SwiftApiMetadata(), ImmutableSet.> of());
+   }
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/SwiftErrorHandlerTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/SwiftErrorHandlerTest.java
new file mode 100644
index 0000000000..bc2a762f57
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/SwiftErrorHandlerTest.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reportMatcher;
+import static org.easymock.EasyMock.verify;
+
+import java.net.URI;
+
+import org.easymock.IArgumentMatcher;
+import org.jclouds.blobstore.ContainerNotFoundException;
+import org.jclouds.blobstore.KeyNotFoundException;
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.io.Payloads;
+import org.jclouds.openstack.swift.v1.handlers.SwiftErrorHandler;
+import org.jclouds.util.Strings2;
+import org.testng.annotations.Test;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "SwiftErrorHandlerTest")
+public class SwiftErrorHandlerTest {
+
+   @Test
+   public void test404SetsKeyNotFoundExceptionMosso() {
+      assertCodeMakes("HEAD", URI
+               .create("http://host/v1/MossoCloudFS_7064cdb1d49d4dcba3c899ac33e8409d/adriancole-blobstore1/key"), 404,
+               "Not Found", "", KeyNotFoundException.class);
+   }
+
+   @Test
+   public void test404SetsKeyNotFoundExceptionSwift() {
+      assertCodeMakes("HEAD", URI
+               .create("http://67.202.39.175:8080/v1/AUTH_7064cdb1d49d4dcba3c899ac33e8409d/adriancole-blobstore1/key"),
+               404, "Not Found", "", KeyNotFoundException.class);
+   }
+
+   @Test
+   public void test404SetsContainerNotFoundExceptionMosso() {
+      assertCodeMakes("HEAD", URI
+               .create("http://host/v1/MossoCloudFS_7064cdb1d49d4dcba3c899ac33e8409d/adriancole-blobstore1"), 404,
+               "Not Found", "", ContainerNotFoundException.class);
+   }
+
+   @Test
+   public void test404SetsContainerNotFoundExceptionSwift() {
+      assertCodeMakes("HEAD", URI
+               .create("http://67.202.39.175:8080/v1/AUTH_7064cdb1d49d4dcba3c899ac33e8409d/adriancole-blobstore1"),
+               404, "Not Found", "", ContainerNotFoundException.class);
+   }
+
+   private void assertCodeMakes(String method, URI uri, int statusCode, String message, String content,
+            Class expected) {
+      assertCodeMakes(method, uri, statusCode, message, "text/plain", content, expected);
+   }
+
+   private void assertCodeMakes(String method, URI uri, int statusCode, String message, String contentType,
+            String content, Class expected) {
+
+      SwiftErrorHandler function = new SwiftErrorHandler();
+
+      HttpCommand command = createMock(HttpCommand.class);
+      HttpRequest request = new HttpRequest(method, uri);
+      HttpResponse response = new HttpResponse(statusCode, message, Payloads.newInputStreamPayload(Strings2
+               .toInputStream(content)));
+      response.getPayload().getContentMetadata().setContentType(contentType);
+
+      expect(command.getCurrentRequest()).andReturn(request).atLeastOnce();
+      command.setException(classEq(expected));
+
+      replay(command);
+
+      function.handleError(command, response);
+
+      verify(command);
+   }
+
+   public static Exception classEq(final Class in) {
+      reportMatcher(new IArgumentMatcher() {
+
+         @Override
+         public void appendTo(StringBuffer buffer) {
+            buffer.append("classEq(");
+            buffer.append(in);
+            buffer.append(")");
+         }
+
+         @Override
+         public boolean matches(Object arg) {
+            return arg.getClass() == in;
+         }
+
+      });
+      return null;
+   }
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/AccountClientExpectTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/AccountClientExpectTest.java
new file mode 100644
index 0000000000..5038f6cae4
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/AccountClientExpectTest.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unles required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either expres or implied.  See the License for the
+ * specific language governing permisions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import static org.testng.Assert.assertEquals;
+
+import java.net.URI;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.swift.v1.SwiftClient;
+import org.jclouds.openstack.swift.v1.domain.AccountMetadata;
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftClientExpectTest;
+import org.jclouds.openstack.swift.v1.parse.ParseContainerListTest;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "AccountClientExpectTest")
+public class AccountClientExpectTest extends BaseSwiftClientExpectTest {
+   
+   public void testGetAccountMetadataWhenResponseIs2xx() throws Exception {
+
+      HttpRequest getAccountMetadata = HttpRequest
+            .builder()
+            .method("HEAD")
+            .endpoint(URI.create("https://objects.jclouds.org/v1.0/40806637803162/"))
+            .headers(
+                  ImmutableMultimap. builder()
+                        .put("X-Auth-Token", authToken).build()).build();
+
+      HttpResponse listContainersResponse = HttpResponse.builder().statusCode(204)
+            .headers(ImmutableMultimap.of(
+                     "X-Account-Container-Count", "3",
+                     "X-Account-Bytes-Used", "323479")).build();
+
+      SwiftClient clientWhenContainersExist = requestsSendResponses(keystoneAuthWithUsernameAndPassword,
+            responseWithKeystoneAccess, getAccountMetadata, listContainersResponse);
+
+      assertEquals(
+            clientWhenContainersExist.getAccountClientForRegion("region-a.geo-1").getAccountMetadata(),
+            AccountMetadata.builder().containerCount(3).bytesUsed(323479).build());
+   }
+   
+   public void testListContainersWhenResponseIs2xx() throws Exception {
+
+      HttpRequest listContainers = HttpRequest
+            .builder()
+            .method("GET")
+            .endpoint(URI.create("https://objects.jclouds.org/v1.0/40806637803162/?format=json"))
+            .headers(
+                  ImmutableMultimap. builder().put("Accept", "application/json")
+                        .put("X-Auth-Token", authToken).build()).build();
+
+      HttpResponse listContainersResponse = HttpResponse.builder().statusCode(200)
+            .payload(payloadFromResource("/container_list.json")).build();
+
+      SwiftClient clientWhenContainersExist = requestsSendResponses(keystoneAuthWithUsernameAndPassword,
+            responseWithKeystoneAccess, listContainers, listContainersResponse);
+
+      assertEquals(
+            clientWhenContainersExist.getAccountClientForRegion("region-a.geo-1").listContainers()
+                  .toString(), new ParseContainerListTest().expected().toString());
+   }
+
+   public void testListContainersWhenResponseIs404() throws Exception {
+      HttpRequest listContainers = HttpRequest
+            .builder()
+            .method("GET")
+            .endpoint(URI.create("https://objects.jclouds.org/v1.0/40806637803162/?format=json"))
+            .headers(
+                  ImmutableMultimap. builder().put("Accept", "application/json")
+                        .put("X-Auth-Token", authToken).build()).build();
+
+      HttpResponse listContainersResponse = HttpResponse.builder().statusCode(404).build();
+
+      SwiftClient clientWhenNoContainersExist = requestsSendResponses(keystoneAuthWithUsernameAndPassword,
+            responseWithKeystoneAccess, listContainers, listContainersResponse);
+
+      assertEquals(clientWhenNoContainersExist.getAccountClientForRegion("region-a.geo-1").listContainers(), ImmutableSet.of());
+
+   }
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/AccountClientLiveTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/AccountClientLiveTest.java
new file mode 100644
index 0000000000..1eb410a1db
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/AccountClientLiveTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+import java.util.Set;
+
+import org.jclouds.openstack.swift.v1.domain.AccountMetadata;
+import org.jclouds.openstack.swift.v1.domain.ContainerMetadata;
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftClientLiveTest;
+import org.testng.annotations.Test;
+
+/**
+ * @author Adrian Cole
+ */
+@Test(groups = "live", testName = "ContainerClientLiveTest")
+public class AccountClientLiveTest extends BaseSwiftClientLiveTest {
+
+   @Test
+   public void testGetAccountMetadata() throws Exception {
+      for (String regionId : swiftContext.getApi().getConfiguredRegions()) {
+         AccountClient client = swiftContext.getApi().getAccountClientForRegion(regionId);
+         AccountMetadata account = client.getAccountMetadata();
+         assertNotNull(account);
+         assertTrue(account.getContainerCount() >= 0);
+         assertTrue(account.getBytesUsed() >= 0);
+      }
+   }
+
+   @Test
+   public void testListContainers() throws Exception {
+      for (String regionId : swiftContext.getApi().getConfiguredRegions()) {
+         AccountClient client = swiftContext.getApi().getAccountClientForRegion(regionId);
+         Set response = client.listContainers();
+         assertNotNull(response);
+         for (ContainerMetadata container : response) {
+            assertNotNull(container.getName());
+            assertTrue(container.getCount() >= 0);
+            assertTrue(container.getBytes() >= 0);
+         }
+      }
+   }
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ContainerClientExpectTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ContainerClientExpectTest.java
new file mode 100644
index 0000000000..ecec4c562c
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ContainerClientExpectTest.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftClientExpectTest;
+import org.testng.annotations.Test;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "ContainerAsyncClientTest")
+public class ContainerClientExpectTest extends BaseSwiftClientExpectTest {
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ContainerClientLiveTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ContainerClientLiveTest.java
new file mode 100644
index 0000000000..dcbe482135
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ContainerClientLiveTest.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftClientLiveTest;
+import org.testng.annotations.Test;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "live", testName = "ContainerClientLiveTest")
+public class ContainerClientLiveTest extends BaseSwiftClientLiveTest {
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectClientExpectTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectClientExpectTest.java
new file mode 100644
index 0000000000..f38bf988e1
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectClientExpectTest.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftClientExpectTest;
+import org.testng.annotations.Test;
+
+/**
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "ObjectAsyncClientTest")
+public class ObjectClientExpectTest extends BaseSwiftClientExpectTest {
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectClientLiveTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectClientLiveTest.java
new file mode 100644
index 0000000000..f8d816c48e
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/ObjectClientLiveTest.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftClientLiveTest;
+import org.testng.annotations.Test;
+
+/**
+ * @author Adrian Cole
+ */
+@Test(groups = "live", testName = "ObjectClientLiveTest")
+public class ObjectClientLiveTest extends BaseSwiftClientLiveTest {
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftAsyncClientExpectTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftAsyncClientExpectTest.java
new file mode 100644
index 0000000000..1668d4276b
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftAsyncClientExpectTest.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.internal;
+
+import java.util.Properties;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.swift.v1.SwiftAsyncClient;
+
+import com.google.common.base.Function;
+import com.google.inject.Module;
+
+/**
+ * Base class for writing KeyStone Rest Client Expect tests
+ * 
+ * @author Adrian Cole
+ */
+public class BaseSwiftAsyncClientExpectTest extends BaseSwiftExpectTest {
+   public SwiftAsyncClient createClient(Function fn, Module module, Properties props) {
+      return createInjector(fn, module, props).getInstance(SwiftAsyncClient.class);
+   }
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftClientExpectTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftClientExpectTest.java
new file mode 100644
index 0000000000..b3f4330057
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftClientExpectTest.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.internal;
+
+import org.jclouds.openstack.swift.v1.SwiftClient;
+
+/**
+ * Base class for writing KeyStone Rest Client Expect tests
+ * 
+ * @author Adrian Cole
+ */
+public class BaseSwiftClientExpectTest extends BaseSwiftExpectTest {
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftClientLiveTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftClientLiveTest.java
new file mode 100644
index 0000000000..3eb8dfb0b4
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftClientLiveTest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.internal;
+
+import java.util.Properties;
+
+import org.jclouds.apis.BaseContextLiveTest;
+import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
+import org.jclouds.openstack.swift.v1.SwiftApiMetadata;
+import org.jclouds.openstack.swift.v1.SwiftAsyncClient;
+import org.jclouds.openstack.swift.v1.SwiftClient;
+import org.jclouds.rest.RestContext;
+import org.testng.annotations.AfterGroups;
+import org.testng.annotations.BeforeGroups;
+import org.testng.annotations.Test;
+
+import com.google.common.reflect.TypeToken;
+
+/**
+ * Tests behavior of {@code SwiftClient}
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "live")
+public class BaseSwiftClientLiveTest extends BaseContextLiveTest> {
+
+   public BaseSwiftClientLiveTest() {
+      provider = "openstack-swift";
+   }
+
+   protected RestContext swiftContext;
+
+   @BeforeGroups(groups = { "integration", "live" })
+   @Override
+   public void setupContext() {
+      super.setupContext();
+      swiftContext = context;
+   }
+
+   @Override
+   protected Properties setupProperties() {
+      Properties props = super.setupProperties();
+      setIfTestSystemPropertyPresent(props, KeystoneProperties.CREDENTIAL_TYPE);
+      return props;
+   }
+   
+   @AfterGroups(groups = "live")
+   protected void tearDown() {
+      if (swiftContext != null)
+         swiftContext.close();
+   }
+
+   @Override
+   protected TypeToken> contextType() {
+      return SwiftApiMetadata.CONTEXT_TOKEN;
+   }
+
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftExpectTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftExpectTest.java
new file mode 100644
index 0000000000..67d55b8832
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/internal/BaseSwiftExpectTest.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.internal;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.openstack.keystone.v2_0.internal.KeystoneFixture;
+import org.jclouds.rest.internal.BaseRestClientExpectTest;
+
+/**
+ * Base class for writing Swift Expect tests
+ * 
+ * @author Adrian Cole
+ */
+public class BaseSwiftExpectTest extends BaseRestClientExpectTest {
+   protected HttpRequest keystoneAuthWithUsernameAndPassword;
+   protected HttpRequest keystoneAuthWithAccessKeyAndSecretKey;
+   protected String authToken;
+   protected HttpResponse responseWithKeystoneAccess;
+   protected HttpRequest extensionsOfSwiftRequest;
+   protected HttpResponse extensionsOfSwiftResponse;
+   protected HttpResponse unmatchedExtensionsOfSwiftResponse;
+
+   public BaseSwiftExpectTest() {
+      provider = "openstack-swift";
+      keystoneAuthWithUsernameAndPassword = KeystoneFixture.INSTANCE.initialAuthWithUsernameAndPassword(identity,
+            credential);
+      keystoneAuthWithAccessKeyAndSecretKey = KeystoneFixture.INSTANCE.initialAuthWithAccessKeyAndSecretKey(identity,
+            credential);
+      
+      authToken = KeystoneFixture.INSTANCE.getAuthToken();
+      responseWithKeystoneAccess = KeystoneFixture.INSTANCE.responseWithAccess();
+      // now, createContext arg will need tenant prefix
+      identity = KeystoneFixture.INSTANCE.getTenantName() + ":" + identity;
+   }
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/options/ListContainersOptionsTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/options/ListContainersOptionsTest.java
new file mode 100644
index 0000000000..64b207034b
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/options/ListContainersOptionsTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.options;
+
+import static org.jclouds.openstack.swift.v1.options.ListContainersOptions.Builder.limit;
+import static org.jclouds.openstack.swift.v1.options.ListContainersOptions.Builder.marker;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Collections;
+
+import org.jclouds.http.options.HttpRequestOptions;
+import org.testng.annotations.Test;
+
+/**
+ * Tests possible uses of ListContainerOptions and ListContainerOptions.Builder.*
+ * 
+ * @author Adrian Cole
+ */
+@Test(testName = "ListContainersOptionsTest")
+public class ListContainersOptionsTest {
+
+   @Test
+   public void testAssignability() {
+      assert HttpRequestOptions.class.isAssignableFrom(ListContainersOptions.class);
+      assert !String.class.isAssignableFrom(ListContainersOptions.class);
+   }
+   @Test
+   public void testNoOptionsQueryString() {
+      HttpRequestOptions options = new ListContainersOptions();
+      assertEquals(options.buildQueryParameters().size(), 0);
+   }
+
+   @Test
+   public void testMarker() {
+      ListContainersOptions options = new ListContainersOptions();
+      options.marker("test");
+      assertEquals(options.buildQueryParameters().get("marker"), Collections.singletonList("test"));
+   }
+
+   @Test
+   public void testNullMarker() {
+      ListContainersOptions options = new ListContainersOptions();
+      assertEquals(options.buildQueryParameters().get("marker"), Collections.EMPTY_LIST);
+   }
+
+   @Test
+   public void testMarkerStatic() {
+      ListContainersOptions options = marker("test");
+      assertEquals(options.buildQueryParameters().get("marker"), Collections.singletonList("test"));
+   }
+
+   @Test(expectedExceptions = NullPointerException.class)
+   public void testMarkerNPE() {
+      marker(null);
+   }
+
+   @Test
+   public void testLimit() {
+      ListContainersOptions options = new ListContainersOptions();
+      options.limit(1000);
+      assertEquals(options.buildQueryParameters().get("limit"), Collections.singletonList("1000"));
+   }
+
+   @Test
+   public void testNullLimit() {
+      ListContainersOptions options = new ListContainersOptions();
+      assertEquals(options.buildQueryParameters().get("limit"), Collections.EMPTY_LIST);
+   }
+
+   @Test
+   public void testLimitStatic() {
+      ListContainersOptions options = limit(1000);
+      assertEquals(options.buildQueryParameters().get("limit"), Collections.singletonList("1000"));
+   }
+
+   @Test(expectedExceptions = IllegalStateException.class)
+   public void testLimitNegative() {
+      limit(-1);
+   }
+}
diff --git a/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/parse/ParseContainerListTest.java b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/parse/ParseContainerListTest.java
new file mode 100644
index 0000000000..3c583121c4
--- /dev/null
+++ b/labs/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/parse/ParseContainerListTest.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.openstack.swift.v1.parse;
+
+import java.util.Set;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.core.MediaType;
+
+import org.jclouds.json.BaseSetParserTest;
+import org.jclouds.openstack.swift.v1.domain.ContainerMetadata;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "ParseContainerListTest")
+public class ParseContainerListTest extends BaseSetParserTest {
+
+   @Override
+   public String resource() {
+      return "/container_list.json";
+   }
+
+   @Override
+   @Consumes(MediaType.APPLICATION_JSON)
+   public Set expected() {
+      return ImmutableSet
+            .of(ContainerMetadata.builder()
+                  .name("test_container_1")
+                  .count(2)
+                  .bytes(78)
+                  .build(),
+                ContainerMetadata.builder()
+                  .name("test_container_2")
+                  .count(1)
+                  .bytes(17)
+                  .build());
+   }
+}
diff --git a/labs/openstack-swift/src/test/resources/container_list.json b/labs/openstack-swift/src/test/resources/container_list.json
new file mode 100644
index 0000000000..ef7e791530
--- /dev/null
+++ b/labs/openstack-swift/src/test/resources/container_list.json
@@ -0,0 +1,4 @@
+[
+  {"name":"test_container_1", "count":2, "bytes":78},
+  {"name":"test_container_2", "count":1, "bytes":17}
+]
\ No newline at end of file
diff --git a/labs/openstack-swift/src/test/resources/logback.xml b/labs/openstack-swift/src/test/resources/logback.xml
new file mode 100644
index 0000000000..bd89efb8c0
--- /dev/null
+++ b/labs/openstack-swift/src/test/resources/logback.xml
@@ -0,0 +1,51 @@
+
+
+    
+        target/test-data/jclouds.log
+
+        
+            %d %-5p [%c] [%thread] %m%n
+        
+    
+
+    
+        target/test-data/jclouds-wire.log
+
+        
+            %d %-5p [%c] [%thread] %m%n
+        
+    
+
+    
+        target/test-data/jclouds-blobstore.log
+
+        
+            %d %-5p [%c] [%thread] %m%n
+        
+    
+    
+    
+        
+    
+
+    
+        
+        
+    
+
+    
+        
+        
+    
+
+    
+        
+        
+    
+
+    
+        
+        
+    
+
+
diff --git a/labs/pom.xml b/labs/pom.xml
index 60b5dee8da..b774ad00cb 100644
--- a/labs/pom.xml
+++ b/labs/pom.xml
@@ -41,5 +41,6 @@
        savvis-symphonyvpdc
        dmtf
        carrenza-vcloud-director
+       openstack-swift
     
 

From f1010e13a4cb3a9d02c2d6e25421e4a933df9e17 Mon Sep 17 00:00:00 2001
From: Adrian Cole 
Date: Sat, 21 Apr 2012 09:57:55 -0700
Subject: [PATCH 43/50] fixed aws-ec2 image parsing

---
 .../EC2ComputeServiceContextModule.java       |  2 +-
 .../BaseComputeServiceContextModule.java      |  4 ++--
 .../config/ValueOfConfigurationKeyOrNull.java |  3 +--
 .../ValueOfConfigurationKeyOrNullTest.java    | 18 ++++++++++++++----
 .../jclouds/aws/ec2/AWSEC2ApiMetadata.java    |  2 ++
 .../aws/ec2/AWSEC2ProviderMetadata.java       |  2 --
 .../AWSEC2ComputeServiceContextModule.java    |  7 ++++---
 ...WSEC2ComputeServiceDependenciesModule.java | 19 ++++++++++---------
 .../suppliers/AWSEC2ImageSupplier.java        |  2 +-
 .../aws/ec2/AWSEC2ContextBuilderTest.java     |  5 ++---
 .../AWSEC2TemplateBuilderLiveTest.java        |  2 +-
 11 files changed, 38 insertions(+), 28 deletions(-)

diff --git a/apis/ec2/src/main/java/org/jclouds/ec2/compute/config/EC2ComputeServiceContextModule.java b/apis/ec2/src/main/java/org/jclouds/ec2/compute/config/EC2ComputeServiceContextModule.java
index 6a34b28d5e..dd714a4d0f 100644
--- a/apis/ec2/src/main/java/org/jclouds/ec2/compute/config/EC2ComputeServiceContextModule.java
+++ b/apis/ec2/src/main/java/org/jclouds/ec2/compute/config/EC2ComputeServiceContextModule.java
@@ -72,7 +72,7 @@ public class EC2ComputeServiceContextModule extends BaseComputeServiceContextMod
    }
    
    @Override
-   protected boolean shouldParseImagesOnDemand(Injector injector) {
+   protected boolean shouldEagerlyParseImages(Injector injector) {
       // If no owners to query, then will never lookup all images
       String[] amiOwners = injector.getInstance(Key.get(String[].class, Names.named(PROPERTY_EC2_AMI_OWNERS)));
       return (amiOwners.length > 0);
diff --git a/compute/src/main/java/org/jclouds/compute/config/BaseComputeServiceContextModule.java b/compute/src/main/java/org/jclouds/compute/config/BaseComputeServiceContextModule.java
index a522f13116..da2c3c3cab 100644
--- a/compute/src/main/java/org/jclouds/compute/config/BaseComputeServiceContextModule.java
+++ b/compute/src/main/java/org/jclouds/compute/config/BaseComputeServiceContextModule.java
@@ -227,14 +227,14 @@ public abstract class BaseComputeServiceContextModule extends AbstractModule {
    @Memoized
    protected Supplier> supplyImageCache(AtomicReference authException, @Named(PROPERTY_SESSION_INTERVAL) long seconds,
          final Supplier> imageSupplier, Injector injector) {
-      if (shouldParseImagesOnDemand(injector)) {
+      if (shouldEagerlyParseImages(injector)) {
          return supplyImageCache(authException, seconds, imageSupplier);
       } else {
          return supplyNonParsingImageCache(authException, seconds, imageSupplier, injector);
       }
    }
 
-   protected boolean shouldParseImagesOnDemand(Injector injector) {
+   protected boolean shouldEagerlyParseImages(Injector injector) {
       return true;
    }
 
diff --git a/core/src/main/java/org/jclouds/config/ValueOfConfigurationKeyOrNull.java b/core/src/main/java/org/jclouds/config/ValueOfConfigurationKeyOrNull.java
index ab50c64deb..1f6335f272 100644
--- a/core/src/main/java/org/jclouds/config/ValueOfConfigurationKeyOrNull.java
+++ b/core/src/main/java/org/jclouds/config/ValueOfConfigurationKeyOrNull.java
@@ -19,7 +19,6 @@
 package org.jclouds.config;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Strings.emptyToNull;
 import static com.google.inject.name.Names.named;
 
 import javax.inject.Inject;
@@ -47,7 +46,7 @@ public class ValueOfConfigurationKeyOrNull implements Function {
    public String apply(String configurationKey) {
       checkNotNull(configurationKey, "configurationKey");
       try {
-         return emptyToNull(injector.getInstance(Key.get(String.class, named(configurationKey))));
+         return injector.getInstance(Key.get(String.class, named(configurationKey)));
       } catch (ConfigurationException e) {
          return null;
       }
diff --git a/core/src/test/java/org/jclouds/config/ValueOfConfigurationKeyOrNullTest.java b/core/src/test/java/org/jclouds/config/ValueOfConfigurationKeyOrNullTest.java
index 1af174d9f0..53c8000b82 100644
--- a/core/src/test/java/org/jclouds/config/ValueOfConfigurationKeyOrNullTest.java
+++ b/core/src/test/java/org/jclouds/config/ValueOfConfigurationKeyOrNullTest.java
@@ -20,8 +20,6 @@ package org.jclouds.config;
 
 import static org.testng.Assert.assertEquals;
 
-import java.util.concurrent.ExecutionException;
-
 import org.testng.annotations.Test;
 
 import com.google.inject.AbstractModule;
@@ -37,12 +35,12 @@ import com.google.inject.name.Names;
 public class ValueOfConfigurationKeyOrNullTest {
 
    @Test
-   public void testNotThere() throws InterruptedException, ExecutionException {
+   public void testNotThere() {
       assertEquals(new ValueOfConfigurationKeyOrNull(Guice.createInjector()).apply("foo"), null);
    }
 
    @Test
-   public void testThere() throws InterruptedException, ExecutionException {
+   public void testThere() {
       assertEquals(new ValueOfConfigurationKeyOrNull(Guice.createInjector(new AbstractModule() {
 
          @Override
@@ -53,5 +51,17 @@ public class ValueOfConfigurationKeyOrNullTest {
       })).apply("foo"), "bar");
 
    }
+   
+   @Test
+   public void testEmptyIsThere() {
+      assertEquals(new ValueOfConfigurationKeyOrNull(Guice.createInjector(new AbstractModule() {
 
+         @Override
+         protected void configure() {
+            bindConstant().annotatedWith(Names.named("foo")).to("");
+         }
+
+      })).apply("foo"), "");
+
+   }
 }
diff --git a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ApiMetadata.java b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ApiMetadata.java
index 8431b17c16..86053927bf 100644
--- a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ApiMetadata.java
+++ b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ApiMetadata.java
@@ -19,6 +19,7 @@
 package org.jclouds.aws.ec2;
 
 import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_GENERATE_INSTANCE_NAMES;
+import static org.jclouds.ec2.reference.EC2Constants.PROPERTY_EC2_AMI_OWNERS;
 
 import java.util.Properties;
 
@@ -67,6 +68,7 @@ public class AWSEC2ApiMetadata extends EC2ApiMetadata {
    
    public static Properties defaultProperties() {
       Properties properties = EC2ApiMetadata.defaultProperties();
+      properties.remove(PROPERTY_EC2_AMI_OWNERS);
       // auth fail sometimes happens in EC2, as the rc.local script that injects the
       // authorized key executes after ssh has started.  
       properties.setProperty("jclouds.ssh.max-retries", "7");
diff --git a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ProviderMetadata.java b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ProviderMetadata.java
index 07504d4c1e..78c25d8b8f 100644
--- a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ProviderMetadata.java
+++ b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/AWSEC2ProviderMetadata.java
@@ -22,7 +22,6 @@ import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUE
 import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
 import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_REGIONS;
 import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_SUSPENDED;
-import static org.jclouds.ec2.reference.EC2Constants.PROPERTY_EC2_AMI_OWNERS;
 
 import java.net.URI;
 import java.util.Properties;
@@ -63,7 +62,6 @@ public class AWSEC2ProviderMetadata extends BaseProviderMetadata {
       // from stopping->stopped state on an ec2 micro
       properties.setProperty(TIMEOUT_NODE_SUSPENDED, 120 * 1000 + "");
       properties.putAll(Region.regionProperties());
-      properties.remove(PROPERTY_EC2_AMI_OWNERS);
       // amazon, alestic, canonical, and rightscale
       properties.setProperty(PROPERTY_EC2_AMI_QUERY,
                "owner-id=137112412989,063491364108,099720109477,411009282317;state=available;image-type=machine");
diff --git a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceContextModule.java b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceContextModule.java
index 654078797a..3368d5c120 100644
--- a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceContextModule.java
+++ b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceContextModule.java
@@ -102,10 +102,11 @@ public class AWSEC2ComputeServiceContextModule extends BaseComputeServiceContext
    }
 
    @Override
-   protected boolean shouldParseImagesOnDemand(Injector injector) {
+   protected boolean shouldEagerlyParseImages(Injector injector) {
+      Map queries = injector.getInstance(Key.get(new TypeLiteral>() {
+      }, ImageQuery.class));
       // If no queries defined, then will never lookup all images
-      return injector.getInstance(Key.get(new TypeLiteral>() {
-      }, ImageQuery.class)).size() > 0;
+      return queries.size() > 0;
    }
 
    // duplicates EC2ComputeServiceContextModule; but that's easiest thing to do with guice; could extract to common util
diff --git a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceDependenciesModule.java b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceDependenciesModule.java
index e62a5c49cd..e9c159ffe0 100644
--- a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceDependenciesModule.java
+++ b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/config/AWSEC2ComputeServiceDependenciesModule.java
@@ -18,7 +18,8 @@
  */
 package org.jclouds.aws.ec2.compute.config;
 
-import static org.jclouds.aws.ec2.reference.AWSEC2Constants.*;
+import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUERY;
+import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
 import static org.jclouds.ec2.reference.EC2Constants.PROPERTY_EC2_AMI_OWNERS;
 
 import java.util.Map;
@@ -63,8 +64,8 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Sets;
 import com.google.common.collect.ImmutableMap.Builder;
+import com.google.common.collect.Sets;
 import com.google.inject.Provides;
 import com.google.inject.TypeLiteral;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -100,18 +101,18 @@ public class AWSEC2ComputeServiceDependenciesModule extends EC2ComputeServiceDep
    @ImageQuery
    protected Map imageQuery(ValueOfConfigurationKeyOrNull config) {
       String amiQuery = Strings.emptyToNull(config.apply(PROPERTY_EC2_AMI_QUERY));
-      if (config.apply(PROPERTY_EC2_AMI_OWNERS) != null) {
+      String owners = config.apply(PROPERTY_EC2_AMI_OWNERS);
+      if ("".equals(owners)) {
+         amiQuery = null;
+      } else if (owners != null) {
          StringBuilder query = new StringBuilder();
-         String owners = config.apply(PROPERTY_EC2_AMI_OWNERS).toString();
          if ("*".equals(owners))
             query.append("state=available;image-type=machine");
-         else if (!"".equals(owners))
+         else
             query.append("owner-id=").append(owners).append(";state=available;image-type=machine");
-         else if ("".equals(owners))
-            query = new StringBuilder();
          Logger.getAnonymousLogger().warning(
-                  String.format("Property %s is deprecated, please use new syntax: %s=%s", PROPERTY_EC2_AMI_OWNERS,
-                           PROPERTY_EC2_AMI_QUERY, query.toString()));
+               String.format("Property %s is deprecated, please use new syntax: %s=%s", PROPERTY_EC2_AMI_OWNERS,
+                     PROPERTY_EC2_AMI_QUERY, query.toString()));
          amiQuery = query.toString();
       }
       Builder builder = ImmutableMap. builder();
diff --git a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/suppliers/AWSEC2ImageSupplier.java b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/suppliers/AWSEC2ImageSupplier.java
index e4cf5b0618..88abe55846 100644
--- a/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/suppliers/AWSEC2ImageSupplier.java
+++ b/providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/suppliers/AWSEC2ImageSupplier.java
@@ -97,7 +97,7 @@ public class AWSEC2ImageSupplier implements Supplier> {
    @Override
    public Set get() {
       String amiQuery = queries.get(PROPERTY_EC2_AMI_QUERY);
-      String ccAmiQuery= queries.get(PROPERTY_EC2_CC_AMI_QUERY);
+      String ccAmiQuery = queries.get(PROPERTY_EC2_CC_AMI_QUERY);
 
       Future> normalImages = images(regions.get(), amiQuery, PROPERTY_EC2_AMI_QUERY);
       ImmutableSet clusterImages;
diff --git a/providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/AWSEC2ContextBuilderTest.java b/providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/AWSEC2ContextBuilderTest.java
index bd2e719c94..b693d571cc 100644
--- a/providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/AWSEC2ContextBuilderTest.java
+++ b/providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/AWSEC2ContextBuilderTest.java
@@ -60,12 +60,11 @@ public class AWSEC2ContextBuilderTest {
       assertEquals(queries.get(PROPERTY_EC2_AMI_QUERY), "state=available;image-type=machine");
    }
 
-   public void testStaysPutWhenBlank() {
+   public void testBlankAmiOwnersRemovesAmiQuery() {
       Properties input = new Properties();
       input.setProperty(PROPERTY_EC2_AMI_OWNERS, "");
       Map queries = queriesForProperties(input);
       assertEquals(queries.get(PROPERTY_EC2_AMI_OWNERS), null);
-      assertEquals(queries.get(PROPERTY_EC2_AMI_QUERY), new AWSEC2ProviderMetadata().getDefaultProperties()
-               .getProperty(PROPERTY_EC2_AMI_QUERY));
+      assertEquals(queries.get(PROPERTY_EC2_AMI_QUERY), null);
    }
 }
diff --git a/providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/compute/AWSEC2TemplateBuilderLiveTest.java b/providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/compute/AWSEC2TemplateBuilderLiveTest.java
index 55e762f445..02391a390a 100644
--- a/providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/compute/AWSEC2TemplateBuilderLiveTest.java
+++ b/providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/compute/AWSEC2TemplateBuilderLiveTest.java
@@ -59,7 +59,7 @@ import com.google.inject.Module;
  * 
  * @author Adrian Cole
  */
-@Test(groups = "live")
+@Test(groups = "live", testName = "AWSEC2TemplateBuilderLiveTest")
 public class AWSEC2TemplateBuilderLiveTest extends EC2TemplateBuilderLiveTest {
 
    public AWSEC2TemplateBuilderLiveTest() {

From 9194a896a6a9773506723ec6b1248a1361bdbf28 Mon Sep 17 00:00:00 2001
From: Richard Downer 
Date: Mon, 23 Apr 2012 17:16:19 +0300
Subject: [PATCH 44/50] Modify a warning message to state the problem in full
 right at the beginning of the message (instead of having the crucial part of
 the message drifting in the middle of an ocean of toString()ed objects)

---
 .../compute/functions/TemplateToOperatingSystem.java          | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apis/cloudstack/src/main/java/org/jclouds/cloudstack/compute/functions/TemplateToOperatingSystem.java b/apis/cloudstack/src/main/java/org/jclouds/cloudstack/compute/functions/TemplateToOperatingSystem.java
index 6c63d78e54..fb4df381fa 100644
--- a/apis/cloudstack/src/main/java/org/jclouds/cloudstack/compute/functions/TemplateToOperatingSystem.java
+++ b/apis/cloudstack/src/main/java/org/jclouds/cloudstack/compute/functions/TemplateToOperatingSystem.java
@@ -72,14 +72,14 @@ public class TemplateToOperatingSystem implements Function
Date: Mon, 23 Apr 2012 12:00:25 -0400
Subject: [PATCH 45/50] Using the Javadoc plugin's 'aggregate-jar' goal instead
 of 'aggregate' for the jclouds-multi Javadoc build, so the Javadocs are
 actually attached and deployed

---
 pom.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 8bee94934b..2092ec5300 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,16 +61,16 @@
                         javadoc
                         package
                         
-                            aggregate
+                            aggregate-jar
                         
                     
                                 
                 
-                    ${sourceEncoding}                        
+                    -J-Xmx512m
+                    ${project.build.sourceEncoding}
                     true
                     
-                        http://java.sun.com/javase/6/docs/api/
-                        http://java.sun.com/javaee/6/docs/api/
+                        http://download.oracle.com/javase/6/docs/api/