Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issu 301: refactored utilities that depend on guava. introduced Crypto, CryptoStreams, and more Payloads to help deal with encrypted payloads and headers

This commit is contained in:
Adrian Cole 2010-08-01 02:07:53 -07:00
parent a9a0c53fb2
commit 6f180ddb4e
166 changed files with 3101 additions and 2789 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
archetypes/json-client-archetype/src/main/resources/archetype-resources/src/main/java/config/__providerName__RestClientModule.java
View File

@ -56,7 +56,7 @@ import org.jclouds.http.annotation.ServerError;
import org.jclouds.http.filters.BasicAuthentication;
import org.jclouds.rest.ConfiguresRestClient;
import org.jclouds.rest.config.RestClientModule;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import ${package}.${providerName};
import ${package}.${providerName}Client;
@ -85,9 +85,9 @@ public class ${providerName}RestClientModule extends
public BasicAuthentication provideBasicAuthentication(
@Named(${providerName}Constants.PROPERTY_${ucaseProviderName}_USER) String user,
@Named(${providerName}Constants.PROPERTY_${ucaseProviderName}_PASSWORD) String password,
EncryptionService encryptionService)
Crypto crypto)
throws UnsupportedEncodingException {
return new BasicAuthentication(user, password, encryptionService);
return new BasicAuthentication(user, password, crypto);
}
@Provides

4
atmos/src/main/java/org/jclouds/atmosonline/saas/binders/BindMetadataToHeaders.java
View File

@ -22,7 +22,7 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.atmosonline.saas.domain.AtmosObject;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.Binder;
@ -32,7 +32,7 @@ public class BindMetadataToHeaders implements Binder {
@Inject
protected BindMetadataToHeaders(BindUserMetadataToHeaders metaBinder,
EncryptionService encryptionService) {
Crypto crypto) {
this.metaBinder = metaBinder;
}

30
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java
View File

@ -19,8 +19,6 @@
package org.jclouds.atmosonline.saas.blobstore;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.concurrent.ConcurrentUtils.compose;
import static org.jclouds.concurrent.ConcurrentUtils.makeListenable;
import java.net.URI;
import java.util.Set;
@ -54,8 +52,9 @@ import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.concurrent.Futures;
import org.jclouds.crypto.Crypto;
import org.jclouds.domain.Location;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.options.GetOptions;
import com.google.common.base.Function;
@ -73,7 +72,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
private final BlobToObject blob2Object;
private final BlobStoreListOptionsToListOptions container2ContainerListOptions;
private final DirectoryEntryListToResourceMetadataList container2ResourceList;
private final EncryptionService encryptionService;
private final Crypto crypto;
private final BlobToHttpGetOptions blob2ObjectGetOptions;
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
@ -83,7 +82,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
Set<? extends Location> locations, AtmosStorageAsyncClient async, AtmosStorageClient sync,
ObjectToBlob object2Blob, ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object,
BlobStoreListOptionsToListOptions container2ContainerListOptions,
DirectoryEntryListToResourceMetadataList container2ResourceList, EncryptionService encryptionService,
DirectoryEntryListToResourceMetadataList container2ResourceList, Crypto crypto,
BlobToHttpGetOptions blob2ObjectGetOptions, Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
super(context, blobUtils, service, defaultLocation, locations);
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
@ -95,7 +94,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
this.object2Blob = checkNotNull(object2Blob, "object2Blob");
this.blob2Object = checkNotNull(blob2Object, "blob2Object");
this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
this.encryptionService = checkNotNull(encryptionService, "encryptionService");
this.crypto = checkNotNull(crypto, "crypto");
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider");
}
@ -104,7 +103,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<BlobMetadata> blobMetadata(String container, String key) {
return compose(async.headFile(container + "/" + key), new Function<AtmosObject, BlobMetadata>() {
return Futures.compose(async.headFile(container + "/" + key), new Function<AtmosObject, BlobMetadata>() {
@Override
public BlobMetadata apply(AtmosObject from) {
return object2BlobMd.apply(from);
@ -119,7 +118,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<Boolean> createContainerInLocation(Location location, String container) {
return compose(async.createDirectory(container), new Function<URI, Boolean>() {
return Futures.compose(async.createDirectory(container), new Function<URI, Boolean>() {
public Boolean apply(URI from) {
return true;
@ -133,7 +132,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<Void> createDirectory(String container, String directory) {
return compose(async.createDirectory(container + "/" + directory), new Function<URI, Void>() {
return Futures.compose(async.createDirectory(container + "/" + directory), new Function<URI, Void>() {
public Void apply(URI from) {
return null;// no etag
@ -195,7 +194,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
public ListenableFuture<Blob> getBlob(String container, String key, org.jclouds.blobstore.options.GetOptions options) {
GetOptions httpOptions = blob2ObjectGetOptions.apply(options);
ListenableFuture<AtmosObject> returnVal = async.readFile(container + "/" + key, httpOptions);
return compose(returnVal, object2Blob, service);
return Futures.compose(returnVal, object2Blob, service);
}
/**
@ -203,7 +202,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<PageSet<? extends StorageMetadata>> list() {
return compose(async.listDirectories(), container2ResourceList, service);
return Futures.compose(async.listDirectories(), container2ResourceList, service);
}
/**
@ -215,8 +214,9 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
container = AtmosStorageUtils.adjustContainerIfDirOptionPresent(container, options);
ListOptions nativeOptions = container2ContainerListOptions.apply(options);
ListenableFuture<BoundedSet<? extends DirectoryEntry>> returnVal = async.listDirectory(container, nativeOptions);
ListenableFuture<PageSet<? extends StorageMetadata>> list = compose(returnVal, container2ResourceList, service);
return (ListenableFuture<PageSet<? extends StorageMetadata>>) (options.isDetailed() ? compose(list,
ListenableFuture<PageSet<? extends StorageMetadata>> list = Futures.compose(returnVal, container2ResourceList,
service);
return (ListenableFuture<PageSet<? extends StorageMetadata>>) (options.isDetailed() ? Futures.compose(list,
fetchBlobMetadataProvider.get().setContainerName(container), service) : list);
}
@ -227,11 +227,11 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<String> putBlob(final String container, final Blob blob) {
return makeListenable(service.submit(new Callable<String>() {
return Futures.makeListenable(service.submit(new Callable<String>() {
@Override
public String call() throws Exception {
return AtmosStorageUtils.putBlob(sync, encryptionService, blob2Object, container, blob);
return AtmosStorageUtils.putBlob(sync, crypto, blob2Object, container, blob);
}

10
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosBlobStore.java
View File

@ -44,8 +44,8 @@ import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseBlobStore;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.crypto.Crypto;
import org.jclouds.domain.Location;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.options.GetOptions;
/**
@ -59,7 +59,7 @@ public class AtmosBlobStore extends BaseBlobStore {
private final BlobToObject blob2Object;
private final BlobStoreListOptionsToListOptions container2ContainerListOptions;
private final DirectoryEntryListToResourceMetadataList container2ResourceList;
private final EncryptionService encryptionService;
private final Crypto crypto;
private final BlobToHttpGetOptions blob2ObjectGetOptions;
private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
@ -69,7 +69,7 @@ public class AtmosBlobStore extends BaseBlobStore {
ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object,
BlobStoreListOptionsToListOptions container2ContainerListOptions,
DirectoryEntryListToResourceMetadataList container2ResourceList,
EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions,
Crypto crypto, BlobToHttpGetOptions blob2ObjectGetOptions,
Provider<FetchBlobMetadata> fetchBlobMetadataProvider) {
super(context, blobUtils, defaultLocation, locations);
this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions");
@ -80,7 +80,7 @@ public class AtmosBlobStore extends BaseBlobStore {
this.object2Blob = checkNotNull(object2Blob, "object2Blob");
this.blob2Object = checkNotNull(blob2Object, "blob2Object");
this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd");
this.encryptionService = checkNotNull(encryptionService, "encryptionService");
this.crypto = checkNotNull(crypto, "crypto");
this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider,
"fetchBlobMetadataProvider");
}
@ -204,7 +204,7 @@ public class AtmosBlobStore extends BaseBlobStore {
*/
@Override
public String putBlob(final String container, final Blob blob) {
return AtmosStorageUtils.putBlob(sync, encryptionService, blob2Object, container, blob);
return AtmosStorageUtils.putBlob(sync, crypto, blob2Object, container, blob);
}
/**

6
atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java
View File

@ -19,7 +19,7 @@
package org.jclouds.atmosonline.saas.blobstore.strategy;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import java.util.Arrays;
import java.util.Map;
@ -43,7 +43,7 @@ import org.jclouds.blobstore.internal.BlobRuntimeException;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.ListBlobsInContainer;
import static org.jclouds.concurrent.ConcurrentUtils.*;
import org.jclouds.concurrent.Futures;
import org.jclouds.logging.Logger;
import com.google.common.base.Throwables;
@ -85,7 +85,7 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy {
final BlockingQueue<Boolean> queue = new SynchronousQueue<Boolean>();
Map<String, Future<?>> responses = Maps.newHashMap();
for (BlobMetadata md : getAllBlobMetadata.execute(containerName, options)) {
final ListenableFuture<AtmosObject> future = makeListenable(client.headFile(containerName
final ListenableFuture<AtmosObject> future = Futures.makeListenable(client.headFile(containerName
+ "/" + md.getName()), userExecutor);
future.addListener(new Runnable() {
public void run() {

2
atmos/src/main/java/org/jclouds/atmosonline/saas/domain/AtmosObject.java
View File

@ -18,7 +18,7 @@
*/
package org.jclouds.atmosonline.saas.domain;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.io.PayloadEnclosing;
import com.google.common.collect.Multimap;
import com.google.inject.internal.Nullable;

2
atmos/src/main/java/org/jclouds/atmosonline/saas/domain/internal/AtmosObjectImpl.java
View File

@ -26,9 +26,9 @@ import org.jclouds.atmosonline.saas.domain.AtmosObject;
import org.jclouds.atmosonline.saas.domain.MutableContentMetadata;
import org.jclouds.atmosonline.saas.domain.SystemMetadata;
import org.jclouds.atmosonline.saas.domain.UserMetadata;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.http.internal.PayloadEnclosingImpl;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadEnclosing;
import org.jclouds.io.payloads.DelegatingPayload;
import com.google.common.collect.LinkedHashMultimap;

38
atmos/src/main/java/org/jclouds/atmosonline/saas/filters/SignRequest.java
View File

@ -36,13 +36,15 @@ import javax.inject.Singleton;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.atmosonline.saas.reference.AtmosStorageHeaders;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.TimeStamp;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.io.InputSuppliers;
import org.jclouds.logging.Logger;
import org.jclouds.util.Utils;
@ -62,7 +64,7 @@ public class SignRequest implements HttpRequestFilter {
private final String uid;
private final byte[] key;
private final Provider<String> timeStampProvider;
private final EncryptionService encryptionService;
private final Crypto crypto;
private final HttpUtils utils;
@Resource
@ -74,20 +76,19 @@ public class SignRequest implements HttpRequestFilter {
@Inject
public SignRequest(SignatureWire signatureWire, @Named(PROPERTY_IDENTITY) String uid,
@Named(PROPERTY_CREDENTIAL) String encodedKey,
@TimeStamp Provider<String> timeStampProvider, EncryptionService encryptionService,
HttpUtils utils) {
@Named(PROPERTY_CREDENTIAL) String encodedKey, @TimeStamp Provider<String> timeStampProvider,
Crypto crypto, HttpUtils utils) {
this.signatureWire = signatureWire;
this.uid = uid;
this.key = encryptionService.fromBase64(encodedKey);
this.key = CryptoStreams.base64(encodedKey);
this.timeStampProvider = timeStampProvider;
this.encryptionService = encryptionService;
this.crypto = crypto;
this.utils = utils;
}
public void filter(HttpRequest request) throws HttpException {
String toSign = replaceUIDHeader(request).removeOldSignature(request).replaceDateHeader(
request).createStringToSign(request);
String toSign = replaceUIDHeader(request).removeOldSignature(request).replaceDateHeader(request)
.createStringToSign(request);
calculateAndReplaceAuthHeader(request, toSign);
utils.logRequest(signatureLog, request, "<<");
}
@ -111,19 +112,17 @@ public class SignRequest implements HttpRequestFilter {
return buffer.toString();
}
private void calculateAndReplaceAuthHeader(HttpRequest request, String toSign)
throws HttpException {
private void calculateAndReplaceAuthHeader(HttpRequest request, String toSign) throws HttpException {
String signature = signString(toSign);
if (signatureWire.enabled())
signatureWire.input(Utils.toInputStream(signature));
request.getHeaders().replaceValues(AtmosStorageHeaders.SIGNATURE,
Collections.singletonList(signature));
request.getHeaders().replaceValues(AtmosStorageHeaders.SIGNATURE, Collections.singletonList(signature));
}
public String signString(String toSign) {
String signature;
try {
signature = encryptionService.base64(encryptionService.hmacSha1(toSign, key));
signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign), crypto.hmacSHA1(key)));
} catch (Exception e) {
throw new HttpException("error signing request", e);
}
@ -140,8 +139,7 @@ public class SignRequest implements HttpRequestFilter {
}
SignRequest replaceDateHeader(HttpRequest request) {
request.getHeaders().replaceValues(HttpHeaders.DATE,
Collections.singletonList(timeStampProvider.get()));
request.getHeaders().replaceValues(HttpHeaders.DATE, Collections.singletonList(timeStampProvider.get()));
return this;
}
@ -172,9 +170,8 @@ public class SignRequest implements HttpRequestFilter {
}
private void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload()
.getContentType())).append("\n");
buffer.append(utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentType()))
.append("\n");
}
@VisibleForTesting
@ -182,8 +179,7 @@ public class SignRequest implements HttpRequestFilter {
// Only the value is used, not the header
// name. If a request does not include the header, this is an empty string.
for (String header : new String[] { "Range" })
toSign.append(utils.valueOrEmpty(request.getHeaders().get(header)).toLowerCase()).append(
"\n");
toSign.append(utils.valueOrEmpty(request.getHeaders().get(header)).toLowerCase()).append("\n");
// Standard HTTP header, in UTC format. Only the date value is used, not the header name.
toSign.append(request.getHeaders().get(HttpHeaders.DATE).iterator().next()).append("\n");
}

33
atmos/src/main/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeaders.java
View File

@ -1,6 +1,6 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
f * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
@ -28,8 +28,8 @@ import javax.inject.Singleton;
import org.jclouds.atmosonline.saas.domain.FileType;
import org.jclouds.atmosonline.saas.domain.SystemMetadata;
import org.jclouds.atmosonline.saas.reference.AtmosStorageHeaders;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.DateService;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpResponse;
import com.google.common.base.Function;
@ -41,18 +41,14 @@ import com.google.common.collect.Maps;
@Singleton
public class ParseSystemMetadataFromHeaders implements Function<HttpResponse, SystemMetadata> {
private final DateService dateService;
private final EncryptionService encryptionService;
@Inject
public ParseSystemMetadataFromHeaders(DateService dateService,
EncryptionService encryptionService) {
public ParseSystemMetadataFromHeaders(DateService dateService) {
this.dateService = dateService;
this.encryptionService = encryptionService;
}
public SystemMetadata apply(HttpResponse from) {
String meta = checkNotNull(from.getFirstHeaderOrNull(AtmosStorageHeaders.META),
AtmosStorageHeaders.META);
String meta = checkNotNull(from.getFirstHeaderOrNull(AtmosStorageHeaders.META), AtmosStorageHeaders.META);
Map<String, String> metaMap = Maps.newHashMap();
String[] metas = meta.split(", ");
for (String entry : metas) {
@ -60,17 +56,14 @@ public class ParseSystemMetadataFromHeaders implements Function<HttpResponse, Sy
metaMap.put(entrySplit[0], entrySplit[1]);
}
assert metaMap.size() >= 12 : String.format("Should be 12 entries in %s", metaMap);
byte[] md5 = metaMap.containsKey("content-md5") ? encryptionService.fromHex(metaMap
.get("content-md5")) : null;
return new SystemMetadata(md5, dateService.iso8601SecondsDateParse(checkNotNull(metaMap
.get("atime"), "atime")), dateService.iso8601SecondsDateParse(checkNotNull(metaMap
.get("ctime"), "ctime")), checkNotNull(metaMap.get("gid"), "gid"), dateService
.iso8601SecondsDateParse(checkNotNull(metaMap.get("itime"), "itime")), dateService
.iso8601SecondsDateParse(checkNotNull(metaMap.get("mtime"), "mtime")), Integer
.parseInt(checkNotNull(metaMap.get("nlink"), "nlink")), checkNotNull(metaMap
.get("objectid"), "objectid"), checkNotNull(metaMap.get("objname"), "objname"),
checkNotNull(metaMap.get("policyname"), "policyname"), Long.parseLong(checkNotNull(
metaMap.get("size"), "size")), FileType.fromValue(checkNotNull(metaMap
.get("type"), "type")), checkNotNull(metaMap.get("uid"), "uid"));
byte[] md5 = metaMap.containsKey("content-md5") ? CryptoStreams.hex(metaMap.get("content-md5")) : null;
return new SystemMetadata(md5, dateService.iso8601SecondsDateParse(checkNotNull(metaMap.get("atime"), "atime")),
dateService.iso8601SecondsDateParse(checkNotNull(metaMap.get("ctime"), "ctime")), checkNotNull(metaMap
.get("gid"), "gid"), dateService.iso8601SecondsDateParse(checkNotNull(metaMap.get("itime"),
"itime")), dateService.iso8601SecondsDateParse(checkNotNull(metaMap.get("mtime"), "mtime")),
Integer.parseInt(checkNotNull(metaMap.get("nlink"), "nlink")), checkNotNull(metaMap.get("objectid"),
"objectid"), checkNotNull(metaMap.get("objname"), "objname"), checkNotNull(metaMap
.get("policyname"), "policyname"), Long.parseLong(checkNotNull(metaMap.get("size"), "size")),
FileType.fromValue(checkNotNull(metaMap.get("type"), "type")), checkNotNull(metaMap.get("uid"), "uid"));
}
}

24
atmos/src/main/java/org/jclouds/atmosonline/saas/util/AtmosStorageUtils.java
View File

@ -30,7 +30,8 @@ import org.jclouds.atmosonline.saas.domain.AtmosStorageError;
import org.jclouds.atmosonline.saas.filters.SignRequest;
import org.jclouds.atmosonline.saas.xml.ErrorHandler;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpCommand;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpResponse;
@ -56,10 +57,9 @@ public class AtmosStorageUtils {
@Inject
Provider<ErrorHandler> errorHandlerProvider;
public AtmosStorageError parseAtmosStorageErrorFromContent(HttpCommand command,
HttpResponse response, InputStream content) throws HttpException {
AtmosStorageError error = (AtmosStorageError) factory.create(errorHandlerProvider.get())
.parse(content);
public AtmosStorageError parseAtmosStorageErrorFromContent(HttpCommand command, HttpResponse response,
InputStream content) throws HttpException {
AtmosStorageError error = (AtmosStorageError) factory.create(errorHandlerProvider.get()).parse(content);
if (error.getCode() == 1032) {
error.setStringSigned(signer.createStringToSign(command.getRequest()));
}
@ -67,13 +67,12 @@ public class AtmosStorageUtils {
}
public static String putBlob(final AtmosStorageClient sync, EncryptionService encryptionService,
BlobToObject blob2Object, String container, Blob blob) {
public static String putBlob(final AtmosStorageClient sync, Crypto crypto, BlobToObject blob2Object,
String container, Blob blob) {
final String path = container + "/" + blob.getMetadata().getName();
deleteAndEnsureGone(sync, path);
if (blob.getMetadata().getContentMD5() != null)
blob.getMetadata().getUserMetadata().put("content-md5",
encryptionService.hex(blob.getMetadata().getContentMD5()));
blob.getMetadata().getUserMetadata().put("content-md5", CryptoStreams.hex(blob.getMetadata().getContentMD5()));
sync.createFile(container, blob2Object.apply(blob));
return path;
}
@ -93,10 +92,9 @@ public class AtmosStorageUtils {
}
}
public AtmosStorageError parseAtmosStorageErrorFromContent(HttpCommand command,
HttpResponse response, String content) throws HttpException {
return parseAtmosStorageErrorFromContent(command, response, new ByteArrayInputStream(content
.getBytes()));
public AtmosStorageError parseAtmosStorageErrorFromContent(HttpCommand command, HttpResponse response, String content)
throws HttpException {
return parseAtmosStorageErrorFromContent(command, response, new ByteArrayInputStream(content.getBytes()));
}
public static String adjustContainerIfDirOptionPresent(String container,

96
atmos/src/test/java/org/jclouds/atmosonline/saas/AtmosStorageClientLiveTest.java
View File

@ -89,8 +89,7 @@ public class AtmosStorageClientLiveTest {
private final String metadataValue;
private final String compare;
private ObjectMatches(AtmosStorageClient connection, String name, String metadataValue,
String compare) {
private ObjectMatches(AtmosStorageClient connection, String name, String metadataValue, String compare) {
this.connection = connection;
this.name = name;
this.metadataValue = metadataValue;
@ -115,16 +114,12 @@ public class AtmosStorageClientLiveTest {
private BlobStoreContext context;
@BeforeGroups(groups = { "live" })
public void setupClient() throws InterruptedException, ExecutionException, TimeoutException,
IOException {
String identity = checkNotNull(System.getProperty("jclouds.test.identity"),
"jclouds.test.identity");
String credential = checkNotNull(System.getProperty("jclouds.test.credential"),
"jclouds.test.credential");
context = new BlobStoreContextFactory().createContext("atmosonline", identity, credential,
ImmutableSet.<Module> of(new Log4JLoggingModule()));
RestContext<AtmosStorageClient, AtmosStorageAsyncClient> restContext = context
.getProviderSpecificContext();
public void setupClient() throws InterruptedException, ExecutionException, TimeoutException, IOException {
String identity = checkNotNull(System.getProperty("jclouds.test.identity"), "jclouds.test.identity");
String credential = checkNotNull(System.getProperty("jclouds.test.credential"), "jclouds.test.credential");
context = new BlobStoreContextFactory().createContext("atmosonline", identity, credential, ImmutableSet
.<Module> of(new Log4JLoggingModule()));
RestContext<AtmosStorageClient, AtmosStorageAsyncClient> restContext = context.getProviderSpecificContext();
connection = restContext.getApi();
for (DirectoryEntry entry : connection.listDirectories()) {
if (entry.getObjectName().startsWith(containerPrefix)) {
@ -169,8 +164,8 @@ public class AtmosStorageClientLiveTest {
createOrReplaceObject("object2", "here is my data!", "meta-value1");
createOrReplaceObject("object3", "here is my data!", "meta-value1");
createOrReplaceObject("object4", "here is my data!", "meta-value1");
BoundedSet<? extends DirectoryEntry> r2 = connection.listDirectory(privateDirectory,
ListOptions.Builder.limit(1));
BoundedSet<? extends DirectoryEntry> r2 = connection
.listDirectory(privateDirectory, ListOptions.Builder.limit(1));
assertEquals(r2.size(), 1);
assert r2.getToken() != null;
assertEquals(Iterables.getLast(Sets.newTreeSet(r2)).getObjectName(), "object2");
@ -196,8 +191,7 @@ public class AtmosStorageClientLiveTest {
// loop to gather metrics
for (boolean stream : new Boolean[] { true, false }) {
for (int i = 0; i < 10; i++) {
System.err.printf("upload/delete/create attempt %d type %s%n", i + 1, stream ? "stream"
: "string");
System.err.printf("upload/delete/create attempt %d type %s%n", i + 1, stream ? "stream" : "string");
// try updating
createOrUpdateWithErrorLoop(stream, "there is my data", "2");
@ -209,8 +203,7 @@ public class AtmosStorageClientLiveTest {
}
}
private void createOrUpdateWithErrorLoop(boolean stream, String data, String metadataValue)
throws Exception {
private void createOrUpdateWithErrorLoop(boolean stream, String data, String metadataValue) throws Exception {
createOrReplaceObject("object", makeData(data, stream), metadataValue);
assertEventuallyObjectMatches("object", data, metadataValue);
}
@ -219,14 +212,13 @@ public class AtmosStorageClientLiveTest {
return stream ? Utils.toInputStream(in) : in;
}
private void createOrReplaceObject(String name, Object data, String metadataValue)
throws Exception {
private void createOrReplaceObject(String name, Object data, String metadataValue) throws Exception {
// Test PUT with string data, ETag hash, and a piece of metadata
AtmosObject object = connection.newObject();
object.getContentMetadata().setName(name);
object.setPayload(Payloads.newPayload(data));
object.getContentMetadata().setContentLength(16l);
context.utils().encryption().generateMD5BufferingIfNotRepeatable(object);
Payloads.calculateMD5(object);
object.getContentMetadata().setContentType("text/plain");
object.getUserMetadata().getMetadata().put("Metadata", metadataValue);
replaceObject(object);
@ -243,9 +235,8 @@ public class AtmosStorageClientLiveTest {
try {
assertion.run();
if (i > 0)
System.err.printf("%d attempts and %dms asserting %s%n", i + 1, System
.currentTimeMillis()
- start, assertion.getClass().getSimpleName());
System.err.printf("%d attempts and %dms asserting %s%n", i + 1, System.currentTimeMillis() - start,
assertion.getClass().getSimpleName());
return;
} catch (AssertionError e) {
error = e;
@ -257,10 +248,9 @@ public class AtmosStorageClientLiveTest {
}
protected void assertEventuallyObjectMatches(final String name, final String compare,
final String metadataValue) throws InterruptedException {
assertEventually(new ObjectMatches(connection, privateDirectory + "/" + name, metadataValue,
compare));
protected void assertEventuallyObjectMatches(final String name, final String compare, final String metadataValue)
throws InterruptedException {
assertEventually(new ObjectMatches(connection, privateDirectory + "/" + name, metadataValue, compare));
}
protected void assertEventuallyHeadMatches(final String name, final String metadataValue)
@ -268,17 +258,15 @@ public class AtmosStorageClientLiveTest {
assertEventually(new HeadMatches(connection, privateDirectory + "/" + name, metadataValue));
}
private static void verifyHeadObject(AtmosStorageClient connection, String path,
String metadataValue) throws InterruptedException, ExecutionException,
TimeoutException, IOException {
private static void verifyHeadObject(AtmosStorageClient connection, String path, String metadataValue)
throws InterruptedException, ExecutionException, TimeoutException, IOException {
AtmosObject getBlob = connection.headFile(path);
assertEquals(Utils.toStringAndClose(getBlob.getPayload().getInput()), "");
verifyMetadata(metadataValue, getBlob);
}
private static void verifyObject(AtmosStorageClient connection, String path, String compare,
String metadataValue) throws InterruptedException, ExecutionException,
TimeoutException, IOException {
private static void verifyObject(AtmosStorageClient connection, String path, String compare, String metadataValue)
throws InterruptedException, ExecutionException, TimeoutException, IOException {
AtmosObject getBlob = connection.readFile(path);
assertEquals(Utils.toStringAndClose(getBlob.getPayload().getInput()), compare);
verifyMetadata(metadataValue, getBlob);
@ -304,8 +292,8 @@ public class AtmosStorageClientLiveTest {
try {
Utils.toStringAndClose(URI.create(
"http://accesspoint.emccis.com/rest/objects/"
+ getBlob.getSystemMetadata().getObjectID()).toURL().openStream());
"http://accesspoint.emccis.com/rest/objects/" + getBlob.getSystemMetadata().getObjectID()).toURL()
.openStream());
assert false : "shouldn't have worked, since it is private";
} catch (IOException e) {
@ -323,29 +311,25 @@ public class AtmosStorageClientLiveTest {
long time = System.currentTimeMillis();
try {
connection.createFile(privateDirectory, object);
System.err.printf("%s %s; %dms%n", "created",
object.getPayload() instanceof InputStreamPayload ? "stream" : "string", System
.currentTimeMillis()
- time);
System.err.printf("%s %s; %dms%n", "created", object.getPayload() instanceof InputStreamPayload ? "stream"
: "string", System.currentTimeMillis() - time);
} catch (Exception e) {
String message = Throwables.getRootCause(e).getMessage();
System.err.printf("failure %s %s; %dms: [%s]%n", "creating",
object.getPayload() instanceof InputStreamPayload ? "stream" : "string", System
.currentTimeMillis()
object.getPayload() instanceof InputStreamPayload ? "stream" : "string", System.currentTimeMillis()
- time, message);
throw e;
}
}
private void deleteConfirmed(final String path) throws InterruptedException, ExecutionException,
TimeoutException {
private void deleteConfirmed(final String path) throws InterruptedException, ExecutionException, TimeoutException {
long time = System.currentTimeMillis();
deleteConsistencyAware(path);
System.err.printf("confirmed deletion after %dms%n", System.currentTimeMillis() - time);
}
protected void deleteImmediateAndVerifyWithHead(final String path) throws InterruptedException,
ExecutionException, TimeoutException {
protected void deleteImmediateAndVerifyWithHead(final String path) throws InterruptedException, ExecutionException,
TimeoutException {
try {
connection.deletePath(path);
} catch (KeyNotFoundException ex) {
@ -357,8 +341,8 @@ public class AtmosStorageClientLiveTest {
}
protected void deleteConsistencyAware(final String path) throws InterruptedException,
ExecutionException, TimeoutException {
protected void deleteConsistencyAware(final String path) throws InterruptedException, ExecutionException,
TimeoutException {
try {
connection.deletePath(path);
} catch (KeyNotFoundException ex) {
@ -370,8 +354,7 @@ public class AtmosStorageClientLiveTest {
}, INCONSISTENCY_WINDOW);
}
protected void retryAndCheckSystemMetadataAndPutIfPresentReplaceStrategy(AtmosObject object)
throws Exception {
protected void retryAndCheckSystemMetadataAndPutIfPresentReplaceStrategy(AtmosObject object) throws Exception {
int failures = 0;
while (true) {
@ -390,8 +373,7 @@ public class AtmosStorageClientLiveTest {
object.getPayload() instanceof InputStreamPayload ? "stream" : "string");
}
private void checkSystemMetadataAndPutIfPresentReplaceStrategy(AtmosObject object)
throws Exception {
private void checkSystemMetadataAndPutIfPresentReplaceStrategy(AtmosObject object) throws Exception {
long time = System.currentTimeMillis();
boolean update = true;
try {
@ -405,15 +387,13 @@ public class AtmosStorageClientLiveTest {
else
connection.createFile(privateDirectory, object);
System.err.printf("%s %s; %dms%n", update ? "updated" : "created",
object.getPayload() instanceof InputStreamPayload ? "stream" : "string", System
.currentTimeMillis()
object.getPayload() instanceof InputStreamPayload ? "stream" : "string", System.currentTimeMillis()
- time);
} catch (Exception e) {
String message = Throwables.getRootCause(e).getMessage();
System.err.printf("failure %s %s; %dms: [%s]%n", update ? "updating" : "creating", object
.getPayload() instanceof InputStreamPayload ? "stream" : "string", System
.currentTimeMillis()
- time, message);
System.err.printf("failure %s %s; %dms: [%s]%n", update ? "updating" : "creating",
object.getPayload() instanceof InputStreamPayload ? "stream" : "string", System.currentTimeMillis()
- time, message);
throw e;
}
}

28
atmos/src/test/java/org/jclouds/atmosonline/saas/functions/ParseSystemMetadataFromHeadersTest.java
View File

@ -22,8 +22,8 @@ import static org.testng.Assert.assertEquals;
import org.jclouds.atmosonline.saas.domain.FileType;
import org.jclouds.atmosonline.saas.domain.SystemMetadata;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.DateService;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpResponse;
import org.jclouds.io.Payloads;
import org.testng.annotations.Test;
@ -41,29 +41,23 @@ public class ParseSystemMetadataFromHeadersTest {
public void test() {
Injector injector = Guice.createInjector();
ParseSystemMetadataFromHeaders parser = injector
.getInstance(ParseSystemMetadataFromHeaders.class);
ParseSystemMetadataFromHeaders parser = injector.getInstance(ParseSystemMetadataFromHeaders.class);
DateService dateService = injector.getInstance(DateService.class);
EncryptionService encryptionService = injector.getInstance(EncryptionService.class);
HttpResponse response = new HttpResponse(200, "ok", Payloads.newStringPayload(""));
response
.getHeaders()
.put(
"x-emc-meta",
"content-md5=1f3870be274f6c49b3e31a0c6728957f, atime=2009-10-12T16:09:42Z, mtime=2009-10-19T04:37:00Z,"
+ " ctime=2009-10-19T04:37:00Z, itime=2009-10-12T16:09:42Z, type=directory, uid=root, "
+ "gid=rootr, objectid=4980cdb2b010109b04a44f7bb83f5f04ad354c638ae5, "
+ "objname=e913e09366364e9ba384b8fead643d43, size=4096, nlink=1, policyname=default");
SystemMetadata expected = new SystemMetadata(encryptionService
.fromHex("1f3870be274f6c49b3e31a0c6728957f"),
response.getHeaders().put(
"x-emc-meta",
"content-md5=1f3870be274f6c49b3e31a0c6728957f, atime=2009-10-12T16:09:42Z, mtime=2009-10-19T04:37:00Z,"
+ " ctime=2009-10-19T04:37:00Z, itime=2009-10-12T16:09:42Z, type=directory, uid=root, "
+ "gid=rootr, objectid=4980cdb2b010109b04a44f7bb83f5f04ad354c638ae5, "
+ "objname=e913e09366364e9ba384b8fead643d43, size=4096, nlink=1, policyname=default");
SystemMetadata expected = new SystemMetadata(CryptoStreams.hex("1f3870be274f6c49b3e31a0c6728957f"),
dateService.iso8601SecondsDateParse("2009-10-12T16:09:42Z"), dateService
.iso8601SecondsDateParse("2009-10-19T04:37:00Z"), "rootr", dateService
.iso8601SecondsDateParse("2009-10-12T16:09:42Z"), dateService
.iso8601SecondsDateParse("2009-10-19T04:37:00Z"), 1,
"4980cdb2b010109b04a44f7bb83f5f04ad354c638ae5", "e913e09366364e9ba384b8fead643d43",
"default", 4096l, FileType.DIRECTORY, "root"
.iso8601SecondsDateParse("2009-10-19T04:37:00Z"), 1, "4980cdb2b010109b04a44f7bb83f5f04ad354c638ae5",
"e913e09366364e9ba384b8fead643d43", "default", 4096l, FileType.DIRECTORY, "root"
);
SystemMetadata data = parser.apply(response);

18
atmos/src/test/java/org/jclouds/atmosonline/saas/internal/StubAtmosStorageAsyncClient.java
View File

@ -21,7 +21,6 @@ package org.jclouds.atmosonline.saas.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
import static com.google.common.util.concurrent.Futures.immediateFuture;
import static org.jclouds.concurrent.ConcurrentUtils.compose;
import java.net.URI;
import java.util.concurrent.ExecutionException;
@ -47,6 +46,7 @@ import org.jclouds.blobstore.TransientAsyncBlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.functions.HttpGetOptionsListToGetOptions;
import org.jclouds.concurrent.Futures;
import org.jclouds.http.options.GetOptions;
import com.google.common.base.Function;
@ -97,7 +97,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
container = directoryName;
path = null;
}
return compose(blobStore.createContainerInLocation(null, container), new Function<Boolean, URI>() {
return Futures.compose(blobStore.createContainerInLocation(null, container), new Function<Boolean, URI>() {
public URI apply(Boolean from) {
if (path != null) {
@ -123,7 +123,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
object.getContentMetadata().setName(path + "/" + file);
}
Blob blob = object2Blob.apply(object);
return compose(blobStore.putBlob(container, blob), new Function<String, URI>() {
return Futures.compose(blobStore.putBlob(container, blob), new Function<String, URI>() {
public URI apply(String from) {
return URI.create(uri);
@ -135,7 +135,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
public ListenableFuture<Void> deletePath(String path) {
if (path.indexOf('/') == path.length() - 1) {
// chop off the trailing slash
return compose(blobStore.deleteContainerImpl(path.substring(0, path.length() - 1)),
return Futures.compose(blobStore.deleteContainerImpl(path.substring(0, path.length() - 1)),
new Function<Boolean, Void>() {
public Void apply(Boolean from) {
@ -160,7 +160,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
else {
String container = path.substring(0, path.indexOf('/'));
path = path.substring(path.indexOf('/') + 1);
return compose(blobStore.blobMetadata(container, path), new Function<BlobMetadata, UserMetadata>() {
return Futures.compose(blobStore.blobMetadata(container, path), new Function<BlobMetadata, UserMetadata>() {
public UserMetadata apply(BlobMetadata from) {
return blob2ObjectInfo.apply(from).getUserMetadata();
}
@ -172,7 +172,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
String container = path.substring(0, path.indexOf('/'));
path = path.substring(path.indexOf('/') + 1);
try {
return compose(blobStore.getBlob(container, path), blob2Object, service);
return Futures.compose(blobStore.getBlob(container, path), blob2Object, service);
} catch (Exception e) {
return immediateFailedFuture(Throwables.getRootCause(e));
}
@ -181,7 +181,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
public ListenableFuture<BoundedSet<? extends DirectoryEntry>> listDirectories(ListOptions... optionsList) {
// org.jclouds.blobstore.options.ListOptions options = container2ContainerListOptions
// .apply(optionsList);
return compose(blobStore.list(), resource2ObjectList, service);
return Futures.compose(blobStore.list(), resource2ObjectList, service);
}
public ListenableFuture<BoundedSet<? extends DirectoryEntry>> listDirectory(String directoryName,
@ -194,7 +194,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
if (!path.equals(""))
options.inDirectory(path);
}
return compose(blobStore.list(container, options), resource2ObjectList, service);
return Futures.compose(blobStore.list(container, options), resource2ObjectList, service);
}
public AtmosObject newObject() {
@ -222,7 +222,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient {
String container = path.substring(0, path.indexOf('/'));
String blobName = path.substring(path.indexOf('/') + 1);
org.jclouds.blobstore.options.GetOptions getOptions = httpGetOptionsConverter.apply(options);
return compose(blobStore.getBlob(container, blobName, getOptions), blob2Object, service);
return Futures.compose(blobStore.getBlob(container, blobName, getOptions), blob2Object, service);
}
public ListenableFuture<Void> updateFile(String parent, AtmosObject object) {

11
aws/core/src/main/java/org/jclouds/aws/ec2/binders/BindS3UploadPolicyAndSignature.java
View File

@ -25,10 +25,13 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.aws.filters.FormSigner;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.Binder;
import com.google.common.base.Charsets;
/**
*
* @author Adrian Cole
@ -36,16 +39,14 @@ import org.jclouds.rest.Binder;
@Singleton
public class BindS3UploadPolicyAndSignature implements Binder {
private final FormSigner signer;
private final EncryptionService encryptionService;
@Inject
BindS3UploadPolicyAndSignature(FormSigner signer, EncryptionService encryptionService) {
BindS3UploadPolicyAndSignature(FormSigner signer, Crypto crypto) {
this.signer = signer;
this.encryptionService = encryptionService;
}
public void bindToRequest(HttpRequest request, Object input) {
String encodedJson = encryptionService.base64(checkNotNull(input, "json").toString().getBytes());
String encodedJson = CryptoStreams.base64(checkNotNull(input, "json").toString().getBytes(Charsets.UTF_8));
addFormParamTo(request, "Storage.S3.UploadPolicy", encodedJson);
String signature = signer.sign(encodedJson);
addFormParamTo(request, "Storage.S3.UploadPolicySignature", signature);

3
aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java
View File

@ -36,7 +36,7 @@ import static org.jclouds.aws.ec2.util.EC2Utils.getAllRunningInstancesInRegion;
import static org.jclouds.aws.ec2.util.EC2Utils.parseHandle;
import static org.jclouds.compute.domain.OsFamily.CENTOS;
import static org.jclouds.compute.domain.OsFamily.UBUNTU;
import static org.jclouds.concurrent.ConcurrentUtils.transformParallel;
import static org.jclouds.concurrent.FutureIterables.transformParallel;
import java.net.URI;
import java.security.SecureRandom;
@ -124,7 +124,6 @@ import com.google.common.base.Splitter;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
import com.google.inject.AbstractModule;

2
aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/DescribeImagesParallel.java
View File

@ -19,7 +19,7 @@
package org.jclouds.aws.ec2.compute.strategy;
import static com.google.common.collect.Iterables.concat;
import static org.jclouds.concurrent.ConcurrentUtils.transformParallel;
import static org.jclouds.concurrent.FutureIterables.transformParallel;
import java.util.Set;
import java.util.Map.Entry;

11
aws/core/src/main/java/org/jclouds/aws/ec2/functions/ConvertUnencodedBytesToBase64EncodedString.java
View File

@ -21,10 +21,9 @@ package org.jclouds.aws.ec2.functions;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import com.google.common.base.Function;
@ -36,17 +35,13 @@ import com.google.common.base.Function;
@Singleton
public class ConvertUnencodedBytesToBase64EncodedString implements Function<Object, String> {
@Inject
EncryptionService encryptionService;
@Override
public String apply(Object from) {
checkArgument(checkNotNull(from, "input") instanceof byte[],
"this binder is only valid for byte []!");
checkArgument(checkNotNull(from, "input") instanceof byte[], "this binder is only valid for byte []!");
byte[] unencodedData = (byte[]) from;
checkArgument(checkNotNull(unencodedData, "unencodedData").length <= 16 * 1024,
"userData cannot be larger than 16kb");
return encryptionService.base64(unencodedData);
return CryptoStreams.base64(unencodedData);
}
}

10
aws/core/src/main/java/org/jclouds/aws/ec2/xml/UnencodeStringValueHandler.java
View File

@ -18,9 +18,7 @@
*/
package org.jclouds.aws.ec2.xml;
import javax.inject.Inject;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import com.google.common.base.Charsets;
@ -32,12 +30,8 @@ import com.google.common.base.Charsets;
*/
public class UnencodeStringValueHandler extends StringValueHandler {
@Inject
private EncryptionService encryptionService;
@Override
public String getResult() {
return super.getResult() == null ? null : new String(encryptionService.fromBase64(super
.getResult()), Charsets.UTF_8);
return super.getResult() == null ? null : new String(CryptoStreams.base64(super.getResult()), Charsets.UTF_8);
}
}

39
aws/core/src/main/java/org/jclouds/aws/filters/FormSigner.java
View File

@ -43,13 +43,15 @@ import javax.inject.Singleton;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.Constants;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.TimeStamp;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.io.InputSuppliers;
import org.jclouds.logging.Logger;
import org.jclouds.rest.RequestSigner;
import org.jclouds.util.Utils;
@ -69,13 +71,13 @@ import com.google.common.collect.Multimap;
@Singleton
public class FormSigner implements HttpRequestFilter, RequestSigner {
public static String[] mandatoryParametersForSignature = new String[] { ACTION,
SIGNATURE_METHOD, SIGNATURE_VERSION, VERSION };
public static String[] mandatoryParametersForSignature = new String[] { ACTION, SIGNATURE_METHOD, SIGNATURE_VERSION,
VERSION };
private final SignatureWire signatureWire;
private final String accessKey;
private final String secretKey;
private final Provider<String> dateService;
private final EncryptionService encryptionService;
private final Crypto crypto;
private final HttpUtils utils;
@Resource
@ -83,24 +85,20 @@ public class FormSigner implements HttpRequestFilter, RequestSigner {
private Logger signatureLog = Logger.NULL;
@Inject
public FormSigner(SignatureWire signatureWire,
@Named(Constants.PROPERTY_IDENTITY) String accessKey,
@Named(Constants.PROPERTY_CREDENTIAL) String secretKey,
@TimeStamp Provider<String> dateService, EncryptionService encryptionService,
HttpUtils utils) {
public FormSigner(SignatureWire signatureWire, @Named(Constants.PROPERTY_IDENTITY) String accessKey,
@Named(Constants.PROPERTY_CREDENTIAL) String secretKey, @TimeStamp Provider<String> dateService,
Crypto crypto, HttpUtils utils) {
this.signatureWire = signatureWire;
this.accessKey = accessKey;
this.secretKey = secretKey;
this.dateService = dateService;
this.encryptionService = encryptionService;
this.crypto = crypto;
this.utils = utils;
}
public void filter(HttpRequest request) throws HttpException {
checkNotNull(request.getFirstHeaderOrNull(HttpHeaders.HOST),
"request is not ready to sign; host not present");
Multimap<String, String> decodedParams = parseQueryToMap(request.getPayload().getRawContent()
.toString());
checkNotNull(request.getFirstHeaderOrNull(HttpHeaders.HOST), "request is not ready to sign; host not present");
Multimap<String, String> decodedParams = parseQueryToMap(request.getPayload().getRawContent().toString());
addSigningParams(decodedParams);
validateParams(decodedParams);
String stringToSign = createStringToSign(request, decodedParams);
@ -141,8 +139,7 @@ public class FormSigner implements HttpRequestFilter, RequestSigner {
@VisibleForTesting
void validateParams(Multimap<String, String> params) {
for (String parameter : mandatoryParametersForSignature) {
checkState(params.containsKey(parameter), "parameter " + parameter
+ " is required for signature");
checkState(params.containsKey(parameter), "parameter " + parameter + " is required for signature");
}
}
@ -155,8 +152,8 @@ public class FormSigner implements HttpRequestFilter, RequestSigner {
public String sign(String stringToSign) {
String signature;
try {
signature = encryptionService.base64(encryptionService.hmacSha256(stringToSign, secretKey
.getBytes()));
signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(stringToSign), crypto
.hmacSHA256(secretKey.getBytes())));
if (signatureWire.enabled())
signatureWire.input(Utils.toInputStream(signature));
} catch (Exception e) {
@ -172,8 +169,7 @@ public class FormSigner implements HttpRequestFilter, RequestSigner {
// StringToSign = HTTPVerb + "\n" +
stringToSign.append(request.getMethod()).append("\n");
// ValueOfHostHeaderInLowercase + "\n" +
stringToSign.append(request.getFirstHeaderOrNull(HttpHeaders.HOST).toLowerCase())
.append("\n");
stringToSign.append(request.getFirstHeaderOrNull(HttpHeaders.HOST).toLowerCase()).append("\n");
// HTTPRequestURI + "\n" +
stringToSign.append(request.getEndpoint().getPath()).append("\n");
// CanonicalizedFormString <from the preceding step>
@ -204,8 +200,7 @@ public class FormSigner implements HttpRequestFilter, RequestSigner {
}
public String createStringToSign(HttpRequest input) {
return createStringToSign(input, parseQueryToMap(input.getPayload().getRawContent()
.toString()));
return createStringToSign(input, parseQueryToMap(input.getPayload().getRawContent().toString()));
}
}

17
aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3AsyncBlobStore.java
View File

@ -19,11 +19,9 @@
package org.jclouds.aws.s3.blobstore;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.concurrent.ConcurrentUtils.compose;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import com.google.common.util.concurrent.ListenableFuture;
import javax.inject.Inject;
import javax.inject.Named;
@ -56,11 +54,13 @@ import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.concurrent.Futures;
import org.jclouds.domain.Location;
import org.jclouds.http.options.GetOptions;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ListenableFuture;
/**
*
@ -106,7 +106,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<PageSet<? extends StorageMetadata>> list() {
return compose(async.listOwnedBuckets(),
return Futures.compose(async.listOwnedBuckets(),
new Function<Set<BucketMetadata>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(Set<BucketMetadata> from) {
return new PageSetImpl<StorageMetadata>(Iterables.transform(from, bucket2ResourceMd), null);
@ -150,9 +150,10 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
public ListenableFuture<PageSet<? extends StorageMetadata>> list(String container, ListContainerOptions options) {
ListBucketOptions httpOptions = container2BucketListOptions.apply(options);
ListenableFuture<ListBucketResponse> returnVal = async.listBucket(container, httpOptions);
ListenableFuture<PageSet<? extends StorageMetadata>> list = compose(returnVal, bucket2ResourceList, service);
return (options.isDetailed()) ? compose(list, fetchBlobMetadataProvider.get().setContainerName(container),
service) : list;
ListenableFuture<PageSet<? extends StorageMetadata>> list = Futures.compose(returnVal, bucket2ResourceList,
service);
return (options.isDetailed()) ? Futures.compose(list,
fetchBlobMetadataProvider.get().setContainerName(container), service) : list;
}
/**
@ -185,7 +186,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<BlobMetadata> blobMetadata(String container, String key) {
return compose(async.headObject(container, key), new Function<ObjectMetadata, BlobMetadata>() {
return Futures.compose(async.headObject(container, key), new Function<ObjectMetadata, BlobMetadata>() {
@Override
public BlobMetadata apply(ObjectMetadata from) {
@ -206,7 +207,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore {
@Override
public ListenableFuture<Blob> getBlob(String container, String key, org.jclouds.blobstore.options.GetOptions options) {
GetOptions httpOptions = blob2ObjectGetOptions.apply(options);
return compose(async.getObject(container, key, httpOptions), object2Blob, service);
return Futures.compose(async.getObject(container, key, httpOptions), object2Blob, service);
}
/**

2
aws/core/src/main/java/org/jclouds/aws/s3/domain/S3Object.java
View File

@ -18,7 +18,7 @@
*/
package org.jclouds.aws.s3.domain;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.io.PayloadEnclosing;
import com.google.common.collect.Multimap;
import com.google.inject.internal.Nullable;

2
aws/core/src/main/java/org/jclouds/aws/s3/domain/internal/S3ObjectImpl.java
View File

@ -25,9 +25,9 @@ import javax.inject.Inject;
import org.jclouds.aws.s3.domain.AccessControlList;
import org.jclouds.aws.s3.domain.MutableObjectMetadata;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.http.internal.PayloadEnclosingImpl;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadEnclosing;
import org.jclouds.io.payloads.DelegatingPayload;
import com.google.common.collect.LinkedHashMultimap;

44
aws/core/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java
View File

@ -43,13 +43,15 @@ import javax.ws.rs.core.HttpHeaders;
import org.jclouds.Constants;
import org.jclouds.aws.s3.Bucket;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.TimeStamp;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.io.InputSuppliers;
import org.jclouds.logging.Logger;
import org.jclouds.rest.RequestSigner;
import org.jclouds.rest.internal.GeneratedHttpRequest;
@ -71,13 +73,13 @@ import com.google.common.collect.Iterables;
public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSigner {
private final String[] firstHeadersToSign = new String[] { HttpHeaders.DATE };
public static Set<String> SPECIAL_QUERIES = ImmutableSet.of("acl", "torrent", "logging",
"location", "requestPayment");
public static Set<String> SPECIAL_QUERIES = ImmutableSet.of("acl", "torrent", "logging", "location",
"requestPayment");
private final SignatureWire signatureWire;
private final String accessKey;
private final String secretKey;
private final Provider<String> timeStampProvider;
private final EncryptionService encryptionService;
private final Crypto crypto;
private final HttpUtils utils;
@Resource
@ -90,15 +92,11 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
private final boolean isVhostStyle;
@Inject
public RequestAuthorizeSignature(SignatureWire signatureWire,
@Named(PROPERTY_AUTH_TAG) String authTag,
public RequestAuthorizeSignature(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,
@Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
@Named(PROPERTY_S3_SERVICE_PATH) String servicePath,
@Named(PROPERTY_HEADER_TAG) String headerTag,
@Named(PROPERTY_IDENTITY) String accessKey,
@Named(PROPERTY_CREDENTIAL) String secretKey,
@TimeStamp Provider<String> timeStampProvider, EncryptionService encryptionService,
HttpUtils utils) {
@Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
@Named(PROPERTY_IDENTITY) String accessKey, @Named(PROPERTY_CREDENTIAL) String secretKey,
@TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
this.isVhostStyle = isVhostStyle;
this.servicePath = servicePath;
this.headerTag = headerTag;
@ -107,7 +105,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
this.accessKey = accessKey;
this.secretKey = secretKey;
this.timeStampProvider = timeStampProvider;
this.encryptionService = encryptionService;
this.crypto = crypto;
this.utils = utils;
}
@ -145,8 +143,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
public String sign(String toSign) {
String signature;
try {
signature = encryptionService.base64(encryptionService.hmacSha1(toSign, secretKey
.getBytes()));
signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign), crypto.hmacSHA1(secretKey
.getBytes())));
} catch (Exception e) {
throw new HttpException("error signing request", e);
}
@ -158,8 +156,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
}
void replaceDateHeader(HttpRequest request) {
request.getHeaders().replaceValues(HttpHeaders.DATE,
Collections.singletonList(timeStampProvider.get()));
request.getHeaders().replaceValues(HttpHeaders.DATE, Collections.singletonList(timeStampProvider.get()));
}
void appendAmzHeaders(HttpRequest request, StringBuilder toSign) {
@ -177,12 +174,10 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
}
void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload()
.getContentMD5())).append("\n");
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload()
.getContentType())).append("\n");
buffer.append(utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMD5()))
.append("\n");
buffer.append(utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentType()))
.append("\n");
}
void appendHttpHeaders(HttpRequest request, StringBuilder toSign) {
@ -192,8 +187,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
@VisibleForTesting
void appendBucketName(HttpRequest req, StringBuilder toSign) {
checkArgument(req instanceof GeneratedHttpRequest<?>,
"this should be a generated http request");
checkArgument(req instanceof GeneratedHttpRequest<?>, "this should be a generated http request");
GeneratedHttpRequest<?> request = GeneratedHttpRequest.class.cast(req);
String bucketName = null;

12
aws/core/src/main/java/org/jclouds/aws/s3/functions/ParseObjectMetadataFromHeaders.java
View File

@ -29,7 +29,7 @@ import org.jclouds.aws.s3.blobstore.functions.BlobToObjectMetadata;
import org.jclouds.aws.s3.domain.MutableObjectMetadata;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.functions.ParseSystemAndUserMetadataFromHeaders;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpResponse;
import org.jclouds.rest.InvocationContext;
@ -45,20 +45,16 @@ import com.google.common.base.Function;
* @see <a href="http://docs.amazonwebservices.com/AmazonS3/latest/RESTObjectGET.html" />
* @author Adrian Cole
*/
public class ParseObjectMetadataFromHeaders implements
Function<HttpResponse, MutableObjectMetadata>, InvocationContext {
public class ParseObjectMetadataFromHeaders implements Function<HttpResponse, MutableObjectMetadata>, InvocationContext {
private final ParseSystemAndUserMetadataFromHeaders blobMetadataParser;
private final BlobToObjectMetadata blobToObjectMetadata;
private final EncryptionService encryptionService;
private final String userMdPrefix;
@Inject
public ParseObjectMetadataFromHeaders(ParseSystemAndUserMetadataFromHeaders blobMetadataParser,
BlobToObjectMetadata blobToObjectMetadata, EncryptionService encryptionService,
@Named(PROPERTY_USER_METADATA_PREFIX) String userMdPrefix) {
BlobToObjectMetadata blobToObjectMetadata, @Named(PROPERTY_USER_METADATA_PREFIX) String userMdPrefix) {
this.blobMetadataParser = blobMetadataParser;
this.blobToObjectMetadata = blobToObjectMetadata;
this.encryptionService = encryptionService;
this.userMdPrefix = userMdPrefix;
}
@ -71,7 +67,7 @@ public class ParseObjectMetadataFromHeaders implements
MutableObjectMetadata to = blobToObjectMetadata.apply(base);
addETagTo(from, to);
to.setSize(attemptToParseSizeAndRangeFromHeaders(from));
to.setContentMD5(encryptionService.fromHex(Utils.replaceAll(to.getETag(), '"', "")));
to.setContentMD5(CryptoStreams.hex(Utils.replaceAll(to.getETag(), '"', "")));
to.setCacheControl(from.getFirstHeaderOrNull(HttpHeaders.CACHE_CONTROL));
to.setContentDisposition(from.getFirstHeaderOrNull("Content-Disposition"));
to.setContentEncoding(from.getFirstHeaderOrNull(HttpHeaders.CONTENT_ENCODING));

16
aws/core/src/main/java/org/jclouds/aws/s3/xml/ListBucketHandler.java
View File

@ -29,8 +29,8 @@ import org.jclouds.aws.s3.domain.ObjectMetadata;
import org.jclouds.aws.s3.domain.ObjectMetadata.StorageClass;
import org.jclouds.aws.s3.domain.internal.BucketListObjectMetadata;
import org.jclouds.aws.s3.domain.internal.ListBucketResponseImpl;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.DateService;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.functions.ParseSax;
import org.jclouds.util.Utils;
import org.xml.sax.Attributes;
@ -54,7 +54,6 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<ListBucketResp
private StringBuilder currentText = new StringBuilder();
private final DateService dateParser;
private final EncryptionService encryptionService;
private String bucketName;
private String prefix;
@ -64,16 +63,15 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<ListBucketResp
private boolean isTruncated;
@Inject
public ListBucketHandler(DateService dateParser, EncryptionService encryptionService) {
public ListBucketHandler(DateService dateParser) {
this.dateParser = dateParser;
this.encryptionService = encryptionService;
this.contents = Sets.newLinkedHashSet();
this.commonPrefixes = Sets.newLinkedHashSet();
}
public ListBucketResponse getResult() {
return new ListBucketResponseImpl(bucketName, contents, prefix, marker, nextMarker,
maxResults, delimiter, isTruncated, commonPrefixes);
return new ListBucketResponseImpl(bucketName, contents, prefix, marker, nextMarker, maxResults, delimiter,
isTruncated, commonPrefixes);
}
private boolean inCommonPrefixes;
@ -102,15 +100,15 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<ListBucketResp
currentLastModified = dateParser.iso8601DateParse(currentText.toString().trim());
} else if (qName.equals("ETag")) {
currentETag = currentText.toString().trim();
currentMD5 = encryptionService.fromHex(Utils.replaceAll(currentETag, '"', ""));
currentMD5 = CryptoStreams.hex(Utils.replaceAll(currentETag, '"', ""));
} else if (qName.equals("Size")) {
currentSize = new Long(currentText.toString().trim());
} else if (qName.equals("Owner")) {
} else if (qName.equals("StorageClass")) {
currentStorageClass = ObjectMetadata.StorageClass.valueOf(currentText.toString().trim());
} else if (qName.equals("Contents")) {
contents.add(new BucketListObjectMetadata(currentKey, currentLastModified, currentETag,
currentMD5, currentSize, currentOwner, currentStorageClass));
contents.add(new BucketListObjectMetadata(currentKey, currentLastModified, currentETag, currentMD5,
currentSize, currentOwner, currentStorageClass));
} else if (qName.equals("Name")) {
this.bucketName = currentText.toString().trim();
} else if (qName.equals("Prefix")) {

13
aws/core/src/main/java/org/jclouds/aws/sqs/xml/MD5Handler.java
View File

@ -18,9 +18,7 @@
*/
package org.jclouds.aws.sqs.xml;
import javax.inject.Inject;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.functions.ParseSax;
/**
@ -34,13 +32,6 @@ public class MD5Handler extends ParseSax.HandlerWithResult<byte[]> {
private StringBuilder currentText = new StringBuilder();
byte[] md5;
private final EncryptionService encryptionService;
@Inject
MD5Handler(EncryptionService encryptionService) {
this.encryptionService = encryptionService;
}
public byte[] getResult() {
return md5;
}
@ -48,7 +39,7 @@ public class MD5Handler extends ParseSax.HandlerWithResult<byte[]> {
public void endElement(String uri, String name, String qName) {
if (qName.equals("MD5OfMessageBody")) {
String md5Hex = currentText.toString().trim();
this.md5 = encryptionService.fromHex(md5Hex);
this.md5 = CryptoStreams.hex(md5Hex);
}
currentText = new StringBuilder();
}

8
aws/core/src/main/java/org/jclouds/aws/sqs/xml/RegexMD5Handler.java
View File

@ -23,7 +23,7 @@ import java.util.regex.Pattern;
import javax.inject.Inject;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ReturnStringIf2xx;
@ -40,11 +40,9 @@ import com.google.inject.Singleton;
public class RegexMD5Handler implements Function<HttpResponse, byte[]> {
Pattern pattern = Pattern.compile("<MD5OfMessageBody>([\\S&&[^<]]+)</MD5OfMessageBody>");
private final ReturnStringIf2xx returnStringIf200;
private final EncryptionService encryptionService;
@Inject
RegexMD5Handler(EncryptionService encryptionService, ReturnStringIf2xx returnStringIf200) {
this.encryptionService = encryptionService;
RegexMD5Handler(ReturnStringIf2xx returnStringIf200) {
this.returnStringIf200 = returnStringIf200;
}
@ -55,7 +53,7 @@ public class RegexMD5Handler implements Function<HttpResponse, byte[]> {
if (content != null) {
Matcher matcher = pattern.matcher(content);
if (matcher.find()) {
value = encryptionService.fromHex(matcher.group(1));
value = CryptoStreams.hex(matcher.group(1));
}
}
return value;

4
aws/core/src/test/java/org/jclouds/aws/ec2/services/AMIClientLiveTest.java
View File

@ -37,7 +37,6 @@ import org.jclouds.aws.ec2.domain.Image;
import org.jclouds.aws.ec2.domain.RootDeviceType;
import org.jclouds.aws.ec2.domain.Image.ImageType;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.rest.ResourceNotFoundException;
import org.jclouds.rest.RestContext;
import org.jclouds.rest.RestContextFactory;
import org.testng.annotations.AfterTest;
@ -78,9 +77,8 @@ public class AMIClientLiveTest {
client = context.getApi().getAMIServices();
}
@Test(expectedExceptions = ResourceNotFoundException.class)
public void testDescribeImageNotExists() {
client.describeImagesInRegion(null, imageIds("ami-cdf819a3"));
assertEquals(client.describeImagesInRegion(null, imageIds("ami-cdf819a3")).size(), 0);
}
@Test(expectedExceptions = AWSResponseException.class)

29
aws/core/src/test/java/org/jclouds/aws/s3/functions/ParseObjectMetadataFromHeadersTest.java
View File

@ -23,8 +23,6 @@ import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import static org.testng.Assert.assertEquals;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Date;
import java.util.Map;
@ -37,14 +35,12 @@ import org.jclouds.aws.s3.domain.internal.MutableObjectMetadataImpl;
import org.jclouds.blobstore.domain.MutableBlobMetadata;
import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
import org.jclouds.blobstore.functions.ParseSystemAndUserMetadataFromHeaders;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.encryption.internal.JCEEncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpResponse;
import org.jclouds.io.Payloads;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
/**
@ -53,17 +49,6 @@ import com.google.common.collect.ImmutableMap;
@Test(testName = "s3.ParseObjectMetadataFromHeadersTest")
public class ParseObjectMetadataFromHeadersTest {
protected volatile static EncryptionService encryptionService;
static {
try {
encryptionService = new JCEEncryptionService();
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
} catch (CertificateException e) {
Throwables.propagate(e);
}
}
@Test
void testNormal() throws Exception {
HttpResponse http = new HttpResponse(400, "boa", Payloads.newStringPayload(""));
@ -72,8 +57,8 @@ public class ParseObjectMetadataFromHeadersTest {
http.getHeaders().put(HttpHeaders.CACHE_CONTROL, "cacheControl");
http.getHeaders().put("Content-Disposition", "contentDisposition");
http.getHeaders().put(HttpHeaders.CONTENT_ENCODING, "encoding");
ParseObjectMetadataFromHeaders parser = new ParseObjectMetadataFromHeaders(blobParser(http, "\"abc\""),
blobToObjectMetadata, encryptionService, "x-amz-meta-");
ParseObjectMetadataFromHeaders parser = new ParseObjectMetadataFromHeaders(blobParser(http, "\"abcd\""),
blobToObjectMetadata, "x-amz-meta-");
MutableObjectMetadata response = parser.apply(http);
assertEquals(response, expects);
}
@ -87,9 +72,9 @@ public class ParseObjectMetadataFromHeadersTest {
http.getHeaders().put(HttpHeaders.CACHE_CONTROL, "cacheControl");
http.getHeaders().put("Content-Disposition", "contentDisposition");
http.getHeaders().put(HttpHeaders.CONTENT_ENCODING, "encoding");
http.getHeaders().put("x-amz-meta-object-eTag", "\"abc\"");
http.getHeaders().put("x-amz-meta-object-eTag", "\"abcd\"");
ParseObjectMetadataFromHeaders parser = new ParseObjectMetadataFromHeaders(blobParser(http, null),
blobToObjectMetadata, encryptionService, "x-amz-meta-");
blobToObjectMetadata, "x-amz-meta-");
MutableObjectMetadata response = parser.apply(http);
assertEquals(response, expects);
}
@ -120,9 +105,9 @@ public class ParseObjectMetadataFromHeadersTest {
expects.setCacheControl("cacheControl");
expects.setContentDisposition("contentDisposition");
expects.setContentEncoding("encoding");
expects.setContentMD5(encryptionService.fromHex("abc"));
expects.setContentMD5(CryptoStreams.hex("abcd"));
expects.setContentType("type");
expects.setETag("\"abc\"");
expects.setETag("\"abcd\"");
expects.setKey("key");
expects.setLastModified(now);
expects.setOwner(null);

8
aws/core/src/test/java/org/jclouds/aws/s3/internal/StubS3AsyncClient.java
View File

@ -21,7 +21,6 @@ package org.jclouds.aws.s3.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
import static com.google.common.util.concurrent.Futures.immediateFuture;
import static org.jclouds.concurrent.ConcurrentUtils.compose;
import java.util.Date;
import java.util.Map;
@ -67,6 +66,7 @@ import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.MutableBlobMetadata;
import org.jclouds.blobstore.functions.HttpGetOptionsListToGetOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.concurrent.Futures;
import org.jclouds.date.DateService;
import org.jclouds.domain.Location;
import org.jclouds.domain.LocationScope;
@ -144,7 +144,7 @@ public class StubS3AsyncClient implements S3AsyncClient {
public ListenableFuture<ListBucketResponse> listBucket(final String name, ListBucketOptions... optionsList) {
ListContainerOptions options = bucket2ContainerListOptions.apply(optionsList);
return compose(blobStore.list(name, options), resource2BucketList, service);
return Futures.compose(blobStore.list(name, options), resource2BucketList, service);
}
public ListenableFuture<ObjectMetadata> copyObject(final String sourceBucket, final String sourceObject,
@ -272,11 +272,11 @@ public class StubS3AsyncClient implements S3AsyncClient {
public ListenableFuture<S3Object> getObject(final String bucketName, final String key, final GetOptions... options) {
org.jclouds.blobstore.options.GetOptions getOptions = httpGetOptionsConverter.apply(options);
return compose(blobStore.getBlob(bucketName, key, getOptions), blob2Object, service);
return Futures.compose(blobStore.getBlob(bucketName, key, getOptions), blob2Object, service);
}
public ListenableFuture<ObjectMetadata> headObject(String bucketName, String key) {
return compose(blobStore.blobMetadata(bucketName, key), new Function<BlobMetadata, ObjectMetadata>() {
return Futures.compose(blobStore.blobMetadata(bucketName, key), new Function<BlobMetadata, ObjectMetadata>() {
@Override
public ObjectMetadata apply(BlobMetadata from) {
return blob2ObjectMetadata.apply(from);

102
aws/core/src/test/java/org/jclouds/aws/s3/xml/ListBucketHandlerTest.java
View File

@ -21,8 +21,6 @@ package org.jclouds.aws.s3.xml;
import static org.testng.Assert.assertEquals;
import java.io.InputStream;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.TreeSet;
import org.jclouds.aws.s3.domain.CanonicalUser;
@ -31,9 +29,8 @@ import org.jclouds.aws.s3.domain.ObjectMetadata;
import org.jclouds.aws.s3.domain.ObjectMetadata.StorageClass;
import org.jclouds.aws.s3.domain.internal.BucketListObjectMetadata;
import org.jclouds.aws.s3.domain.internal.ListBucketResponseImpl;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.DateService;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.encryption.internal.JCEEncryptionService;
import org.jclouds.http.HttpException;
import org.jclouds.http.functions.BaseHandlerTest;
import org.jclouds.http.functions.ParseSax;
@ -41,7 +38,6 @@ import org.jclouds.util.Utils;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
@Test(groups = "unit", testName = "s3.ListBucketHandlerTest")
@ -50,17 +46,6 @@ public class ListBucketHandlerTest extends BaseHandlerTest {
public static final String listBucketWithSlashDelimiterAndCommonPrefixApps = "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"> <Delimiter>/</Delimiter> <CommonPrefixes><Prefix>apps/</Prefix></CommonPrefixes></ListBucketResult>";
private DateService dateService;
protected volatile static EncryptionService encryptionService;
static {
try {
encryptionService = new JCEEncryptionService();
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
} catch (CertificateException e) {
Throwables.propagate(e);
}
}
@BeforeTest
@Override
protected void setUpInjector() {
@ -74,47 +59,50 @@ public class ListBucketHandlerTest extends BaseHandlerTest {
CanonicalUser owner = new CanonicalUser("e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0",
"ferncam");
ListBucketResponse expected = new ListBucketResponseImpl("adriancole.org.jclouds.aws.s3.amazons3testdelimiter",
ImmutableList.of(
(ObjectMetadata) new BucketListObjectMetadata("apps/0", dateService
.iso8601DateParse("2009-05-07T18:27:08.000Z"), "\"c82e6a0025c31c5de5947fda62ac51ab\"",
encryptionService.fromHex("c82e6a0025c31c5de5947fda62ac51ab"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/1", dateService
.iso8601DateParse("2009-05-07T18:27:09.000Z"), "\"944fab2c5a9a6bacf07db5e688310d7a\"",
encryptionService.fromHex("944fab2c5a9a6bacf07db5e688310d7a"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/2", dateService
.iso8601DateParse("2009-05-07T18:27:09.000Z"), "\"a227b8888045c8fd159fb495214000f0\"",
encryptionService.fromHex("a227b8888045c8fd159fb495214000f0"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/3", dateService
.iso8601DateParse("2009-05-07T18:27:09.000Z"), "\"c9caa76c3dec53e2a192608ce73eef03\"",
encryptionService.fromHex("c9caa76c3dec53e2a192608ce73eef03"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/4", dateService
.iso8601DateParse("2009-05-07T18:27:09.000Z"), "\"1ce5d0dcc6154a647ea90c7bdf82a224\"",
encryptionService.fromHex("1ce5d0dcc6154a647ea90c7bdf82a224"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/5", dateService
.iso8601DateParse("2009-05-07T18:27:09.000Z"), "\"79433524d87462ee05708a8ef894ed55\"",
encryptionService.fromHex("79433524d87462ee05708a8ef894ed55"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/6", dateService
.iso8601DateParse("2009-05-07T18:27:10.000Z"), "\"dd00a060b28ddca8bc5a21a49e306f67\"",
encryptionService.fromHex("dd00a060b28ddca8bc5a21a49e306f67"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/7", dateService
.iso8601DateParse("2009-05-07T18:27:10.000Z"), "\"8cd06eca6e819a927b07a285d750b100\"",
encryptionService.fromHex("8cd06eca6e819a927b07a285d750b100"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/8", dateService
.iso8601DateParse("2009-05-07T18:27:10.000Z"), "\"174495094d0633b92cbe46603eee6bad\"",
encryptionService.fromHex("174495094d0633b92cbe46603eee6bad"), 8, owner,
StorageClass.STANDARD),
(ObjectMetadata) new BucketListObjectMetadata("apps/9", dateService
.iso8601DateParse("2009-05-07T18:27:10.000Z"), "\"cd8a19b26fea8a827276df0ad11c580d\"",
encryptionService.fromHex("cd8a19b26fea8a827276df0ad11c580d"), 8, owner,
StorageClass.STANDARD)), "apps/", null, null, 1000, null, false, new TreeSet<String>());
ImmutableList
.of(
(ObjectMetadata) new BucketListObjectMetadata("apps/0", dateService
.iso8601DateParse("2009-05-07T18:27:08.000Z"),
"\"c82e6a0025c31c5de5947fda62ac51ab\"", CryptoStreams
.hex("c82e6a0025c31c5de5947fda62ac51ab"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/1", dateService.iso8601DateParse("2009-05-07T18:27:09.000Z"),
"\"944fab2c5a9a6bacf07db5e688310d7a\"", CryptoStreams
.hex("944fab2c5a9a6bacf07db5e688310d7a"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/2", dateService.iso8601DateParse("2009-05-07T18:27:09.000Z"),
"\"a227b8888045c8fd159fb495214000f0\"", CryptoStreams
.hex("a227b8888045c8fd159fb495214000f0"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/3", dateService.iso8601DateParse("2009-05-07T18:27:09.000Z"),
"\"c9caa76c3dec53e2a192608ce73eef03\"", CryptoStreams
.hex("c9caa76c3dec53e2a192608ce73eef03"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/4", dateService.iso8601DateParse("2009-05-07T18:27:09.000Z"),
"\"1ce5d0dcc6154a647ea90c7bdf82a224\"", CryptoStreams
.hex("1ce5d0dcc6154a647ea90c7bdf82a224"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/5", dateService.iso8601DateParse("2009-05-07T18:27:09.000Z"),
"\"79433524d87462ee05708a8ef894ed55\"", CryptoStreams
.hex("79433524d87462ee05708a8ef894ed55"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/6", dateService.iso8601DateParse("2009-05-07T18:27:10.000Z"),
"\"dd00a060b28ddca8bc5a21a49e306f67\"", CryptoStreams
.hex("dd00a060b28ddca8bc5a21a49e306f67"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/7", dateService.iso8601DateParse("2009-05-07T18:27:10.000Z"),
"\"8cd06eca6e819a927b07a285d750b100\"", CryptoStreams
.hex("8cd06eca6e819a927b07a285d750b100"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/8", dateService.iso8601DateParse("2009-05-07T18:27:10.000Z"),
"\"174495094d0633b92cbe46603eee6bad\"", CryptoStreams
.hex("174495094d0633b92cbe46603eee6bad"), 8, owner,
StorageClass.STANDARD), (ObjectMetadata) new BucketListObjectMetadata(
"apps/9", dateService.iso8601DateParse("2009-05-07T18:27:10.000Z"),
"\"cd8a19b26fea8a827276df0ad11c580d\"", CryptoStreams
.hex("cd8a19b26fea8a827276df0ad11c580d"), 8, owner,
StorageClass.STANDARD)), "apps/", null, null, 1000, null, false,
new TreeSet<String>());
ListBucketResponse result = (ListBucketResponse) factory.create(injector.getInstance(ListBucketHandler.class))
.parse(is);

19
aws/core/src/test/java/org/jclouds/aws/sqs/SQSClientLiveTest.java
View File

@ -30,10 +30,10 @@ import java.util.SortedSet;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.aws.domain.Region;
import org.jclouds.aws.sqs.domain.Queue;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.rest.RestContext;
import org.jclouds.rest.RestContextFactory;
import org.jclouds.util.Utils;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeGroups;
import org.testng.annotations.Test;
@ -63,14 +63,14 @@ public class SQSClientLiveTest {
String credential = checkNotNull(System.getProperty("jclouds.test.credential"), "jclouds.test.credential");
context = new RestContextFactory().createContext("sqs", identity, credential, ImmutableSet
.<Module> of(new Log4JLoggingModule()));
.<Module> of(new Log4JLoggingModule()));
this.client = context.getApi();
}
@Test
void testListQueuesInRegion() throws InterruptedException {
for (String region : Lists.newArrayList(null, Region.EU_WEST_1, Region.US_EAST_1, Region.US_WEST_1,
Region.AP_SOUTHEAST_1)) {
Region.AP_SOUTHEAST_1)) {
SortedSet<Queue> allResults = Sets.newTreeSet(client.listQueuesInRegion(region));
assertNotNull(allResults);
if (allResults.size() >= 1) {
@ -87,7 +87,7 @@ public class SQSClientLiveTest {
String queueName = PREFIX + "1";
for (final String region : Lists.newArrayList(null, Region.EU_WEST_1, Region.US_EAST_1, Region.US_WEST_1,
Region.AP_SOUTHEAST_1)) {
Region.AP_SOUTHEAST_1)) {
try {
SortedSet<Queue> result = Sets.newTreeSet(client.listQueuesInRegion(region, queuePrefix(queueName)));
if (result.size() >= 1) {
@ -121,9 +121,9 @@ public class SQSClientLiveTest {
}
@Test(dependsOnMethods = "testCreateQueue")
void testSendMessage() throws InterruptedException {
void testSendMessage() throws InterruptedException, IOException {
String message = "hardyharhar";
byte[] md5 = context.utils().encryption().md5(Utils.toInputStream(message));
byte[] md5 = CryptoStreams.md5(message.getBytes());
for (Queue queue : queues) {
assertEquals(client.sendMessage(queue, message), md5);
}
@ -144,9 +144,8 @@ public class SQSClientLiveTest {
private static final int INCONSISTENCY_WINDOW = 10000;
/**
* Due to eventual consistency, container commands may not return correctly
* immediately. Hence, we will try up to the inconsistency window to see if
* the assertion completes.
* Due to eventual consistency, container commands may not return correctly immediately. Hence,
* we will try up to the inconsistency window to see if the assertion completes.
*/
protected static void assertEventually(Runnable assertion) throws InterruptedException {
long start = System.currentTimeMillis();
@ -156,7 +155,7 @@ public class SQSClientLiveTest {
assertion.run();
if (i > 0)
System.err.printf("%d attempts and %dms asserting %s%n", i + 1, System.currentTimeMillis() - start,
assertion.getClass().getSimpleName());
assertion.getClass().getSimpleName());
return;
} catch (AssertionError e) {
error = e;

47
aws/demos/speedtest-sqs/src/main/java/org/jclouds/aws/sqs/SpeedTest.java
View File

@ -18,16 +18,17 @@
*/
package org.jclouds.aws.sqs;
import static org.jclouds.concurrent.ConcurrentUtils.sameThreadExecutor;
import static org.jclouds.aws.sqs.options.ListQueuesOptions.Builder.queuePrefix;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.concurrent.Future;
import org.jclouds.aws.domain.Region;
import org.jclouds.aws.sqs.domain.Queue;
import org.jclouds.concurrent.MoreExecutors;
import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
import org.jclouds.logging.ConsoleLogger;
import org.jclouds.logging.Logger;
@ -38,7 +39,6 @@ import org.jclouds.rest.RestContextFactory;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.concurrent.Future;
import com.google.inject.Module;
/**
@ -50,8 +50,8 @@ import com.google.inject.Module;
* @author Adrian Cole
*/
public class SpeedTest {
private static final ImmutableSet<String> REGIONS = ImmutableSet.of(Region.EU_WEST_1,
Region.US_EAST_1, Region.US_WEST_1, Region.AP_SOUTHEAST_1);
private static final ImmutableSet<String> REGIONS = ImmutableSet.of(Region.EU_WEST_1, Region.US_EAST_1,
Region.US_WEST_1, Region.AP_SOUTHEAST_1);
public static final int PARAMETERS = 4;
public static final String INVALID_SYNTAX = "Invalid number of parameters. Syntax is: \"accesskeyid\" \"secretkey\" \"queueName\" \"messageCount\" ";
@ -84,11 +84,10 @@ public class SpeedTest {
int messageCount = Integer.parseInt(args[3]);
Set<Module> modules = isEnterprise ? ImmutableSet.<Module> of(new NullLoggingModule(),
new EnterpriseConfigurationModule()) : ImmutableSet
.<Module> of(new NullLoggingModule());
new EnterpriseConfigurationModule()) : ImmutableSet.<Module> of(new NullLoggingModule());
RestContext<SQSClient, SQSAsyncClient> context = new RestContextFactory().createContext(
"sqs", accesskeyid, secretkey, modules);
RestContext<SQSClient, SQSAsyncClient> context = new RestContextFactory().createContext("sqs", accesskeyid,
secretkey, modules);
try {
Set<Queue> queues = Sets.newHashSet();
@ -122,30 +121,25 @@ public class SpeedTest {
}
}
private static void runTests(int messageCount, String contextName,
RestContext<SQSClient, SQSAsyncClient> context, Set<Queue> queues)
throws InterruptedException {
private static void runTests(int messageCount, String contextName, RestContext<SQSClient, SQSAsyncClient> context,
Set<Queue> queues) throws InterruptedException {
String message = "1";
long timeOut = messageCount * 200; // minimum rate should be at least 5/second
for (Queue queue : queues) {
logger.info("context: %s, region: %s, queueName: %s", contextName, queue.getRegion(),
queue.getName());
logger.info("context: %s, region: %s, queueName: %s", contextName, queue.getRegion(), queue.getName());
// fire off all the messages for the test
Map<QueueMessage, Future<byte[]>> responses = Maps.newHashMap();
for (int i = 0; i < messageCount; i++) {
responses.put(new QueueMessage(queue, message), context.getAsyncApi().sendMessage(
queue, message));
responses.put(new QueueMessage(queue, message), context.getAsyncApi().sendMessage(queue, message));
}
Map<QueueMessage, Exception> exceptions = awaitCompletion(responses, sameThreadExecutor(),
timeOut, traceLogger, String.format("context: %s, region: %s", contextName, queue
.getRegion()));
Map<QueueMessage, Exception> exceptions = awaitCompletion(responses, MoreExecutors.sameThreadExecutor(),
timeOut, traceLogger, String.format("context: %s, region: %s", contextName, queue.getRegion()));
if (exceptions.size() > 0)
logger.error("problems in context: %s, region: %s: %s", contextName, queue.getRegion(),
exceptions);
logger.error("problems in context: %s, region: %s: %s", contextName, queue.getRegion(), exceptions);
System.gc();
logger.info("pausing 5 seconds before the next run");
@ -153,21 +147,20 @@ public class SpeedTest {
}
}
private static void createQueues(String queueName,
RestContext<SQSClient, SQSAsyncClient> nullLoggingDefaultContext, Set<Queue> queues) {
private static void createQueues(String queueName, RestContext<SQSClient, SQSAsyncClient> nullLoggingDefaultContext,
Set<Queue> queues) {
for (String region : REGIONS) {
logger.info("creating queue: %s in region %s", queueName, region);
queues.add(nullLoggingDefaultContext.getApi().createQueueInRegion(region, queueName));
}
}
private static boolean purgeQueues(String queueName,
RestContext<SQSClient, SQSAsyncClient> nullLoggingDefaultContext) {
private static boolean purgeQueues(String queueName, RestContext<SQSClient, SQSAsyncClient> nullLoggingDefaultContext) {
boolean deleted = false;
for (String region : REGIONS) {
try {
SortedSet<Queue> result = Sets.newTreeSet(nullLoggingDefaultContext.getApi()
.listQueuesInRegion(region, queuePrefix(queueName)));
SortedSet<Queue> result = Sets.newTreeSet(nullLoggingDefaultContext.getApi().listQueuesInRegion(region,
queuePrefix(queueName)));
if (result.size() >= 1) {
nullLoggingDefaultContext.getApi().deleteQueue(result.last());
logger.info("deleted queue: %s in region %s", queueName, region);

29
azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/AzureAsyncBlobStore.java
View File

@ -19,12 +19,10 @@
package org.jclouds.azure.storage.blob.blobstore;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.concurrent.ConcurrentUtils.compose;
import static org.jclouds.azure.storage.options.ListOptions.Builder.includeMetadata;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import com.google.common.util.concurrent.ListenableFuture;
import javax.inject.Inject;
import javax.inject.Named;
@ -54,11 +52,13 @@ import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.blobstore.internal.BaseAsyncBlobStore;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.concurrent.Futures;
import org.jclouds.domain.Location;
import org.jclouds.http.options.GetOptions;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ListenableFuture;
/**
* @author Adrian Cole
@ -101,15 +101,16 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>> list() {
return compose(
async.listContainers(includeMetadata()),
new Function<BoundedSet<ContainerProperties>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(
BoundedSet<ContainerProperties> from) {
return new PageSetImpl<StorageMetadata>(Iterables.transform(from, container2ResourceMd), from
.getNextMarker());
}
}, service);
return Futures
.compose(
async.listContainers(includeMetadata()),
new Function<BoundedSet<ContainerProperties>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() {
public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(
BoundedSet<ContainerProperties> from) {
return new PageSetImpl<StorageMetadata>(Iterables.transform(from, container2ResourceMd),
from.getNextMarker());
}
}, service);
}
/**
@ -146,7 +147,7 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore {
public ListenableFuture<PageSet<? extends StorageMetadata>> list(String container, ListContainerOptions options) {
ListBlobsOptions azureOptions = blobStore2AzureContainerListOptions.apply(options);
ListenableFuture<ListBlobsResponse> returnVal = async.listBlobs(container, azureOptions.includeMetadata());
return compose(returnVal, azure2BlobStoreResourceList, service);
return Futures.compose(returnVal, azure2BlobStoreResourceList, service);
}
/**
@ -172,7 +173,7 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore {
public ListenableFuture<Blob> getBlob(String container, String key, org.jclouds.blobstore.options.GetOptions options) {
GetOptions azureOptions = blob2ObjectGetOptions.apply(options);
ListenableFuture<AzureBlob> returnVal = async.getBlob(container, key, azureOptions);
return compose(returnVal, azureBlob2Blob, service);
return Futures.compose(returnVal, azureBlob2Blob, service);
}
/**
@ -224,7 +225,7 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore {
*/
@Override
public ListenableFuture<BlobMetadata> blobMetadata(String container, String key) {
return compose(async.getBlobProperties(container, key), new Function<BlobProperties, BlobMetadata>() {
return Futures.compose(async.getBlobProperties(container, key), new Function<BlobProperties, BlobMetadata>() {
@Override
public BlobMetadata apply(BlobProperties from) {

2
azure/src/main/java/org/jclouds/azure/storage/blob/domain/AzureBlob.java
View File

@ -20,7 +20,7 @@ package org.jclouds.azure.storage.blob.domain;
import javax.annotation.Nullable;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.io.PayloadEnclosing;
import com.google.common.collect.Multimap;

2
azure/src/main/java/org/jclouds/azure/storage/blob/domain/internal/AzureBlobImpl.java
View File

@ -24,9 +24,9 @@ import javax.inject.Inject;
import org.jclouds.azure.storage.blob.domain.AzureBlob;
import org.jclouds.azure.storage.blob.domain.MutableBlobProperties;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.http.internal.PayloadEnclosingImpl;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadEnclosing;
import org.jclouds.io.payloads.DelegatingPayload;
import com.google.common.collect.LinkedHashMultimap;

26
azure/src/main/java/org/jclouds/azure/storage/blob/xml/ContainerNameEnumerationResultsHandler.java
View File

@ -31,8 +31,8 @@ import org.jclouds.azure.storage.blob.domain.LeaseStatus;
import org.jclouds.azure.storage.blob.domain.ListBlobsResponse;
import org.jclouds.azure.storage.blob.domain.internal.BlobPropertiesImpl;
import org.jclouds.azure.storage.blob.domain.internal.HashSetListBlobsResponse;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.DateService;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.functions.ParseSax;
import org.xml.sax.Attributes;
@ -49,8 +49,7 @@ import com.google.common.collect.Sets;
* @see <a href="http://msdn.microsoft.com/en-us/library/dd135734.aspx#samplerequestandresponse" />
* @author Adrian Cole
*/
public class ContainerNameEnumerationResultsHandler extends
ParseSax.HandlerWithResult<ListBlobsResponse> {
public class ContainerNameEnumerationResultsHandler extends ParseSax.HandlerWithResult<ListBlobsResponse> {
private Set<BlobProperties> blobMetadata = Sets.newLinkedHashSet();
private String prefix;
private String marker;
@ -63,7 +62,6 @@ public class ContainerNameEnumerationResultsHandler extends
private StringBuilder currentText = new StringBuilder();
private final EncryptionService encryptionService;
private final DateService dateParser;
private String delimiter;
private String currentName;
@ -81,20 +79,17 @@ public class ContainerNameEnumerationResultsHandler extends
private LeaseStatus currentLeaseStatus;
@Inject
public ContainerNameEnumerationResultsHandler(EncryptionService encryptionService,
DateService dateParser) {
this.encryptionService = encryptionService;
public ContainerNameEnumerationResultsHandler(DateService dateParser) {
this.dateParser = dateParser;
}
public ListBlobsResponse getResult() {
return new HashSetListBlobsResponse(blobMetadata, containerUrl, prefix, marker, maxResults,
nextMarker, delimiter, blobPrefixes);
return new HashSetListBlobsResponse(blobMetadata, containerUrl, prefix, marker, maxResults, nextMarker,
delimiter, blobPrefixes);
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes)
throws SAXException {
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
if (qName.equals("Blob")) {
inBlob = true;
inBlobPrefix = false;
@ -134,10 +129,9 @@ public class ContainerNameEnumerationResultsHandler extends
} else if (qName.equals("LeaseStatus")) {
currentLeaseStatus = LeaseStatus.fromValue(currentText.toString().trim());
} else if (qName.equals("Blob")) {
BlobProperties md = new BlobPropertiesImpl(currentBlobType, currentName, currentUrl,
currentLastModified, currentETag, currentSize, currentContentType,
currentContentMD5, currentContentEncoding, currentContentLanguage,
currentLeaseStatus, currentMetadata);
BlobProperties md = new BlobPropertiesImpl(currentBlobType, currentName, currentUrl, currentLastModified,
currentETag, currentSize, currentContentType, currentContentMD5, currentContentEncoding,
currentContentLanguage, currentLeaseStatus, currentMetadata);
blobMetadata.add(md);
currentBlobType = null;
currentName = null;
@ -166,7 +160,7 @@ public class ContainerNameEnumerationResultsHandler extends
currentSize = Long.parseLong(currentText.toString().trim());
} else if (qName.equals("Content-MD5")) {
if (!currentText.toString().trim().equals(""))
currentContentMD5 = encryptionService.fromBase64(currentText.toString().trim());
currentContentMD5 = CryptoStreams.base64(currentText.toString().trim());
} else if (qName.equals("Content-Type")) {
currentContentType = currentText.toString().trim();
} else if (qName.equals("Content-Encoding")) {

36
azure/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java
View File

@ -32,13 +32,15 @@ import javax.inject.Singleton;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.Constants;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.TimeStamp;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.io.InputSuppliers;
import org.jclouds.logging.Logger;
import org.jclouds.util.Utils;
@ -59,7 +61,7 @@ public class SharedKeyLiteAuthentication implements HttpRequestFilter {
private final String identity;
private final byte[] key;
private final Provider<String> timeStampProvider;
private final EncryptionService encryptionService;
private final Crypto crypto;
private final HttpUtils utils;
@Resource
@ -67,16 +69,14 @@ public class SharedKeyLiteAuthentication implements HttpRequestFilter {
Logger signatureLog = Logger.NULL;
@Inject
public SharedKeyLiteAuthentication(SignatureWire signatureWire,
@Named(Constants.PROPERTY_IDENTITY) String identity,
@Named(Constants.PROPERTY_CREDENTIAL) String encodedKey,
@TimeStamp Provider<String> timeStampProvider, EncryptionService encryptionService,
HttpUtils utils) {
this.encryptionService = encryptionService;
public SharedKeyLiteAuthentication(SignatureWire signatureWire, @Named(Constants.PROPERTY_IDENTITY) String identity,
@Named(Constants.PROPERTY_CREDENTIAL) String encodedKey, @TimeStamp Provider<String> timeStampProvider,
Crypto crypto, HttpUtils utils) {
this.crypto = crypto;
this.utils = utils;
this.signatureWire = signatureWire;
this.identity = identity;
this.key = encryptionService.fromBase64(encodedKey);
this.key = CryptoStreams.base64(encodedKey);
this.timeStampProvider = timeStampProvider;
}
@ -102,16 +102,13 @@ public class SharedKeyLiteAuthentication implements HttpRequestFilter {
}
private void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload()
.getContentMD5())).append("\n");
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload()
.getContentType())).append("\n");
buffer.append(utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMD5()))
.append("\n");
buffer.append(utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentType()))
.append("\n");
}
private void calculateAndReplaceAuthHeader(HttpRequest request, String toSign)
throws HttpException {
private void calculateAndReplaceAuthHeader(HttpRequest request, String toSign) throws HttpException {
String signature = signString(toSign);
if (signatureWire.enabled())
signatureWire.input(Utils.toInputStream(signature));
@ -122,7 +119,7 @@ public class SharedKeyLiteAuthentication implements HttpRequestFilter {
public String signString(String toSign) {
String signature;
try {
signature = encryptionService.base64(encryptionService.hmacSha256(toSign, key));
signature = CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign), crypto.hmacSHA256(key)));
} catch (Exception e) {
throw new HttpException("error signing request", e);
}
@ -134,8 +131,7 @@ public class SharedKeyLiteAuthentication implements HttpRequestFilter {
}
private void replaceDateHeader(HttpRequest request) {
request.getHeaders().replaceValues(HttpHeaders.DATE,
Collections.singletonList(timeStampProvider.get()));
request.getHeaders().replaceValues(HttpHeaders.DATE, Collections.singletonList(timeStampProvider.get()));
}
private void appendCanonicalizedHeaders(HttpRequest request, StringBuilder toSign) {

64
azure/src/test/java/org/jclouds/azure/storage/blob/AzureBlobClientLiveTest.java
View File

@ -42,8 +42,10 @@ import org.jclouds.azure.storage.options.ListOptions;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpResponseException;
import org.jclouds.http.options.GetOptions;
import org.jclouds.io.Payloads;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.util.Utils;
import org.testng.annotations.BeforeTest;
@ -73,8 +75,8 @@ public class AzureBlobClientLiveTest {
public void setupClient() throws IOException {
identity = System.getProperty("jclouds.test.identity");
String credential = System.getProperty("jclouds.test.credential");
context = new BlobStoreContextFactory().createContext("azureblob", identity, credential,
ImmutableSet.<Module> of(new Log4JLoggingModule()));
context = new BlobStoreContextFactory().createContext("azureblob", identity, credential, ImmutableSet
.<Module> of(new Log4JLoggingModule()));
client = (AzureBlobClient) context.getProviderSpecificContext().getApi();
}
@ -97,8 +99,7 @@ public class AzureBlobClientLiveTest {
while (!created) {
privateContainer = containerPrefix + new SecureRandom().nextInt();
try {
created = client.createContainer(privateContainer, withMetadata(ImmutableMultimap.of(
"foo", "bar")));
created = client.createContainer(privateContainer, withMetadata(ImmutableMultimap.of("foo", "bar")));
} catch (UndeclaredThrowableException e) {
HttpResponseException htpe = (HttpResponseException) e.getCause().getCause();
if (htpe.getResponse().getStatusCode() == 409)
@ -111,8 +112,8 @@ public class AzureBlobClientLiveTest {
long containerCount = response.size();
assertTrue(containerCount >= 1);
ListBlobsResponse list = client.listBlobs(privateContainer);
assertEquals(list.getUrl(), URI.create(String.format("https://%s.blob.core.windows.net/%s",
identity, privateContainer)));
assertEquals(list.getUrl(), URI.create(String.format("https://%s.blob.core.windows.net/%s", identity,
privateContainer)));
// TODO .. check to see the container actually exists
}
@ -164,15 +165,14 @@ public class AzureBlobClientLiveTest {
}
}
ListBlobsResponse list = client.listBlobs();
assertEquals(list.getUrl(), URI.create(String.format(
"https://%s.blob.core.windows.net/%%24root", identity)));
assertEquals(list.getUrl(), URI.create(String.format("https://%s.blob.core.windows.net/%%24root", identity)));
}
@Test
public void testListContainersWithOptions() throws Exception {
BoundedSet<ContainerProperties> response = client.listContainers(ListOptions.Builder.prefix(
privateContainer).maxResults(1).includeMetadata());
BoundedSet<ContainerProperties> response = client.listContainers(ListOptions.Builder.prefix(privateContainer)
.maxResults(1).includeMetadata());
assert null != response;
long initialContainerCount = response.size();
assertTrue(initialContainerCount >= 0);
@ -186,8 +186,7 @@ public class AzureBlobClientLiveTest {
// TODO loop for up to 30 seconds checking if they are really gone
}
@Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testCreateContainer",
"testCreatePublicContainer" })
@Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testCreateContainer", "testCreatePublicContainer" })
public void testListOwnedContainers() throws Exception {
// Test default listing
@ -197,8 +196,7 @@ public class AzureBlobClientLiveTest {
// Test listing with options
response = client.listContainers(ListOptions.Builder.prefix(
privateContainer.substring(0, privateContainer.length() - 1)).maxResults(1)
.includeMetadata());
privateContainer.substring(0, privateContainer.length() - 1)).maxResults(1).includeMetadata());
assertEquals(response.size(), 1);
assertEquals(Iterables.getOnlyElement(response).getName(), privateContainer);
assertEquals(Iterables.getOnlyElement(response).getMetadata(), ImmutableMap.of("foo", "bar"));
@ -214,16 +212,14 @@ public class AzureBlobClientLiveTest {
client.deleteContainer("does-not-exist");
}
@Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testListOwnedContainers",
"testObjectOperations" })
@Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testListOwnedContainers", "testObjectOperations" })
public void testDeleteContainer() throws Exception {
client.deleteContainer(privateContainer);
client.deleteContainer(publicContainer);
// TODO loop for up to 30 seconds checking if they are really gone
}
@Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testCreateContainer",
"testCreatePublicContainer" })
@Test(timeOut = 5 * 60 * 1000, dependsOnMethods = { "testCreateContainer", "testCreatePublicContainer" })
public void testObjectOperations() throws Exception {
String data = "Here is my data";
@ -231,20 +227,18 @@ public class AzureBlobClientLiveTest {
AzureBlob object = client.newBlob();
object.getProperties().setName("object");
object.setPayload(data);
context.utils().encryption().generateMD5BufferingIfNotRepeatable(object);
Payloads.calculateMD5(object);
object.getProperties().setContentType("text/plain");
object.getProperties().getMetadata().put("mykey", "metadata-value");
byte[] md5 = object.getProperties().getContentMD5();
String newEtag = client.putBlob(privateContainer, object);
assertEquals(context.utils().encryption().hex(md5), context.utils().encryption().hex(object.getProperties()
.getContentMD5()));
assertEquals(CryptoStreams.hex(md5), CryptoStreams.hex(object.getProperties().getContentMD5()));
// Test HEAD of missing object
assert client.getBlobProperties(privateContainer, "non-existent-object") == null;
// Test HEAD of object
BlobProperties metadata = client.getBlobProperties(privateContainer, object.getProperties()
.getName());
BlobProperties metadata = client.getBlobProperties(privateContainer, object.getProperties().getName());
// TODO assertEquals(metadata.getName(), object.getProperties().getName());
// we can't check this while hacking around lack of content-md5, as GET of the first byte will
// show incorrect length 1, the returned size, as opposed to the real length. This is an ok
@ -254,8 +248,7 @@ public class AzureBlobClientLiveTest {
// assertEquals(metadata.getSize(), data.length());
assertEquals(metadata.getContentType(), "text/plain");
// Azure doesn't return the Content-MD5 on head request..
assertEquals(context.utils().encryption().hex(md5), context.utils().encryption().hex(object.getProperties()
.getContentMD5()));
assertEquals(CryptoStreams.hex(md5), CryptoStreams.hex(object.getProperties().getContentMD5()));
assertEquals(metadata.getETag(), newEtag);
assertEquals(metadata.getMetadata().entrySet().size(), 1);
assertEquals(metadata.getMetadata().get("mykey"), "metadata-value");
@ -276,8 +269,7 @@ public class AzureBlobClientLiveTest {
// TODO assertEquals(getBlob.getName(), object.getProperties().getName());
assertEquals(getBlob.getPayload().getContentLength(), new Long(data.length()));
assertEquals(getBlob.getProperties().getContentType(), "text/plain");
assertEquals(context.utils().encryption().hex(md5), context.utils().encryption().hex(getBlob.getProperties()
.getContentMD5()));
assertEquals(CryptoStreams.hex(md5), CryptoStreams.hex(getBlob.getProperties().getContentMD5()));
assertEquals(newEtag, getBlob.getProperties().getETag());
// wait until we can update metadata
// assertEquals(getBlob.getProperties().getMetadata().entries().size(), 2);
@ -291,15 +283,12 @@ public class AzureBlobClientLiveTest {
assertEquals(metadata.getMetadata().get("mykey"), "metadata-value");
// test listing
ListBlobsResponse response = client.listBlobs(privateContainer, ListBlobsOptions.Builder
.prefix(
object.getProperties().getName().substring(0,
object.getProperties().getName().length() - 1)).maxResults(1)
.includeMetadata());
ListBlobsResponse response = client.listBlobs(privateContainer, ListBlobsOptions.Builder.prefix(
object.getProperties().getName().substring(0, object.getProperties().getName().length() - 1))
.maxResults(1).includeMetadata());
assertEquals(response.size(), 1);
assertEquals(Iterables.getOnlyElement(response).getName(), object.getProperties().getName());
assertEquals(Iterables.getOnlyElement(response).getMetadata(), ImmutableMap.of("mykey",
"metadata-value"));
assertEquals(Iterables.getOnlyElement(response).getMetadata(), ImmutableMap.of("mykey", "metadata-value"));
// Test PUT with invalid ETag (as if object's data was corrupted in transit)
String correctEtag = newEtag;
@ -318,8 +307,7 @@ public class AzureBlobClientLiveTest {
object.setPayload(bais);
object.getPayload().setContentLength(new Long(data.getBytes().length));
newEtag = client.putBlob(privateContainer, object);
assertEquals(context.utils().encryption().hex(md5), context.utils().encryption().hex(getBlob.getProperties()
.getContentMD5()));
assertEquals(CryptoStreams.hex(md5), CryptoStreams.hex(getBlob.getProperties().getContentMD5()));
// Test GET with options
// Non-matching ETag
@ -333,8 +321,8 @@ public class AzureBlobClientLiveTest {
// Matching ETag TODO this shouldn't fail!!!
try {
getBlob = client.getBlob(privateContainer, object.getProperties().getName(),
GetOptions.Builder.ifETagMatches(newEtag));
getBlob = client.getBlob(privateContainer, object.getProperties().getName(), GetOptions.Builder
.ifETagMatches(newEtag));
assertEquals(getBlob.getProperties().getETag(), newEtag);
} catch (HttpResponseException e) {
assertEquals(e.getResponse().getStatusCode(), 412);

47
blobstore/src/main/clojure/org/jclouds/blobstore.clj
View File

@ -43,7 +43,7 @@ See http://code.google.com/p/jclouds for details."
AsyncBlobStore BlobStore BlobStoreContext BlobStoreContextFactory
domain.BlobMetadata domain.StorageMetadata domain.Blob
options.ListContainerOptions]
[org.jclouds.encryption.internal JCEEncryptionService]
[org.jclouds.io Payloads]
[java.util Arrays]
[java.security DigestOutputStream MessageDigest]
[com.google.common.collect ImmutableSet]))
@ -135,12 +135,12 @@ Options can also be specified for extension modules
(.list blobstore container-name list-options))
(apply list-container *blobstore* blobstore args)))
(defn- list-blobs-chunk [container prefix blobstore & [marker]]
(defn- list-blobs-chunk [container prefix #^BlobStore blobstore & [marker]]
(apply list-container blobstore container
:in-directory prefix (when (string? marker)
[:after-marker marker])))
(defn- list-blobs-chunks [container prefix blobstore marker]
(defn- list-blobs-chunks [container prefix #^BlobStore blobstore marker]
(when marker
(let [chunk (list-blobs-chunk container prefix blobstore marker)]
(lazy-seq (cons chunk
@ -149,7 +149,7 @@ Options can also be specified for extension modules
(defn list-blobs
"Returns a lazy seq of all blobs in the given container."
([container prefix blobstore]
([container prefix #^BlobStore blobstore]
(apply concat (list-blobs-chunks container prefix blobstore :start))))
(defn locations
@ -164,84 +164,84 @@ Options can also be specified for extension modules
(create-container container-name nil *blobstore*))
([container-name location]
(create-container container-name location *blobstore*))
([container-name location blobstore]
([container-name location #^BlobStore blobstore]
(.createContainerInLocation blobstore location container-name)))
(defn clear-container
"Clear a container."
([container-name]
(clear-container container-name *blobstore*))
([container-name blobstore]
([container-name #^BlobStore blobstore]
(.clearContainer blobstore container-name)))
(defn delete-container
"Delete a container."
([container-name]
(delete-container container-name *blobstore*))
([container-name blobstore]
([container-name #^BlobStore blobstore]
(.deleteContainer blobstore container-name)))
(defn container-exists?
"Predicate to check presence of a container"
([container-name]
(container-exists? container-name *blobstore*))
([container-name blobstore]
([container-name #^BlobStore blobstore]
(.containerExists blobstore container-name)))
(defn directory-exists?
"Predicate to check presence of a directory"
([container-name path]
(directory-exists? container-name path *blobstore*))
([container-name path blobstore]
([container-name path #^BlobStore blobstore]
(.directoryExists blobstore container-name path)))
(defn create-directory
"Create a directory path."
([container-name path]
(create-directory container-name path *blobstore*))
([container-name path blobstore]
([container-name path #^BlobStore blobstore]
(.createDirectory blobstore container-name path)))
(defn delete-directory
"Delete a directory path."
([container-name path]
(delete-directory container-name path *blobstore*))
([container-name path blobstore]
([container-name path #^BlobStore blobstore]
(.deleteDirectory blobstore container-name path)))
(defn blob-exists?
"Predicate to check presence of a blob"
([container-name path]
(blob-exists? container-name path *blobstore*))
([container-name path blobstore]
([container-name path #^BlobStore blobstore]
(.blobExists blobstore container-name path)))
(defn put-blob
"Put a blob. Metadata in the blob determines location."
([container-name blob]
(put-blob container-name blob *blobstore*))
([container-name blob blobstore]
([container-name blob #^BlobStore blobstore]
(.putBlob blobstore container-name blob)))
(defn blob-metadata
"Get metadata from given path"
([container-name path]
(blob-metadata container-name path *blobstore*))
([container-name path blobstore]
([container-name path #^BlobStore blobstore]
(.blobMetadata blobstore container-name path)))
(defn get-blob
"Get blob from given path"
([container-name path]
(get-blob container-name path *blobstore*))
([container-name path blobstore]
([container-name path #^BlobStore blobstore]
(.getBlob blobstore container-name path)))
(defn remove-blob
"Remove blob from given path"
([container-name path]
(remove-blob container-name path *blobstore*))
([container-name path blobstore]
([container-name path #^BlobStore blobstore]
(.removeBlob blobstore container-name path)))
(defn count-blobs
@ -281,23 +281,18 @@ example:
"add a content md5 to a blob, or make a new blob that has an md5.
note that this implies rebuffering, if the blob's payload isn't repeatable"
([#^Blob blob]
(md5-blob *blobstore*))
([blob-or-name blobstore-or-payload]
(if (blobstore? blobstore-or-payload)
(-> (blobstore-context blobstore-or-payload)
.utils
.encryption
(.generateMD5BufferingIfNotRepeatable blob-or-name))
(md5-blob blob-or-name blobstore-or-payload *blobstore*)))
(Payloads/calculateMD5 blob))
([#^String name payload]
(blob name payload *blobstore*))
([#^String name payload #^BlobStore blobstore]
(md5-blob (blob name payload blobstore) blobstore)))
(md5-blob (blob name payload blobstore))))
(defn upload-blob
"Create anrepresenting text data:
container, name, string -> etag"
([container-name name data]
(upload-blob container-name name data *blobstore*))
([container-name name data blobstore]
([container-name name data #^BlobStore blobstore]
(put-blob container-name
(md5-blob name data blobstore) blobstore)))

44
blobstore/src/main/java/org/jclouds/blobstore/TransientAsyncBlobStore.java
View File

@ -77,19 +77,22 @@ import org.jclouds.blobstore.options.GetOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.strategy.IfDirectoryReturnNameStrategy;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.DateService;
import org.jclouds.domain.Location;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpCommand;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.HttpResponseException;
import org.jclouds.http.options.HttpRequestOptions;
import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.ByteArrayPayload;
import org.jclouds.io.payloads.DelegatingPayload;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimaps;
import com.google.common.io.Closeables;
@ -106,7 +109,7 @@ import com.google.inject.internal.Nullable;
public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
protected final DateService dateService;
protected final EncryptionService encryptionService;
protected final Crypto crypto;
protected final ConcurrentMap<String, ConcurrentMap<String, Blob>> containerToBlobs;
protected final ConcurrentMap<String, Location> containerToLocation;
protected final HttpGetOptionsListToGetOptions httpGetOptionsConverter;
@ -114,8 +117,8 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
protected final Factory blobFactory;
@Inject
protected TransientAsyncBlobStore(BlobStoreContext context, DateService dateService,
EncryptionService encryptionService, ConcurrentMap<String, ConcurrentMap<String, Blob>> containerToBlobs,
protected TransientAsyncBlobStore(BlobStoreContext context, DateService dateService, Crypto crypto,
ConcurrentMap<String, ConcurrentMap<String, Blob>> containerToBlobs,
ConcurrentMap<String, Location> containerToLocation,
HttpGetOptionsListToGetOptions httpGetOptionsConverter,
IfDirectoryReturnNameStrategy ifDirectoryReturnName, Blob.Factory blobFactory, BlobUtils blobUtils,
@ -124,7 +127,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
super(context, blobUtils, service, defaultLocation, locations);
this.blobFactory = blobFactory;
this.dateService = dateService;
this.encryptionService = encryptionService;
this.crypto = crypto;
this.containerToBlobs = containerToBlobs;
this.containerToLocation = containerToLocation;
this.httpGetOptionsConverter = httpGetOptionsConverter;
@ -464,20 +467,23 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
payload = (object.getPayload() instanceof DelegatingPayload) ? (DelegatingPayload.class.cast(
object.getPayload()).getDelegate() instanceof ByteArrayPayload) ? ByteArrayPayload.class
.cast(DelegatingPayload.class.cast(object.getPayload()).getDelegate()) : null : null;
if (payload == null || !(payload instanceof ByteArrayPayload)) {
InputStream input = object.getPayload().getInput();
try {
String oldContentType = object.getPayload().getContentType();
payload = encryptionService.generatePayloadWithMD5For(input);
payload.setContentType(oldContentType);
} finally {
Closeables.closeQuietly(input);
try {
if (payload == null || !(payload instanceof ByteArrayPayload)) {
InputStream input = object.getPayload().getInput();
try {
String oldContentType = object.getPayload().getContentType();
payload = (ByteArrayPayload) Payloads.calculateMD5(Payloads.newPayload(object.getPayload().getInput()));
payload.setContentType(oldContentType);
} finally {
Closeables.closeQuietly(input);
}
} else {
if (payload.getContentMD5() == null)
Payloads.calculateMD5(object, crypto.md5());
}
} else {
if (payload.getContentMD5() == null)
payload = (ByteArrayPayload) encryptionService.generateMD5BufferingIfNotRepeatable(payload);
} catch (IOException e) {
Throwables.propagate(e);
}
Blob blob = blobFactory.create(copy(object.getMetadata()));
blob.setPayload(payload);
blob.getMetadata().setLastModified(new Date());
@ -485,7 +491,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
blob.getMetadata().setContentMD5(payload.getContentMD5());
blob.getMetadata().setContentType(payload.getContentType());
String eTag = encryptionService.hex(payload.getContentMD5());
String eTag = CryptoStreams.hex(payload.getContentMD5());
blob.getMetadata().setETag(eTag);
container.put(blob.getMetadata().getName(), blob);
@ -495,7 +501,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore {
blob.getAllHeaders().put(HttpHeaders.ETAG, eTag);
blob.getAllHeaders().put(HttpHeaders.CONTENT_TYPE, payload.getContentType());
blob.getAllHeaders().put(HttpHeaders.CONTENT_LENGTH, payload.getContentLength() + "");
blob.getAllHeaders().put("Content-MD5", encryptionService.base64(payload.getContentMD5()));
blob.getAllHeaders().put("Content-MD5", CryptoStreams.base64(payload.getContentMD5()));
blob.getAllHeaders().putAll(Multimaps.forMap(blob.getMetadata().getUserMetadata()));
return immediateFuture(eTag);

6
blobstore/src/main/java/org/jclouds/blobstore/config/BlobStoreMapModule.java
View File

@ -31,7 +31,7 @@ import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
import org.jclouds.blobstore.strategy.PutBlobsStrategy;
import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import com.google.inject.AbstractModule;
import com.google.inject.Scopes;
@ -84,14 +84,14 @@ public class BlobStoreMapModule extends AbstractModule {
@Inject
PutBlobsStrategy putBlobsStrategy;
@Inject
EncryptionService encryptionService;
Crypto crypto;
@Inject
ListContainerAndRecurseThroughFolders listStrategy;
public InputStreamMap create(String containerName, ListContainerOptions options) {
return new InputStreamMapImpl(connection, blobFactory, getAllBlobs, listStrategy,
containsValueStrategy, putBlobsStrategy, containerName, options,
encryptionService);
crypto);
}
}

2
blobstore/src/main/java/org/jclouds/blobstore/domain/Blob.java
View File

@ -18,7 +18,7 @@
*/
package org.jclouds.blobstore.domain;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.io.PayloadEnclosing;
import com.google.common.collect.Multimap;
import com.google.inject.internal.Nullable;

2
blobstore/src/main/java/org/jclouds/blobstore/domain/internal/BlobImpl.java
View File

@ -25,9 +25,9 @@ import javax.inject.Inject;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.MutableBlobMetadata;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.http.internal.PayloadEnclosingImpl;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadEnclosing;
import org.jclouds.io.payloads.DelegatingPayload;
import com.google.common.collect.LinkedHashMultimap;

17
blobstore/src/main/java/org/jclouds/blobstore/functions/ObjectMD5.java
View File

@ -18,23 +18,26 @@
*/
package org.jclouds.blobstore.functions;
import java.io.IOException;
import javax.inject.Inject;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.io.Payloads;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
public class ObjectMD5 implements Function<Object, byte[]> {
protected final Blob.Factory blobFactory;
protected final EncryptionService encryptionService;
protected final Crypto crypto;
@Inject
ObjectMD5(EncryptionService encryptionService, Blob.Factory blobFactory) {
ObjectMD5(Crypto crypto, Blob.Factory blobFactory) {
this.blobFactory = blobFactory;
this.encryptionService = encryptionService;
this.crypto = crypto;
}
public byte[] apply(Object from) {
@ -46,7 +49,11 @@ public class ObjectMD5 implements Function<Object, byte[]> {
object.setPayload(Payloads.newPayload(from));
}
if (object.getMetadata().getContentMD5() == null)
encryptionService.generateMD5BufferingIfNotRepeatable(object);
try {
Payloads.calculateMD5(object, crypto.md5());
} catch (IOException e) {
Throwables.propagate(e);
}
return object.getPayload().getContentMD5();
}

13
blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java
View File

@ -38,7 +38,6 @@ import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.blobstore.util.internal.BlobUtilsImpl;
import static org.jclouds.concurrent.ConcurrentUtils.*;
import org.jclouds.domain.Location;
import org.jclouds.util.Utils;
@ -114,7 +113,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore {
*/
@Override
public ListenableFuture<Long> countBlobs(final String containerName, final ListContainerOptions options) {
return makeListenable(service.submit(new Callable<Long>() {
return org.jclouds.concurrent.Futures.makeListenable(service.submit(new Callable<Long>() {
public Long call() throws Exception {
return blobUtils.countBlobs(containerName, options);
}
@ -142,7 +141,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore {
*/
@Override
public ListenableFuture<Void> clearContainer(final String containerName, final ListContainerOptions options) {
return makeListenable(service.submit(new Callable<Void>() {
return org.jclouds.concurrent.Futures.makeListenable(service.submit(new Callable<Void>() {
public Void call() throws Exception {
blobUtils.clearContainer(containerName, options);
@ -160,7 +159,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore {
*/
@Override
public ListenableFuture<Void> deleteDirectory(final String containerName, final String directory) {
return makeListenable(service.submit(new Callable<Void>() {
return org.jclouds.concurrent.Futures.makeListenable(service.submit(new Callable<Void>() {
public Void call() throws Exception {
blobUtils.deleteDirectory(containerName, directory);
@ -179,7 +178,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore {
* virtual path
*/
public ListenableFuture<Boolean> directoryExists(final String containerName, final String directory) {
return makeListenable(service.submit(new Callable<Boolean>() {
return org.jclouds.concurrent.Futures.makeListenable(service.submit(new Callable<Boolean>() {
public Boolean call() throws Exception {
return blobUtils.directoryExists(containerName, directory);
@ -200,7 +199,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore {
public ListenableFuture<Void> createDirectory(final String containerName, final String directory) {
return blobUtils.directoryExists(containerName, directory) ? Futures.immediateFuture((Void) null)
: makeListenable(service.submit(new Callable<Void>() {
: org.jclouds.concurrent.Futures.makeListenable(service.submit(new Callable<Void>() {
public Void call() throws Exception {
blobUtils.createDirectory(containerName, directory);
return null;
@ -230,7 +229,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore {
*/
@Override
public ListenableFuture<Void> deleteContainer(final String container) {
return makeListenable(service.submit(new Callable<Void>() {
return org.jclouds.concurrent.Futures.makeListenable(service.submit(new Callable<Void>() {
public Void call() throws Exception {
deleteAndEnsurePathGone(container);

17
blobstore/src/main/java/org/jclouds/blobstore/internal/InputStreamMapImpl.java
View File

@ -23,6 +23,7 @@ import static com.google.common.collect.Lists.newArrayList;
import static org.jclouds.io.Payloads.newPayload;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.Map;
@ -38,8 +39,9 @@ import org.jclouds.blobstore.strategy.ContainsValueInListStrategy;
import org.jclouds.blobstore.strategy.GetBlobsInListStrategy;
import org.jclouds.blobstore.strategy.PutBlobsStrategy;
import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.ByteArrayPayload;
import org.jclouds.io.payloads.FilePayload;
import org.jclouds.io.payloads.InputStreamPayload;
@ -47,6 +49,7 @@ import org.jclouds.io.payloads.StringPayload;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
/**
* Map representation of a live connection to a BlobStore. All put operations will result in ETag
@ -59,16 +62,16 @@ import com.google.common.base.Function;
* @see BaseBlobMap
*/
public class InputStreamMapImpl extends BaseBlobMap<InputStream> implements InputStreamMap {
protected final EncryptionService encryptionService;
protected final Crypto crypto;
@Inject
public InputStreamMapImpl(BlobStore connection, Blob.Factory blobFactory,
GetBlobsInListStrategy getAllBlobs, ListContainerAndRecurseThroughFolders listStrategy,
ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy,
String containerName, ListContainerOptions options, EncryptionService encryptionService) {
String containerName, ListContainerOptions options, Crypto crypto) {
super(connection, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy,
containerName, options);
this.encryptionService = encryptionService;
this.crypto = crypto;
}
@Override
@ -144,7 +147,11 @@ public class InputStreamMapImpl extends BaseBlobMap<InputStream> implements Inpu
Blob newBlobWithMD5(String name, Object value) {
Blob blob = blobstore.newBlob(prefixer.apply(name));
blob.setPayload(newPayload(value));
encryptionService.generateMD5BufferingIfNotRepeatable(blob);
try {
Payloads.calculateMD5(blob, crypto.md5());
} catch (IOException e) {
Throwables.propagate(e);
}
return blob;
}

2
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java
View File

@ -19,7 +19,7 @@
package org.jclouds.blobstore.strategy.internal;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import java.util.Map;
import java.util.concurrent.ExecutorService;

2
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FetchBlobMetadata.java
View File

@ -19,7 +19,7 @@
package org.jclouds.blobstore.strategy.internal;
import static com.google.common.base.Preconditions.checkState;
import static org.jclouds.concurrent.ConcurrentUtils.transformParallel;
import static org.jclouds.concurrent.FutureIterables.transformParallel;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

2
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java
View File

@ -18,7 +18,7 @@
*/
package org.jclouds.blobstore.strategy.internal;
import static org.jclouds.concurrent.ConcurrentUtils.transformParallel;
import static org.jclouds.concurrent.FutureIterables.transformParallel;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

2
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java
View File

@ -18,7 +18,7 @@
*/
package org.jclouds.blobstore.strategy.internal;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import java.util.Map;
import java.util.Set;

2
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/PutBlobsStrategyImpl.java
View File

@ -18,7 +18,7 @@
*/
package org.jclouds.blobstore.strategy.internal;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import java.util.Map;
import java.util.concurrent.ExecutorService;

43
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
View File

@ -24,8 +24,7 @@ import static org.jclouds.blobstore.options.GetOptions.Builder.ifModifiedSince;
import static org.jclouds.blobstore.options.GetOptions.Builder.ifUnmodifiedSince;
import static org.jclouds.blobstore.options.GetOptions.Builder.range;
import static org.jclouds.blobstore.util.BlobStoreUtils.getContentAsStringOrNullAndClose;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.ConcurrentUtils.compose;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
@ -51,11 +50,13 @@ import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.encryption.internal.JCEEncryptionService;
import org.jclouds.concurrent.Futures;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.encryption.internal.JCECrypto;
import org.jclouds.http.BaseJettyTest;
import org.jclouds.http.HttpResponseException;
import org.jclouds.io.Payload;
import org.jclouds.io.InputSuppliers;
import org.jclouds.io.Payloads;
import org.jclouds.logging.Logger;
import org.jclouds.util.Utils;
@ -76,16 +77,15 @@ import com.google.common.io.InputSupplier;
* @author Adrian Cole
*/
public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
private byte[] oneHundredOneConstitutions;
private InputSupplier<InputStream> oneHundredOneConstitutions;
private byte[] oneHundredOneConstitutionsMD5;
@BeforeClass(groups = { "integration", "live" })
@Override
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
Payload result = context.utils().encryption().generatePayloadWithMD5For(getTestDataSupplier().getInput());
oneHundredOneConstitutions = (byte[]) result.getRawContent();
oneHundredOneConstitutionsMD5 = result.getContentMD5();
oneHundredOneConstitutions = getTestDataSupplier();
oneHundredOneConstitutionsMD5 = CryptoStreams.md5(oneHundredOneConstitutions);
}
@SuppressWarnings("unchecked")
@ -112,13 +112,16 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
Map<Integer, Future<?>> responses = Maps.newHashMap();
for (int i = 0; i < 10; i++) {
responses.put(i, compose(context.getAsyncBlobStore().getBlob(containerName, key),
responses.put(i, Futures.compose(context.getAsyncBlobStore().getBlob(containerName, key),
new Function<Blob, Void>() {
@Override
public Void apply(Blob from) {
assertEquals(context.utils().encryption().md5(from.getPayload().getInput()),
oneHundredOneConstitutionsMD5);
try {
assertEquals(CryptoStreams.md5(from.getPayload()), oneHundredOneConstitutionsMD5);
} catch (IOException e) {
Throwables.propagate(e);
}
return null;
}
@ -138,7 +141,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
Blob sourceObject = context.getBlobStore().newBlob(key);
sourceObject.getMetadata().setContentType("text/plain");
sourceObject.getMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
sourceObject.setPayload(oneHundredOneConstitutions);
sourceObject.setPayload(oneHundredOneConstitutions.getInput());
context.getBlobStore().putBlob(containerName, sourceObject);
}
@ -409,7 +412,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
blob.getMetadata().setContentType(type);
blob.setPayload(Payloads.newPayload(content));
if (content instanceof InputStream) {
context.utils().encryption().generateMD5BufferingIfNotRepeatable(blob);
Payloads.calculateMD5(blob, context.utils().crypto().md5());
}
String containerName = getContainerName();
try {
@ -424,10 +427,10 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
}
}
protected volatile static EncryptionService encryptionService;
protected volatile static Crypto crypto;
static {
try {
encryptionService = new JCEEncryptionService();
crypto = new JCECrypto();
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
} catch (CertificateException e) {
@ -436,7 +439,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
}
@Test(groups = { "integration", "live" })
public void testMetadata() throws InterruptedException {
public void testMetadata() throws InterruptedException, IOException {
String key = "hello";
Blob blob = context.getBlobStore().newBlob(key);
@ -447,7 +450,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
// normalize the
// providers.
blob.getMetadata().getUserMetadata().put("Adrian", "powderpuff");
blob.getMetadata().setContentMD5(encryptionService.md5(Utils.toInputStream(TEST_STRING)));
Payloads.calculateMD5(blob, context.utils().crypto().md5());
String containerName = getContainerName();
try {
assertNull(context.getBlobStore().blobMetadata(containerName, "powderpuff"));
@ -473,11 +476,11 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
}
}
protected void validateMetadata(BlobMetadata metadata) {
protected void validateMetadata(BlobMetadata metadata) throws IOException {
assert metadata.getContentType().startsWith("text/plain") : metadata.getContentType();
assertEquals(metadata.getSize(), new Long(TEST_STRING.length()));
assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff");
assertEquals(metadata.getContentMD5(), encryptionService.md5(Utils.toInputStream(TEST_STRING)));
assertEquals(metadata.getContentMD5(), CryptoStreams.md5(InputSuppliers.of(TEST_STRING)));
}
}

19
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobLiveTest.java
View File

@ -26,7 +26,7 @@ import java.net.URL;
import java.net.URLConnection;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.encryption.internal.JCEEncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
@ -41,25 +41,20 @@ import org.testng.annotations.Test;
@Test(groups = { "live" }, testName = "blobstore.BlobLiveTest")
public class BaseBlobLiveTest extends BaseBlobStoreIntegrationTest {
private static final String sysHttpStreamUrl = System
.getProperty("jclouds.blobstore.httpstream.url");
private static final String sysHttpStreamETag = System
.getProperty("jclouds.blobstore.httpstream.md5");
private static final String sysHttpStreamUrl = System.getProperty("jclouds.blobstore.httpstream.url");
private static final String sysHttpStreamETag = System.getProperty("jclouds.blobstore.httpstream.md5");
@Test
@Parameters( { "jclouds.blobstore.httpstream.url", "jclouds.blobstore.httpstream.md5" })
public void testCopyUrl(@Optional String httpStreamUrl, @Optional String httpStreamETag)
throws Exception {
httpStreamUrl = checkNotNull(httpStreamUrl != null ? httpStreamUrl : sysHttpStreamUrl,
"httpStreamUrl");
public void testCopyUrl(@Optional String httpStreamUrl, @Optional String httpStreamETag) throws Exception {
httpStreamUrl = checkNotNull(httpStreamUrl != null ? httpStreamUrl : sysHttpStreamUrl, "httpStreamUrl");
httpStreamETag = checkNotNull(httpStreamETag != null ? httpStreamETag : sysHttpStreamETag,
"httpStreamMd5");
httpStreamETag = checkNotNull(httpStreamETag != null ? httpStreamETag : sysHttpStreamETag, "httpStreamMd5");
String key = "hello";
URL url = new URL(httpStreamUrl);
byte[] md5 = new JCEEncryptionService().fromHex(httpStreamETag);
byte[] md5 = CryptoStreams.hex(httpStreamETag);
URLConnection connection = url.openConnection();
long length = connection.getContentLength();

17
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobMapIntegrationTest.java
View File

@ -36,6 +36,7 @@ import org.jclouds.blobstore.BlobMap;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.io.Payloads;
import org.jclouds.util.Utils;
import org.testng.annotations.Test;
@ -74,7 +75,7 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
}
@Test(groups = { "integration", "live" })
public void testRemove() throws InterruptedException, ExecutionException, TimeoutException {
public void testRemove() throws InterruptedException, ExecutionException, TimeoutException, IOException {
String bucketName = getContainerName();
try {
Map<String, Blob> map = createMap(context, bucketName);
@ -118,7 +119,7 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
.getValue()));
Blob blob = entry.getValue();
blob.setPayload("");
context.utils().encryption().generateMD5BufferingIfNotRepeatable(blob);
Payloads.calculateMD5(blob);
entry.setValue(blob);
}
assertConsistencyAware(new Runnable() {
@ -139,14 +140,14 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
}
@Test(groups = { "integration", "live" })
public void testContains() throws InterruptedException, ExecutionException, TimeoutException {
public void testContains() throws InterruptedException, ExecutionException, TimeoutException, IOException {
String bucketName = getContainerName();
try {
Map<String, Blob> map = createMap(context, bucketName);
putStringWithMD5(map, "one", "apple");
Blob blob = context.getBlobStore().newBlob("one");
blob.setPayload("apple");
context.utils().encryption().generateMD5BufferingIfNotRepeatable(blob);
Payloads.calculateMD5(blob);
assertConsistencyAwareContainsValue(map, blob);
} finally {
returnContainer(bucketName);
@ -174,11 +175,11 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
Map<String, Blob> map = createMap(context, bucketName);
Blob blob = context.getBlobStore().newBlob("one");
blob.setPayload(Utils.toInputStream("apple"));
context.utils().encryption().generateMD5BufferingIfNotRepeatable(blob);
Payloads.calculateMD5(blob);
Blob old = map.put(blob.getMetadata().getName(), blob);
getOneReturnsAppleAndOldValueIsNull(map, old);
blob.setPayload(Utils.toInputStream("bear"));
context.utils().encryption().generateMD5BufferingIfNotRepeatable(blob);
Payloads.calculateMD5(blob);
Blob apple = map.put(blob.getMetadata().getName(), blob);
getOneReturnsBearAndOldValueIsApple(map, apple);
} finally {
@ -239,10 +240,10 @@ public abstract class BaseBlobMapIntegrationTest extends BaseMapIntegrationTest<
}
@Override
protected void putStringWithMD5(Map<String, Blob> map, String key, String text) {
protected void putStringWithMD5(Map<String, Blob> map, String key, String text) throws IOException {
Blob blob = context.getBlobStore().newBlob(key);
blob.setPayload(text);
context.utils().encryption().generateMD5BufferingIfNotRepeatable(blob);
Payloads.calculateMD5(blob);
map.put(key, blob);
}

11
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java
View File

@ -23,9 +23,9 @@ import static com.google.common.collect.Iterables.get;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.afterMarker;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.inDirectory;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults;
import static org.jclouds.util.Utils.toInputStream;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Set;
import java.util.concurrent.ExecutionException;
@ -37,6 +37,9 @@ import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.io.InputSuppliers;
import org.jclouds.io.Payloads;
import org.testng.annotations.Test;
/**
@ -68,7 +71,7 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
}
@Test(groups = { "integration", "live" })
public void testWithDetails() throws InterruptedException {
public void testWithDetails() throws InterruptedException, IOException {
String key = "hello";
Blob object = context.getBlobStore().newBlob(key);
@ -79,7 +82,7 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
// normalize the
// providers.
object.getMetadata().getUserMetadata().put("Adrian", "powderpuff");
object.getMetadata().setContentMD5(context.utils().encryption().md5(toInputStream(TEST_STRING)));
Payloads.calculateMD5(object, context.utils().crypto().md5());
String containerName = getContainerName();
try {
addBlobToContainer(containerName, object);
@ -93,7 +96,7 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
assert metadata.getContentType().startsWith("text/plain") : metadata.getContentType();
assertEquals(metadata.getSize(), new Long(TEST_STRING.length()));
assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff");
assertEquals(metadata.getContentMD5(), context.utils().encryption().md5(toInputStream(TEST_STRING)));
assertEquals(metadata.getContentMD5(), CryptoStreams.md5(InputSuppliers.of(TEST_STRING)));
} finally {
returnContainer(containerName);
}

10
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java
View File

@ -104,7 +104,7 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
ListContainerOptions options);
@Test(groups = { "integration", "live" })
public void testClear() throws InterruptedException, ExecutionException, TimeoutException {
public void testClear() throws InterruptedException, ExecutionException, TimeoutException, IOException {
String containerNameName = getContainerName();
try {
Map<String, V> map = createMap(context, containerNameName);
@ -122,7 +122,7 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
public abstract void testRemove() throws IOException, InterruptedException, ExecutionException, TimeoutException;
@Test(groups = { "integration", "live" })
public void testKeySet() throws InterruptedException, ExecutionException, TimeoutException {
public void testKeySet() throws InterruptedException, ExecutionException, TimeoutException, IOException {
String containerNameName = getContainerName();
try {
Map<String, V> map = createMap(context, containerNameName);
@ -239,7 +239,7 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
}
@Test(groups = { "integration", "live" })
public void testContainsKey() throws InterruptedException, ExecutionException, TimeoutException {
public void testContainsKey() throws InterruptedException, ExecutionException, TimeoutException, IOException {
String containerNameName = getContainerName();
try {
Map<String, V> map = createMap(context, containerNameName);
@ -281,7 +281,7 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
}
@Test(groups = { "integration", "live" })
public void testIsEmpty() throws InterruptedException, ExecutionException, TimeoutException {
public void testIsEmpty() throws InterruptedException, ExecutionException, TimeoutException, IOException {
String containerNameName = getContainerName();
try {
Map<String, V> map = createMap(context, containerNameName);
@ -310,7 +310,7 @@ public abstract class BaseMapIntegrationTest<V> extends BaseBlobStoreIntegration
}
abstract protected void putStringWithMD5(Map<String, V> map, String key, String value) throws InterruptedException,
ExecutionException, TimeoutException;
ExecutionException, TimeoutException, IOException;
protected void fourLeftRemovingOne(Map<String, V> map) throws InterruptedException {
map.remove("one");

14
chef/core/src/main/java/org/jclouds/chef/binders/BindChecksumsToJsonPayload.java
View File

@ -29,11 +29,10 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.core.MediaType;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.binders.BindToStringPayload;
@ -46,17 +45,10 @@ import com.google.common.primitives.Bytes;
*/
@Singleton
public class BindChecksumsToJsonPayload extends BindToStringPayload {
private final EncryptionService encryptionService;
@Inject
BindChecksumsToJsonPayload(EncryptionService encryptionService) {
this.encryptionService = encryptionService;
}
@SuppressWarnings("unchecked")
public void bindToRequest(HttpRequest request, Object input) {
checkArgument(checkNotNull(input, "input") instanceof Set,
"this binder is only valid for Set!");
checkArgument(checkNotNull(input, "input") instanceof Set, "this binder is only valid for Set!");
Set<List<Byte>> md5s = (Set<List<Byte>>) input;
@ -64,7 +56,7 @@ public class BindChecksumsToJsonPayload extends BindToStringPayload {
builder.append("{\"checksums\":{");
for (List<Byte> md5 : md5s)
builder.append(String.format("\"%s\":null,", encryptionService.hex(Bytes.toArray(md5))));
builder.append(String.format("\"%s\":null,", CryptoStreams.hex(Bytes.toArray(md5))));
builder.deleteCharAt(builder.length() - 1);
builder.append("}}");
super.bindToRequest(request, builder.toString());

15
chef/core/src/main/java/org/jclouds/chef/config/BaseChefRestClientModule.java
View File

@ -21,8 +21,9 @@ package org.jclouds.chef.config;
import static org.jclouds.Constants.PROPERTY_CREDENTIAL;
import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL;
import java.io.UnsupportedEncodingException;
import java.io.IOException;
import java.security.PrivateKey;
import java.security.spec.InvalidKeySpecException;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -31,15 +32,17 @@ import javax.inject.Singleton;
import org.jclouds.chef.handlers.ChefClientErrorRetryHandler;
import org.jclouds.chef.handlers.ChefErrorHandler;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.Pems;
import org.jclouds.date.DateService;
import org.jclouds.date.TimeStamp;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpErrorHandler;
import org.jclouds.http.HttpRetryHandler;
import org.jclouds.http.RequiresHttp;
import org.jclouds.http.annotation.ClientError;
import org.jclouds.http.annotation.Redirection;
import org.jclouds.http.annotation.ServerError;
import org.jclouds.io.InputSuppliers;
import org.jclouds.rest.ConfiguresRestClient;
import org.jclouds.rest.config.RestClientModule;
@ -61,7 +64,7 @@ public class BaseChefRestClientModule<S, A> extends RestClientModule<S, A> {
}
protected BaseChefRestClientModule(Class<S> syncClientType, Class<A> asyncClientType,
Map<Class<?>, Class<?>> delegates) {
Map<Class<?>, Class<?>> delegates) {
super(syncClientType, asyncClientType, delegates);
}
@ -86,9 +89,9 @@ public class BaseChefRestClientModule<S, A> extends RestClientModule<S, A> {
@Provides
@Singleton
public PrivateKey provideKey(EncryptionService encryptionService, @Named(PROPERTY_CREDENTIAL) String pem)
throws UnsupportedEncodingException {
return encryptionService.privateKeyFromPEM(pem.getBytes("UTF-8"));
public PrivateKey provideKey(Crypto crypto, @Named(PROPERTY_CREDENTIAL) String pem) throws InvalidKeySpecException,
IOException {
return crypto.rsaKeyFactory().generatePrivate(Pems.privateKeySpec(InputSuppliers.of(pem)));
}
@Override

49
chef/core/src/main/java/org/jclouds/chef/config/ChefParserModule.java
View File

@ -18,11 +18,14 @@
*/
package org.jclouds.chef.config;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Type;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.security.spec.InvalidKeySpecException;
import java.util.Map;
import javax.inject.Inject;
@ -31,7 +34,9 @@ import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.chef.domain.DataBagItem;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.Pems;
import org.jclouds.io.InputSuppliers;
import org.jclouds.json.config.GsonModule.DateAdapter;
import org.jclouds.json.config.GsonModule.Iso8601DateAdapter;
@ -61,11 +66,11 @@ public class ChefParserModule extends AbstractModule {
@Singleton
public static class PrivateKeyAdapterImpl implements PrivateKeyAdapter {
private final EncryptionService encryptionService;
private final Crypto crypto;
@Inject
PrivateKeyAdapterImpl(EncryptionService encryptionService) {
this.encryptionService = encryptionService;
PrivateKeyAdapterImpl(Crypto crypto) {
this.crypto = crypto;
}
@Override
@ -73,10 +78,16 @@ public class ChefParserModule extends AbstractModule {
throws JsonParseException {
String keyText = json.getAsString().replaceAll("\\n", "\n");
try {
return encryptionService.privateKeyFromPEM(keyText.getBytes("UTF-8"));
return crypto.rsaKeyFactory().generatePrivate(Pems.privateKeySpec(InputSuppliers.of(keyText)));
} catch (UnsupportedEncodingException e) {
Throwables.propagate(e);
return null;
} catch (InvalidKeySpecException e) {
Throwables.propagate(e);
return null;
} catch (IOException e) {
Throwables.propagate(e);
return null;
}
}
}
@ -88,11 +99,11 @@ public class ChefParserModule extends AbstractModule {
@Singleton
public static class PublicKeyAdapterImpl implements PublicKeyAdapter {
private final EncryptionService encryptionService;
private final Crypto crypto;
@Inject
PublicKeyAdapterImpl(EncryptionService encryptionService) {
this.encryptionService = encryptionService;
PublicKeyAdapterImpl(Crypto crypto) {
this.crypto = crypto;
}
@Override
@ -100,10 +111,16 @@ public class ChefParserModule extends AbstractModule {
throws JsonParseException {
String keyText = json.getAsString().replaceAll("\\n", "\n");
try {
return encryptionService.publicKeyFromPEM(keyText.getBytes("UTF-8"));
return crypto.rsaKeyFactory().generatePublic(Pems.publicKeySpec(InputSuppliers.of(keyText)));
} catch (UnsupportedEncodingException e) {
Throwables.propagate(e);
return null;
} catch (InvalidKeySpecException e) {
Throwables.propagate(e);
return null;
} catch (IOException e) {
Throwables.propagate(e);
return null;
}
}
}
@ -115,11 +132,11 @@ public class ChefParserModule extends AbstractModule {
@Singleton
public static class X509CertificateAdapterImpl implements X509CertificateAdapter {
private final EncryptionService encryptionService;
private final Crypto crypto;
@Inject
X509CertificateAdapterImpl(EncryptionService encryptionService) {
this.encryptionService = encryptionService;
X509CertificateAdapterImpl(Crypto crypto) {
this.crypto = crypto;
}
@Override
@ -127,10 +144,16 @@ public class ChefParserModule extends AbstractModule {
throws JsonParseException {
String keyText = json.getAsString().replaceAll("\\n", "\n");
try {
return encryptionService.x509CertificateFromPEM(keyText.getBytes("UTF-8"));
return Pems.x509Certificate(InputSuppliers.of(keyText), crypto.certFactory());
} catch (UnsupportedEncodingException e) {
Throwables.propagate(e);
return null;
} catch (IOException e) {
Throwables.propagate(e);
return null;
} catch (CertificateException e) {
Throwables.propagate(e);
return null;
}
}
}

21
chef/core/src/main/java/org/jclouds/chef/filters/SignedHeaderAuth.java
View File

@ -37,17 +37,20 @@ import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.TimeStamp;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.io.InputSuppliers;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.MultipartForm;
import org.jclouds.io.payloads.Part;
import org.jclouds.io.payloads.RSAEncryptingPayload;
import org.jclouds.logging.Logger;
import org.jclouds.util.Utils;
@ -56,6 +59,7 @@ import com.google.common.base.Predicate;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
/**
* Ported from mixlib-authentication in order to sign Chef requests.
@ -72,7 +76,7 @@ public class SignedHeaderAuth implements HttpRequestFilter {
private final String userId;
private final PrivateKey privateKey;
private final Provider<String> timeStampProvider;
private final EncryptionService encryptionService;
private final Crypto crypto;
private final String emptyStringHash;
private final HttpUtils utils;
@ -82,12 +86,12 @@ public class SignedHeaderAuth implements HttpRequestFilter {
@Inject
public SignedHeaderAuth(SignatureWire signatureWire, @Named(PROPERTY_IDENTITY) String userId, PrivateKey privateKey,
@TimeStamp Provider<String> timeStampProvider, EncryptionService encryptionService, HttpUtils utils) {
@TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
this.signatureWire = signatureWire;
this.userId = userId;
this.privateKey = privateKey;
this.timeStampProvider = timeStampProvider;
this.encryptionService = encryptionService;
this.crypto = crypto;
this.emptyStringHash = hashBody(Payloads.newStringPayload(""));
this.utils = utils;
}
@ -129,7 +133,7 @@ public class SignedHeaderAuth implements HttpRequestFilter {
@VisibleForTesting
String hashPath(String path) {
try {
return encryptionService.base64(encryptionService.sha1(Utils.toInputStream(canonicalPath(path))));
return CryptoStreams.base64(CryptoStreams.digest(InputSuppliers.of(canonicalPath(path)), crypto.sha1()));
} catch (Exception e) {
Throwables.propagateIfPossible(e);
throw new HttpException("error creating sigature for path: " + path, e);
@ -154,7 +158,7 @@ public class SignedHeaderAuth implements HttpRequestFilter {
checkArgument(payload != null, "payload was null");
checkArgument(payload.isRepeatable(), "payload must be repeatable: " + payload);
try {
return encryptionService.base64(encryptionService.sha1(payload.getInput()));
return CryptoStreams.base64(CryptoStreams.digest(payload, crypto.sha1()));
} catch (Exception e) {
Throwables.propagateIfPossible(e);
throw new HttpException("error creating sigature for payload: " + payload, e);
@ -182,8 +186,9 @@ public class SignedHeaderAuth implements HttpRequestFilter {
public String sign(String toSign) {
try {
byte[] encrypted = encryptionService.rsaEncrypt(Payloads.newStringPayload(toSign), privateKey);
return encryptionService.base64(encrypted);
byte[] encrypted = ByteStreams.toByteArray(new RSAEncryptingPayload(Payloads.newStringPayload(toSign),
privateKey));
return CryptoStreams.base64(encrypted);
} catch (Exception e) {
throw new HttpException("error signing request", e);
}

2
chef/core/src/main/java/org/jclouds/chef/strategy/internal/DeleteAllClientsAndNodesInListImpl.java
View File

@ -19,7 +19,7 @@
package org.jclouds.chef.strategy.internal;
import static com.google.common.collect.Maps.newHashMap;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import java.util.Map;
import java.util.concurrent.ExecutorService;

2
chef/core/src/main/java/org/jclouds/chef/strategy/internal/GetNodesImpl.java
View File

@ -19,7 +19,7 @@
package org.jclouds.chef.strategy.internal;
import static com.google.common.collect.Iterables.filter;
import static org.jclouds.concurrent.ConcurrentUtils.transformParallel;
import static org.jclouds.concurrent.FutureIterables.transformParallel;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

12
chef/core/src/main/java/org/jclouds/ohai/functions/ByteArrayToMacAddress.java
View File

@ -18,7 +18,6 @@
*/
package org.jclouds.ohai.functions;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.collect.Lists.partition;
import static com.google.common.primitives.Bytes.asList;
@ -26,10 +25,9 @@ import static com.google.common.primitives.Bytes.toArray;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
@ -42,12 +40,6 @@ import com.google.common.base.Joiner;
*/
@Singleton
public class ByteArrayToMacAddress implements Function<byte[], String> {
private final EncryptionService encryptionService;
@Inject
ByteArrayToMacAddress(EncryptionService encryptionService) {
this.encryptionService = checkNotNull(encryptionService, "encryptionService");
}
@Override
public String apply(byte[] from) {
@ -55,7 +47,7 @@ public class ByteArrayToMacAddress implements Function<byte[], String> {
@Override
public String apply(List<Byte> from) {
return encryptionService.hex(toArray(from));
return CryptoStreams.hex(toArray(from));
}
}));

9
chef/core/src/test/java/org/jclouds/chef/ChefAsyncClientTest.java
View File

@ -37,8 +37,8 @@ import org.jclouds.chef.domain.Role;
import org.jclouds.chef.filters.SignedHeaderAuth;
import org.jclouds.chef.filters.SignedHeaderAuthTest;
import org.jclouds.chef.functions.ParseKeySetFromJson;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.TimeStamp;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.RequiresHttp;
import org.jclouds.http.functions.ParseJson;
@ -88,12 +88,11 @@ public class ChefAsyncClientTest extends RestClientTest<ChefAsyncClient> {
}
public void testGetUploadSandboxForChecksums() throws SecurityException, NoSuchMethodException, IOException {
EncryptionService encryptionService = injector.getInstance(EncryptionService.class);
Method method = ChefAsyncClient.class.getMethod("getUploadSandboxForChecksums", Set.class);
GeneratedHttpRequest<ChefAsyncClient> httpRequest = processor.createRequest(method, ImmutableSet.of(Bytes
.asList(encryptionService.fromHex("0189e76ccc476701d6b374e5a1a27347")), Bytes.asList(encryptionService
.fromHex("0c5ecd7788cf4f6c7de2a57193897a6c")), Bytes.asList(encryptionService
.fromHex("1dda05ed139664f1f89b9dec482b77c0"))));
.asList(CryptoStreams.hex("0189e76ccc476701d6b374e5a1a27347")), Bytes.asList(CryptoStreams
.hex("0c5ecd7788cf4f6c7de2a57193897a6c")), Bytes.asList(CryptoStreams
.hex("1dda05ed139664f1f89b9dec482b77c0"))));
assertRequestLineEquals(httpRequest, "POST http://localhost:4000/sandboxes HTTP/1.1");
assertNonPayloadHeadersEqual(httpRequest, "Accept: application/json\nX-Chef-Version: 0.9.8\n");
assertPayloadEquals(

13
chef/core/src/test/java/org/jclouds/chef/ChefClientLiveTest.java
View File

@ -41,6 +41,9 @@ import org.jclouds.chef.domain.Node;
import org.jclouds.chef.domain.Resource;
import org.jclouds.chef.domain.Role;
import org.jclouds.chef.domain.UploadSandbox;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.crypto.Pems;
import org.jclouds.io.InputSuppliers;
import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.FilePayload;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
@ -108,7 +111,7 @@ public class ChefClientLiveTest {
content.setContentType("application/x-binary");
// get an md5 so that you can see if the server already has it or not
adminConnection.utils().encryption().generateMD5BufferingIfNotRepeatable(content);
Payloads.calculateMD5(content);
// Note that java collections cannot effectively do equals or hashcodes on
// byte arrays,
@ -142,8 +145,7 @@ public class ChefClientLiveTest {
@Test(dependsOnMethods = "testCreateClient")
public void testGenerateKeyForClient() throws Exception {
clientKey = validatorConnection.utils().encryption().toPem(
validatorConnection.getApi().generateKeyForClient(PREFIX).getPrivateKey());
clientKey = Pems.pem(validatorConnection.getApi().generateKeyForClient(PREFIX).getPrivateKey());
assertNotNull(clientKey);
clientConnection.close();
@ -163,7 +165,7 @@ public class ChefClientLiveTest {
cookbookO.getTemplates()).build()) {
try {
InputStream stream = adminConnection.utils().http().get(resource.getUrl());
byte[] md5 = adminConnection.utils().encryption().md5(stream);
byte[] md5 = CryptoStreams.md5(InputSuppliers.of(stream));
assertEquals(md5, resource.getChecksum());
} catch (NullPointerException e) {
assert false : "resource not found: " + resource;
@ -205,8 +207,7 @@ public class ChefClientLiveTest {
public void testCreateClient() throws Exception {
validatorConnection.getApi().deleteClient(PREFIX);
clientKey = validatorConnection.utils().encryption().toPem(
validatorConnection.getApi().createClient(PREFIX).getPrivateKey());
clientKey = Pems.pem(validatorConnection.getApi().createClient(PREFIX).getPrivateKey());
System.out.println(clientKey);
assertNotNull(clientKey);

5
chef/core/src/test/java/org/jclouds/chef/binders/BindHexEncodedMD5sToJsonPayloadTest.java
View File

@ -31,7 +31,7 @@ import java.net.URI;
import javax.ws.rs.HttpMethod;
import org.jclouds.chef.config.ChefParserModule;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpRequest;
import org.jclouds.json.config.GsonModule;
import org.testng.annotations.Test;
@ -58,8 +58,7 @@ public class BindHexEncodedMD5sToJsonPayloadTest {
@Test(enabled = false)
public void testCorrect() {
HttpRequest request = new HttpRequest(HttpMethod.POST, URI.create("http://localhost"));
binder.bindToRequest(request, ImmutableSet.of(injector.getInstance(EncryptionService.class).fromHex("abddef"),
injector.getInstance(EncryptionService.class).fromHex("1234")));
binder.bindToRequest(request, ImmutableSet.of(CryptoStreams.hex("abddef"), CryptoStreams.hex("1234")));
assertEquals(request.getPayload().getRawContent(), "{\"checksums\":{\"abddef\":null,\"1234\":null}}");
}

8
chef/core/src/test/java/org/jclouds/chef/filters/SignedHeaderAuthTest.java
View File

@ -34,7 +34,7 @@ import javax.inject.Provider;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
@ -173,7 +173,7 @@ public class SignedHeaderAuthTest {
}
private SignedHeaderAuth signing_obj;
private EncryptionService encryptionService;
private Crypto crypto;
/**
* before class, as we need to ensure that the filter is threadsafe.
@ -187,7 +187,7 @@ public class SignedHeaderAuthTest {
Injector injector = new RestContextFactory().createContextBuilder("chef", USER_ID, PRIVATE_KEY,
ImmutableSet.<Module> of(new MockModule(), new NullLoggingModule()), new Properties()).buildInjector();
encryptionService = injector.getInstance(EncryptionService.class);
crypto = injector.getInstance(Crypto.class);
HttpUtils utils = injector.getInstance(HttpUtils.class);
PrivateKey privateKey = injector.getInstance(PrivateKey.class);
@ -199,7 +199,7 @@ public class SignedHeaderAuthTest {
return TIMESTAMP_ISO8601;
}
}, encryptionService, utils);
}, crypto, utils);
}
}

28
chef/core/src/test/java/org/jclouds/chef/functions/ParseClientFromJsonTest.java
View File

@ -5,18 +5,24 @@ import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.security.PrivateKey;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.security.spec.InvalidKeySpecException;
import org.jclouds.chef.config.ChefParserModule;
import org.jclouds.chef.domain.Client;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.Pems;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ParseJson;
import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.RSADecryptingPayload;
import org.jclouds.io.payloads.RSAEncryptingPayload;
import org.jclouds.json.config.GsonModule;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.io.ByteStreams;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
@ -33,30 +39,30 @@ public class ParseClientFromJsonTest {
private static final String PRIVATE_KEY = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEAyb2ZJJqGm0KKR+8nfQJNsSd+F9tXNMV7CfOcW6jsqs8EZgiV\nR09hD1IYOj4YqM0qJONlgyg4xRWewdSG7QTPj1lJpVAida9sXy2+kzyagZA1Am0O\nZcbqb5hoeIDgcX+eDa79s0u0DomjcfO9EKhvHLBz+zM+3QqPRkPV8nYTbfs+HjVz\nzOU6D1B0XR3+IPZZl2AnWs2d0qhnStHcDUvnRVQ0P482YwN9VgceOZtpPz0DCKEJ\n5Tx5STub8k0/zt/VAMHQafLSuQMLd2s4ZLuOZptN//uAsTmxireqd37z+8ZTdBbJ\n8LEpJ+iCXuSfm5aUh7iw6oxvToY2AL53+jK2UQIDAQABAoIBAQDA88B3i/xWn0vX\nBVxFamCYoecuNjGwXXkSyZew616A+EOCu47bh4aTurdFbYL0YFaAtaWvzlaN2eHg\nDb+HDuTefE29+WkcGk6SshPmiz5T0XOCAICWw6wSVDkHmGwS4jZvbAFm7W8nwGk9\nYhxgxFiRngswJZFopOLoF5WXs2td8guIYNslMpo7tu50iFnBHwKO2ZsPAk8t9nnS\nxlDavKruymEmqHCr3+dtio5eaenJcp3fjoXBQOKUk3ipII29XRB8NqeCVV/7Kxwq\nckqOBEbRwBclckyIbD+RiAgKvOelORjEiE9R42vuqvxRA6k9kd9o7utlX0AUtpEn\n3gZc6LepAoGBAP9ael5Y75+sK2JJUNOOhO8ae45cdsilp2yI0X+UBaSuQs2+dyPp\nkpEHAxd4pmmSvn/8c9TlEZhr+qYbABXVPlDncxpIuw2Ajbk7s/S4XaSKsRqpXL57\nzj/QOqLkRk8+OVV9q6lMeQNqLtEj1u6JPviX70Ro+FQtRttNOYbfdP/fAoGBAMpA\nXjR5woV5sUb+REg9vEuYo8RSyOarxqKFCIXVUNsLOx+22+AK4+CQpbueWN7jotrl\nYD6uT6svWi3AAC7kiY0UI/fjVPRCUi8tVoQUE0TaU5VLITaYOB+W/bBaDE4M9560\n1NuDWO90baA5dfU44iuzva02rGJXK9+nS3o8nk/PAoGBALOL6djnDe4mwAaG6Jco\ncd4xr8jkyPzCRZuyBCSBbwphIUXLc7hDprPky064ncJD1UDmwIdkXd/fpMkg2QmA\n/CUk6LEFjMisqHojOaCL9gQZJPhLN5QUN2x1PJWGjs1vQh8Tkx0iUUCOa8bQPXNR\n+34OTsW6TUna4CSZAycLfhffAoGBAIggVsefBCvuQkF0NeUhmDCRZfhnd8y55RHR\n1HCvqKIlpv+rhcX/zmyBLuteopYyRJRsOiE2FW00i8+rIPRu4Z3Q5nybx7w3PzV9\noHN5R5baE9OyI4KpZWztpYYitZF67NcnAvVULHHOvVJQGnKYfLHJYmrJF7GA1ojM\nAuMdFbjFAoGAPxUhxwFy8gaqBahKUEZn4F81HFP5ihGhkT4QL6AFPO2e+JhIGjuR\n27+85hcFqQ+HHVtFsm81b/a+R7P4UuCRgc8eCjxQMoJ1Xl4n7VbjPbHMnIN0Ryvd\nO4ZpWDWYnCO021JTOUUOJ4J/y0416Bvkw0z59y7sNX7wDBBHHbK/XCc=\n-----END RSA PRIVATE KEY-----\n";
private static final String CERTIFICATE = "-----BEGIN CERTIFICATE-----\nMIIClzCCAgCgAwIBAgIBATANBgkqhkiG9w0BAQUFADCBnjELMAkGA1UEBhMCVVMx\nEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxFjAUBgNVBAoM\nDU9wc2NvZGUsIEluYy4xHDAaBgNVBAsME0NlcnRpZmljYXRlIFNlcnZpY2UxMjAw\nBgNVBAMMKW9wc2NvZGUuY29tL2VtYWlsQWRkcmVzcz1hdXRoQG9wc2NvZGUuY29t\nMB4XDTEwMDczMDIwNDEzMFoXDTIwMDcyNzIwNDEzMFowADCCASIwDQYJKoZIhvcN\nAQEBBQADggEPADCCAQoCggEBAMm9mSSahptCikfvJ30CTbEnfhfbVzTFewnznFuo\n7KrPBGYIlUdPYQ9SGDo+GKjNKiTjZYMoOMUVnsHUhu0Ez49ZSaVQInWvbF8tvpM8\nmoGQNQJtDmXG6m+YaHiA4HF/ng2u/bNLtA6Jo3HzvRCobxywc/szPt0Kj0ZD1fJ2\nE237Ph41c8zlOg9QdF0d/iD2WZdgJ1rNndKoZ0rR3A1L50VUND+PNmMDfVYHHjmb\naT89AwihCeU8eUk7m/JNP87f1QDB0Gny0rkDC3drOGS7jmabTf/7gLE5sYq3qnd+\n8/vGU3QWyfCxKSfogl7kn5uWlIe4sOqMb06GNgC+d/oytlECAwEAATANBgkqhkiG\n9w0BAQUFAAOBgQBftzSZxstWw60GqRTDNN/F2GnrdtnKBoXzHww3r6jtGEylYq20\n5KfKpEx+sPX0gyZuYJiXC2CkEjImAluWKcdN9ZF6VD541sheAjbiaU7q7ZsztTxF\nWUH2tCvHeDXYKPKek3QzL7bYpUhLnCN/XxEv6ibeMDwtI7f5qpk2Aspzcw==\n-----END CERTIFICATE-----\n";
private ParseJson<Client> handler;
private EncryptionService encryptionService;
private Crypto crypto;
private PrivateKey privateKey;
private X509Certificate certificate;
@BeforeTest
protected void setUpInjector() throws IOException {
protected void setUpInjector() throws IOException, CertificateException, InvalidKeySpecException {
Injector injector = Guice.createInjector(new ChefParserModule(), new GsonModule());
handler = injector.getInstance(Key.get(new TypeLiteral<ParseJson<Client>>() {
}));
encryptionService = injector.getInstance(EncryptionService.class);
certificate = encryptionService.x509CertificateFromPEM(CERTIFICATE.getBytes("UTF-8"));
privateKey = encryptionService.privateKeyFromPEM(PRIVATE_KEY.getBytes("UTF-8"));
crypto = injector.getInstance(Crypto.class);
certificate = Pems.x509Certificate(Payloads.newStringPayload(CERTIFICATE), null);
privateKey = crypto.rsaKeyFactory().generatePrivate(Pems.privateKeySpec(Payloads.newStringPayload(PRIVATE_KEY)));
}
public void test() {
public void test() throws IOException {
Client user = new Client(certificate, "jclouds", "adriancole-jcloudstest", "adriancole-jcloudstest", false,
privateKey);
byte[] encrypted = encryptionService.rsaEncrypt(Payloads.newPayload("fooya"), user.getCertificate()
.getPublicKey());
byte[] encrypted = ByteStreams.toByteArray(new RSAEncryptingPayload(Payloads.newPayload("fooya"), user
.getCertificate().getPublicKey()));
assertEquals(encryptionService.rsaDecrypt(Payloads.newPayload(encrypted), user.getPrivateKey()), "fooya"
.getBytes());
assertEquals(ByteStreams.toByteArray(new RSADecryptingPayload(Payloads.newPayload(encrypted), user
.getPrivateKey())), "fooya".getBytes());
assertEquals(handler.apply(new HttpResponse(200, "ok", newInputStreamPayload(ParseClientFromJsonTest.class
.getResourceAsStream("/client.json")))), user);

83
chef/core/src/test/java/org/jclouds/chef/functions/ParseCookbookVersionFromJsonTest.java
View File

@ -11,7 +11,7 @@ import org.jclouds.chef.domain.Attribute;
import org.jclouds.chef.domain.CookbookVersion;
import org.jclouds.chef.domain.Metadata;
import org.jclouds.chef.domain.Resource;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ParseJson;
import org.jclouds.io.Payloads;
@ -51,68 +51,69 @@ public class ParseCookbookVersionFromJsonTest {
@Test(enabled = false)
public void testBrew() throws IOException {
CookbookVersion cookbook = handler.apply(new HttpResponse(200, "ok", Payloads
.newPayload(ParseCookbookVersionFromJsonTest.class.getResourceAsStream("/brew-cookbook.json"))));
.newPayload(ParseCookbookVersionFromJsonTest.class.getResourceAsStream("/brew-cookbook.json"))));
assertEquals(cookbook, handler.apply(new HttpResponse(200, "ok", Payloads.newPayload(Utils.toInputStream(json
.toJson(cookbook))))));
.toJson(cookbook))))));
}
@Test(enabled = false)
public void testTomcat() {
CookbookVersion cookbook = handler.apply(new HttpResponse(200, "ok", Payloads
.newPayload(ParseCookbookVersionFromJsonTest.class.getResourceAsStream("/tomcat-cookbook.json"))));
.newPayload(ParseCookbookVersionFromJsonTest.class.getResourceAsStream("/tomcat-cookbook.json"))));
assertEquals(cookbook, handler.apply(new HttpResponse(200, "ok", Payloads.newPayload(Utils.toInputStream(json
.toJson(cookbook))))));
.toJson(cookbook))))));
}
@Test(enabled = false)
public void testMysql() throws IOException {
CookbookVersion cookbook = handler.apply(new HttpResponse(200, "ok", Payloads
.newPayload(ParseCookbookVersionFromJsonTest.class.getResourceAsStream("/mysql-cookbook.json"))));
.newPayload(ParseCookbookVersionFromJsonTest.class.getResourceAsStream("/mysql-cookbook.json"))));
assertEquals(cookbook, handler.apply(new HttpResponse(200, "ok", Payloads.newPayload(Utils.toInputStream(json
.toJson(cookbook))))));
.toJson(cookbook))))));
}
@Test(enabled = false)
public void testApache() {
EncryptionService encryptionService = injector.getInstance(EncryptionService.class);
assertEquals(
handler.apply(new HttpResponse(200, "ok", Payloads.newPayload(ParseCookbookVersionFromJsonTest.class
.getResourceAsStream("/apache-chef-demo-cookbook.json")))),
new CookbookVersion(
"apache-chef-demo-0.0.0",
ImmutableSet.<Resource> of(),
ImmutableSet.<Resource> of(),
ImmutableSet.<Resource> of(),
new Metadata("Apache v2.0", "Your Name", ImmutableMap.<String, String> of(), ImmutableMap
.<String, Set<String>> of(), "youremail@example.com", ImmutableMap.<String, Set<String>> of(),
"A fabulous new cookbook", ImmutableMap.<String, Set<String>> of(), ImmutableMap
.<String, Set<String>> of(), "0.0.0", ImmutableMap.<String, String> of(), ImmutableMap
.<String, Set<String>> of(), "apache-chef-demo", ImmutableMap.<String, String> of(), "",
ImmutableMap.<String, Attribute> of(), ImmutableMap.<String, String> of()),
ImmutableSet.<Resource> of(),
"apache-chef-demo",
ImmutableSet.<Resource> of(),
ImmutableSet.<Resource> of(),
ImmutableSet.<Resource> of(),
"0.0.0",
ImmutableSet.<Resource> of(),
ImmutableSet
.<Resource> of(
new Resource(
"README",
URI
.create("https://s3.amazonaws.com/opscode-platform-production-data/organization-486ca3ac66264fea926aa0b4ff74341c/checksum-11637f98942eafbf49c71b7f2f048b78?AWSAccessKeyId=AKIAJOZTD2N26S7W6APA&Expires=1277766181&Signature=zgpNl6wSxjTNovqZu2nJq0JztU8%3D"),
encryptionService.fromHex("11637f98942eafbf49c71b7f2f048b78"), "README", "default"),
new Resource(
"Rakefile",
URI
.create("https://s3.amazonaws.com/opscode-platform-production-data/organization-486ca3ac66264fea926aa0b4ff74341c/checksum-ebcf925a1651b4e04b9cd8aac2bc54eb?AWSAccessKeyId=AKIAJOZTD2N26S7W6APA&Expires=1277766181&Signature=EFzzDSKKytTl7b%2FxrCeNLh05zj4%3D"),
encryptionService.fromHex("ebcf925a1651b4e04b9cd8aac2bc54eb"), "Rakefile",
"default"))));
handler.apply(new HttpResponse(200, "ok", Payloads.newPayload(ParseCookbookVersionFromJsonTest.class
.getResourceAsStream("/apache-chef-demo-cookbook.json")))),
new CookbookVersion(
"apache-chef-demo-0.0.0",
ImmutableSet.<Resource> of(),
ImmutableSet.<Resource> of(),
ImmutableSet.<Resource> of(),
new Metadata("Apache v2.0", "Your Name", ImmutableMap.<String, String> of(), ImmutableMap
.<String, Set<String>> of(), "youremail@example.com", ImmutableMap
.<String, Set<String>> of(), "A fabulous new cookbook", ImmutableMap
.<String, Set<String>> of(), ImmutableMap.<String, Set<String>> of(), "0.0.0",
ImmutableMap.<String, String> of(), ImmutableMap.<String, Set<String>> of(),
"apache-chef-demo", ImmutableMap.<String, String> of(), "", ImmutableMap
.<String, Attribute> of(), ImmutableMap.<String, String> of()),
ImmutableSet.<Resource> of(),
"apache-chef-demo",
ImmutableSet.<Resource> of(),
ImmutableSet.<Resource> of(),
ImmutableSet.<Resource> of(),
"0.0.0",
ImmutableSet.<Resource> of(),
ImmutableSet
.<Resource> of(
new Resource(
"README",
URI
.create("https://s3.amazonaws.com/opscode-platform-production-data/organization-486ca3ac66264fea926aa0b4ff74341c/checksum-11637f98942eafbf49c71b7f2f048b78?AWSAccessKeyId=AKIAJOZTD2N26S7W6APA&Expires=1277766181&Signature=zgpNl6wSxjTNovqZu2nJq0JztU8%3D"),
CryptoStreams.hex("11637f98942eafbf49c71b7f2f048b78"), "README",
"default"),
new Resource(
"Rakefile",
URI
.create("https://s3.amazonaws.com/opscode-platform-production-data/organization-486ca3ac66264fea926aa0b4ff74341c/checksum-ebcf925a1651b4e04b9cd8aac2bc54eb?AWSAccessKeyId=AKIAJOZTD2N26S7W6APA&Expires=1277766181&Signature=EFzzDSKKytTl7b%2FxrCeNLh05zj4%3D"),
CryptoStreams.hex("ebcf925a1651b4e04b9cd8aac2bc54eb"), "Rakefile",
"default"))));
}
}

33
chef/core/src/test/java/org/jclouds/chef/functions/ParseUploadSandboxFromJsonTest.java
View File

@ -9,7 +9,7 @@ import java.util.List;
import org.jclouds.chef.config.ChefParserModule;
import org.jclouds.chef.domain.ChecksumStatus;
import org.jclouds.chef.domain.UploadSandbox;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.functions.ParseJson;
import org.jclouds.io.Payloads;
@ -43,22 +43,21 @@ public class ParseUploadSandboxFromJsonTest {
}
public void test() {
EncryptionService encryptionService = injector.getInstance(EncryptionService.class);
assertEquals(
handler.apply(new HttpResponse(200, "ok", Payloads.newPayload(ParseUploadSandboxFromJsonTest.class
.getResourceAsStream("/upload-site.json")))),
new UploadSandbox(
URI
.create("https://api.opscode.com/organizations/jclouds/sandboxes/d454f71e2a5f400c808d0c5d04c2c88c"),
ImmutableMap
.<List<Byte>, ChecksumStatus> of(
Bytes.asList(encryptionService.fromHex("0c5ecd7788cf4f6c7de2a57193897a6c")),
new ChecksumStatus(
URI
.create("https://s3.amazonaws.com/opscode-platform-production-data/organization-486ca3ac66264fea926aa0b4ff74341c/sandbox-d454f71e2a5f400c808d0c5d04c2c88c/checksum-0c5ecd7788cf4f6c7de2a57193897a6c?AWSAccessKeyId=AKIAJOZTD2N26S7W6APA&Expires=1277344702&Signature=FtKyqvYEjhhEKmRY%2B0M8aGPMM7g%3D"),
true), Bytes.asList(encryptionService.fromHex("0189e76ccc476701d6b374e5a1a27347")),
new ChecksumStatus(), Bytes.asList(encryptionService
.fromHex("1dda05ed139664f1f89b9dec482b77c0")), new ChecksumStatus()),
"d454f71e2a5f400c808d0c5d04c2c88c"));
handler.apply(new HttpResponse(200, "ok", Payloads.newPayload(ParseUploadSandboxFromJsonTest.class
.getResourceAsStream("/upload-site.json")))),
new UploadSandbox(
URI
.create("https://api.opscode.com/organizations/jclouds/sandboxes/d454f71e2a5f400c808d0c5d04c2c88c"),
ImmutableMap
.<List<Byte>, ChecksumStatus> of(
Bytes.asList(CryptoStreams.hex("0c5ecd7788cf4f6c7de2a57193897a6c")),
new ChecksumStatus(
URI
.create("https://s3.amazonaws.com/opscode-platform-production-data/organization-486ca3ac66264fea926aa0b4ff74341c/sandbox-d454f71e2a5f400c808d0c5d04c2c88c/checksum-0c5ecd7788cf4f6c7de2a57193897a6c?AWSAccessKeyId=AKIAJOZTD2N26S7W6APA&Expires=1277344702&Signature=FtKyqvYEjhhEKmRY%2B0M8aGPMM7g%3D"),
true), Bytes.asList(CryptoStreams
.hex("0189e76ccc476701d6b374e5a1a27347")), new ChecksumStatus(),
Bytes.asList(CryptoStreams.hex("1dda05ed139664f1f89b9dec482b77c0")),
new ChecksumStatus()), "d454f71e2a5f400c808d0c5d04c2c88c"));
}
}

6
chef/core/src/test/java/org/jclouds/ohai/functions/ByteArrayToMacAddressTest.java
View File

@ -27,7 +27,7 @@ import static org.testng.Assert.assertEquals;
import java.io.IOException;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@ -43,16 +43,14 @@ import com.google.inject.Injector;
public class ByteArrayToMacAddressTest {
private ByteArrayToMacAddress converter;
private EncryptionService encryptionService;
@BeforeTest
protected void setUpInjector() throws IOException {
Injector injector = Guice.createInjector();
converter = injector.getInstance(ByteArrayToMacAddress.class);
encryptionService = injector.getInstance(EncryptionService.class);
}
public void test() {
assertEquals(converter.apply(encryptionService.fromHex("0026bb09e6c4")), "00:26:bb:09:e6:c4");
assertEquals(converter.apply(CryptoStreams.hex("0026bb09e6c4")), "00:26:bb:09:e6:c4");
}
}

11
chef/servlet/src/main/java/org/jclouds/chef/servlet/ChefRegistrationListener.java
View File

@ -43,10 +43,13 @@ import org.jclouds.chef.ChefService;
import org.jclouds.chef.domain.Client;
import org.jclouds.chef.reference.ChefConstants;
import org.jclouds.chef.servlet.functions.InitParamsToProperties;
import org.jclouds.crypto.Pems;
import org.jclouds.logging.Logger;
import org.jclouds.logging.jdk.JDKLogger;
import org.jclouds.rest.RestContextFactory;
import com.google.common.base.Throwables;
/**
* Registers a new node in Chef and binds its name to {@link ChefConstants.CHEF_NODE}, its role to
* {@link ChefConstants.CHEF_ROLE} and the {@link ChefService} for the client to
@ -129,14 +132,14 @@ public class ChefRegistrationListener implements ServletContextListener {
clientProperties.putAll(overrides);
removeCredentials(clientProperties);
clientProperties.setProperty("chef.identity", id);
clientProperties.setProperty("chef.credential", validatorClient.getContext().utils().encryption().toPem(
client.getPrivateKey()));
clientProperties.setProperty("chef.credential", Pems.pem(client.getPrivateKey()));
clientService = createService(clientProperties);
clientService.createNodeAndPopulateAutomaticAttributes(id, singleton("role[" + role + "]"));
return clientService;
} catch (RuntimeException e) {
} catch (Exception e) {
logger.error(e, "error creating node %s", id);
throw e;
Throwables.propagate(e);
return null;
}
}

4
compute/src/main/java/org/jclouds/compute/internal/BaseComputeService.java
View File

@ -26,8 +26,8 @@ package org.jclouds.compute.internal;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.compute.util.ComputeServiceUtils.installNewCredentials;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.ConcurrentUtils.transformParallel;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import static org.jclouds.concurrent.FutureIterables.transformParallel;
import java.util.Map;
import java.util.Set;

4
compute/src/main/java/org/jclouds/compute/internal/UtilsImpl.java
View File

@ -25,8 +25,8 @@ import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.compute.Utils;
import org.jclouds.crypto.Crypto;
import org.jclouds.date.DateService;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.json.Json;
import org.jclouds.logging.Logger.LoggerFactory;
import org.jclouds.rest.HttpAsyncClient;
@ -46,7 +46,7 @@ public class UtilsImpl extends org.jclouds.rest.internal.UtilsImpl implements Ut
@Inject
UtilsImpl(Json json, HttpClient simpleClient, HttpAsyncClient simpleAsyncClient,
EncryptionService encryption, DateService date,
Crypto encryption, DateService date,
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userThreads,
@Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioThreads,
LoggerFactory loggerFactory) {

2
compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java
View File

@ -22,7 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static org.jclouds.compute.util.ComputeServiceUtils.installNewCredentials;
import static org.jclouds.compute.util.ComputeServiceUtils.isKeyAuth;
import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
import java.util.List;
import java.util.Map;

2
core/src/main/clojure/org/jclouds/core.clj
View File

@ -34,7 +34,7 @@
:enterprise 'org.jclouds.enterprise.config.EnterpriseConfigurationModule
:apachehc 'org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule
:ning 'org.jclouds.http.ning.config.NingHttpCommandExecutorServiceModule
:bouncycastle 'org.jclouds.encryption.bouncycastle.config.BouncyCastleEncryptionServiceModule
:bouncycastle 'org.jclouds.encryption.bouncycastle.config.BouncyCastleCryptoModule
:joda 'org.jclouds.date.joda.config.JodaDateServiceModule
:gae 'org.jclouds.gae.config.GoogleAppEngineConfigurationModule})

561
core/src/main/java/org/jclouds/concurrent/ConcurrentUtils.java
View File

@ -1,561 +0,0 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.concurrent;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Maps.newHashMap;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.AbstractExecutorService;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.Nullable;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
import org.jclouds.logging.Logger;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.ForwardingObject;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ExecutionList;
import com.google.common.util.concurrent.ForwardingFuture;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
/**
* Adapt things from Guava.
*
* @author Adrian Cole
*/
@Singleton
public class ConcurrentUtils {
@Resource
private static Logger logger = Logger.CONSOLE;
@Inject(optional = true)
@Named(Constants.PROPERTY_MAX_RETRIES)
private static int maxRetries = 5;
@Inject(optional = true)
private static BackoffLimitedRetryHandler retryHandler = BackoffLimitedRetryHandler.INSTANCE;
public static <F, T> Iterable<T> transformParallel(final Iterable<F> fromIterable,
final Function<? super F, Future<T>> function) {
return transformParallel(fromIterable, function, sameThreadExecutor(), null);
}
public static <F, T> Iterable<T> transformParallel(final Iterable<F> fromIterable,
final Function<? super F, Future<T>> function, ExecutorService exec, @Nullable Long maxTime) {
return transformParallel(fromIterable, function, exec, maxTime, logger, "transforming");
}
public static <F, T> Iterable<T> transformParallel(final Iterable<F> fromIterable,
final Function<? super F, Future<T>> function, ExecutorService exec, @Nullable Long maxTime, Logger logger,
String logPrefix) {
return transformParallel(fromIterable, function, exec, maxTime, logger, logPrefix, retryHandler, maxRetries);
}
public static <F, T> Iterable<T> transformParallel(Iterable<F> fromIterable,
Function<? super F, Future<T>> function, ExecutorService exec, @Nullable Long maxTime, Logger logger,
String logPrefix, BackoffLimitedRetryHandler retryHandler, int maxRetries) {
Map<F, Exception> exceptions = newHashMap();
Map<F, Future<T>> responses = newHashMap();
for (int i = 0; i < maxRetries; i++) {
for (F from : fromIterable) {
responses.put(from, function.apply(from));
}
exceptions = awaitCompletion(responses, exec, maxTime, logger, logPrefix);
if (exceptions.size() > 0) {
fromIterable = exceptions.keySet();
retryHandler.imposeBackoffExponentialDelay(i + 1, String.format("error %s: %s: %s", logPrefix,
fromIterable, exceptions));
} else {
break;
}
}
if (exceptions.size() > 0)
throw new RuntimeException(String.format("error %s: %s: %s", logPrefix, fromIterable, exceptions));
return unwrap(responses.values());
}
public static <T> Map<T, Exception> awaitCompletion(Map<T, ? extends Future<?>> responses, ExecutorService exec,
@Nullable Long maxTime, final Logger logger, final String logPrefix) {
if (responses.size() == 0)
return ImmutableMap.of();
final int total = responses.size();
final CountDownLatch doneSignal = new CountDownLatch(total);
final AtomicInteger complete = new AtomicInteger(0);
final AtomicInteger errors = new AtomicInteger(0);
final long start = System.currentTimeMillis();
final Map<T, Exception> errorMap = Maps.newHashMap();
for (final java.util.Map.Entry<T, ? extends Future<?>> future : responses.entrySet()) {
makeListenable(future.getValue(), exec).addListener(new Runnable() {
public void run() {
try {
future.getValue().get();
complete.incrementAndGet();
} catch (Exception e) {
errors.incrementAndGet();
logException(logger, logPrefix, total, complete.get(), errors.get(), start, e);
errorMap.put(future.getKey(), e);
}
doneSignal.countDown();
}
}, exec);
}
try {
if (maxTime != null)
doneSignal.await(maxTime, TimeUnit.MILLISECONDS);
else
doneSignal.await();
if (errors.get() > 0) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
RuntimeException exception = new RuntimeException(message);
logger.error(exception, message);
}
if (logger.isTraceEnabled()) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
logger.trace(message);
}
} catch (InterruptedException e) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
TimeoutException exception = new TimeoutException(message);
logger.error(exception, message);
Throwables.propagate(exception);
}
return errorMap;
}
public static <T> Iterable<T> unwrap(Iterable<Future<T>> values) {
return Iterables.transform(values, new Function<Future<T>, T>() {
@Override
public T apply(Future<T> from) {
try {
return from.get();
} catch (InterruptedException e) {
Throwables.propagate(e);
} catch (ExecutionException e) {
Throwables.propagate(e);
}
return null;
}
});
}
private static void logException(Logger logger, String logPrefix, int total, int complete, int errors, long start,
Exception e) {
String message = message(logPrefix, total, complete, errors, start);
logger.error(e, message);
}
private static String message(String prefix, int size, int complete, int errors, long start) {
return String.format("%s, completed: %d/%d, errors: %d, rate: %dms/op", prefix, complete, size, errors,
(long) ((System.currentTimeMillis() - start) / ((double) size)));
}
protected static boolean timeOut(long start, Long maxTime) {
return maxTime != null ? System.currentTimeMillis() < start + maxTime : false;
}
/**
* Just like {@code Futures#makeListenable} except that we pass in an executorService.
* <p/>
* Temporary hack until http://code.google.com/p/guava-libraries/issues/detail?id=317 is fixed.
*/
public static <T> ListenableFuture<T> makeListenable(Future<T> future, ExecutorService executorService) {
if (future instanceof ListenableFuture<?>) {
return (ListenableFuture<T>) future;
}
return ListenableFutureAdapter.create(future, executorService);
}
/**
* Just like {@code Futures#compose} except that we check the type of the executorService before
* creating the Future. If we are single threaded, invoke the function lazy as opposed to
* chaining, so that we don't invoke get() early.
*/
public static <I, O> ListenableFuture<O> compose(Future<I> future, final Function<? super I, ? extends O> function,
ExecutorService executorService) {
if (future instanceof ListenableFutureAdapter<?>) {
ListenableFutureAdapter<I> lf = (ListenableFutureAdapter<I>) future;
if (lf.futureListener.executor.getClass().isAnnotationPresent(SingleThreaded.class))
return LazyListenableFutureFunctionAdapter.create(((ListenableFutureAdapter<I>) future).futureListener,
function);
else
return Futures.compose(lf, function, executorService);
} else if (executorService.getClass().isAnnotationPresent(SingleThreaded.class)) {
return LazyListenableFutureFunctionAdapter.create(future, function, executorService);
} else {
return Futures.compose(makeListenable(future, executorService), function, executorService);
}
}
public static class FutureListener<T> {
private final Future<T> future;
private final ExecutorService executor;
private final ExecutionList executionList = new ExecutionList();
private final AtomicBoolean hasListeners = new AtomicBoolean(false);
static <T> FutureListener<T> create(Future<T> future, ExecutorService executor) {
return new FutureListener<T>(future, executor);
}
private FutureListener(Future<T> future, ExecutorService executor) {
this.future = checkNotNull(future, "future");
this.executor = checkNotNull(executor, "executor");
}
public void addListener(Runnable listener, Executor exec) {
// When a listener is first added, we run a task that will wait for
// the future to finish, and when it is done will run the listeners.
if (!hasListeners.get() && hasListeners.compareAndSet(false, true)) {
executor.execute(new Runnable() {
/* @Override */
public void run() {
try {
future.get();
} catch (CancellationException e) {
// The task was cancelled, so it is done, run the listeners.
} catch (InterruptedException e) {
// This thread was interrupted. This should never happen, so we
// throw an IllegalStateException.
throw new IllegalStateException("Adapter thread interrupted!", e);
} catch (ExecutionException e) {
// The task caused an exception, so it is done, run the listeners.
}
executionList.run();
}
});
}
executionList.add(listener, exec);
}
Future<T> getFuture() {
return future;
}
ExecutorService getExecutor() {
return executor;
}
}
public static class ListenableFutureAdapter<T> extends ForwardingFuture<T> implements ListenableFuture<T> {
private final FutureListener<T> futureListener;
static <T> ListenableFutureAdapter<T> create(Future<T> future, ExecutorService executor) {
return new ListenableFutureAdapter<T>(future, executor);
}
private ListenableFutureAdapter(Future<T> future, ExecutorService executor) {
this.futureListener = FutureListener.create(future, executor);
}
@Override
protected Future<T> delegate() {
return futureListener.getFuture();
}
@Override
public void addListener(Runnable listener, Executor exec) {
futureListener.addListener(listener, exec);
}
}
public static class LazyListenableFutureFunctionAdapter<I, O> extends ForwardingObject implements
ListenableFuture<O> {
private final FutureListener<I> futureListener;
private final Function<? super I, ? extends O> function;
static <I, O> LazyListenableFutureFunctionAdapter<I, O> create(Future<I> future,
Function<? super I, ? extends O> function, ExecutorService executor) {
return new LazyListenableFutureFunctionAdapter<I, O>(future, function, executor);
}
static <I, O> LazyListenableFutureFunctionAdapter<I, O> create(FutureListener<I> futureListener,
Function<? super I, ? extends O> function) {
return new LazyListenableFutureFunctionAdapter<I, O>(futureListener, function);
}
private LazyListenableFutureFunctionAdapter(Future<I> future, Function<? super I, ? extends O> function,
ExecutorService executor) {
this(FutureListener.create(future, executor), function);
}
private LazyListenableFutureFunctionAdapter(FutureListener<I> futureListener,
Function<? super I, ? extends O> function) {
this.futureListener = checkNotNull(futureListener, "futureListener");
this.function = checkNotNull(function, "function");
}
/*
* Concurrency detail:
*
* <p>To preserve the idempotency of calls to this.get(*) calls to the function are only
* applied once. A lock is required to prevent multiple applications of the function. The
* calls to future.get(*) are performed outside the lock, as is required to prevent calls to
* get(long, TimeUnit) to persist beyond their timeout.
*
* <p>Calls to future.get(*) on every call to this.get(*) also provide the cancellation
* behavior for this.
*
* <p>(Consider: in thread A, call get(), in thread B call get(long, TimeUnit). Thread B may
* have to wait for Thread A to finish, which would be unacceptable.)
*
* <p>Note that each call to Future<O>.get(*) results in a call to Future<I>.get(*), but the
* function is only applied once, so Future<I>.get(*) is assumed to be idempotent.
*/
private final Object lock = new Object();
private boolean set = false;
private O value = null;
@Override
public O get() throws InterruptedException, ExecutionException {
return apply(futureListener.getFuture().get());
}
@Override
public O get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
return apply(futureListener.getFuture().get(timeout, unit));
}
private O apply(I raw) {
synchronized (lock) {
if (!set) {
value = function.apply(raw);
set = true;
}
return value;
}
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
return futureListener.getFuture().cancel(mayInterruptIfRunning);
}
@Override
public boolean isCancelled() {
return futureListener.getFuture().isCancelled();
}
@Override
public boolean isDone() {
return futureListener.getFuture().isDone();
}
@Override
public void addListener(Runnable listener, Executor exec) {
futureListener.addListener(listener, exec);
}
@Override
protected Object delegate() {
return futureListener.getFuture();
}
}
/**
* Taken from {@link MoreExecutors#sameThreadExecutor} as it was hidden and therefore incapable
* of instanceof checks.
*
*
*
* Creates an executor service that runs each task in the thread that invokes {@code
* execute/submit}, as in {@link CallerRunsPolicy} This applies both to individually submitted
* tasks and to collections of tasks submitted via {@code invokeAll} or {@code invokeAny}. In the
* latter case, tasks will run serially on the calling thread. Tasks are run to completion before
* a {@code Future} is returned to the caller (unless the executor has been shutdown).
*
* <p>
* Although all tasks are immediately executed in the thread that submitted the task, this
* {@code ExecutorService} imposes a small locking overhead on each task submission in order to
* implement shutdown and termination behavior.
*
* <p>
* The implementation deviates from the {@code ExecutorService} specification with regards to the
* {@code shutdownNow} method. First, "best-effort" with regards to canceling running tasks is
* implemented as "no-effort". No interrupts or other attempts are made to stop threads executing
* tasks. Second, the returned list will always be empty, as any submitted task is considered to
* have started execution. This applies also to tasks given to {@code invokeAll} or {@code
* invokeAny} which are pending serial execution, even the subset of the tasks that have not yet
* started execution. It is unclear from the {@code ExecutorService} specification if these
* should be included, and it's much easier to implement the interpretation that they not be.
* Finally, a call to {@code shutdown} or {@code shutdownNow} may result in concurrent calls to
* {@code invokeAll/invokeAny} throwing RejectedExecutionException, although a subset of the
* tasks may already have been executed.
*/
public static ExecutorService sameThreadExecutor() {
return new SameThreadExecutorService();
}
// See sameThreadExecutor javadoc for behavioral notes.
@SingleThreaded
public static class SameThreadExecutorService extends AbstractExecutorService {
/**
* Lock used whenever accessing the state variables (runningTasks, shutdown,
* terminationCondition) of the executor
*/
private final Lock lock = new ReentrantLock();
/** Signaled after the executor is shutdown and running tasks are done */
private final Condition termination = lock.newCondition();
private SameThreadExecutorService() {
}
/*
* Conceptually, these two variables describe the executor being in one of three states: -
* Active: shutdown == false - Shutdown: runningTasks > 0 and shutdown == true - Terminated:
* runningTasks == 0 and shutdown == true
*/
private int runningTasks = 0;
private boolean shutdown = false;
@Override
public void execute(Runnable command) {
startTask();
try {
command.run();
} finally {
endTask();
}
}
@Override
public boolean isShutdown() {
lock.lock();
try {
return shutdown;
} finally {
lock.unlock();
}
}
@Override
public void shutdown() {
lock.lock();
try {
shutdown = true;
} finally {
lock.unlock();
}
}
// See sameThreadExecutor javadoc for unusual behavior of this method.
@Override
public List<Runnable> shutdownNow() {
shutdown();
return Collections.emptyList();
}
@Override
public boolean isTerminated() {
lock.lock();
try {
return shutdown && runningTasks == 0;
} finally {
lock.unlock();
}
}
@Override
public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
long nanos = unit.toNanos(timeout);
lock.lock();
try {
for (;;) {
if (isTerminated()) {
return true;
} else if (nanos <= 0) {
return false;
} else {
nanos = termination.awaitNanos(nanos);
}
}
} finally {
lock.unlock();
}
}
/**
* Checks if the executor has been shut down and increments the running task count.
*
* @throws RejectedExecutionException
* if the executor has been previously shutdown
*/
private void startTask() {
lock.lock();
try {
if (isShutdown()) {
throw new RejectedExecutionException("Executor already shutdown");
}
runningTasks++;
} finally {
lock.unlock();
}
}
/**
* Decrements the running task count.
*/
private void endTask() {
lock.lock();
try {
runningTasks--;
if (isTerminated()) {
termination.signalAll();
}
} finally {
lock.unlock();
}
}
}
}

185
core/src/main/java/org/jclouds/concurrent/FutureIterables.java Normal file
View File

@ -0,0 +1,185 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.concurrent;
import static com.google.common.collect.Maps.newHashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
import javax.annotation.Resource;
import javax.inject.Named;
import org.jclouds.Constants;
import org.jclouds.http.handlers.BackoffLimitedRetryHandler;
import org.jclouds.logging.Logger;
import com.google.common.annotations.Beta;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.inject.Inject;
/**
* functions related to or replacing those in {@link com.google.common.collect.Iterables} dealing with Futures
*
* @author Adrian Cole
*/
@Beta
public class FutureIterables {
@Resource
private static Logger logger = Logger.CONSOLE;
@Inject(optional = true)
@Named(Constants.PROPERTY_MAX_RETRIES)
private static int maxRetries = 5;
@Inject(optional = true)
private static BackoffLimitedRetryHandler retryHandler = BackoffLimitedRetryHandler.INSTANCE;
public static <F, T> Iterable<T> transformParallel(final Iterable<F> fromIterable,
final Function<? super F, Future<T>> function) {
return transformParallel(fromIterable, function, org.jclouds.concurrent.MoreExecutors.sameThreadExecutor(), null);
}
public static <F, T> Iterable<T> transformParallel(final Iterable<F> fromIterable,
final Function<? super F, Future<T>> function, ExecutorService exec, @Nullable Long maxTime) {
return transformParallel(fromIterable, function, exec, maxTime, logger, "transforming");
}
public static <F, T> Iterable<T> transformParallel(final Iterable<F> fromIterable,
final Function<? super F, Future<T>> function, ExecutorService exec, @Nullable Long maxTime, Logger logger,
String logPrefix) {
return transformParallel(fromIterable, function, exec, maxTime, logger, logPrefix, retryHandler, maxRetries);
}
public static <F, T> Iterable<T> transformParallel(Iterable<F> fromIterable,
Function<? super F, Future<T>> function, ExecutorService exec, @Nullable Long maxTime, Logger logger,
String logPrefix, BackoffLimitedRetryHandler retryHandler, int maxRetries) {
Map<F, Exception> exceptions = newHashMap();
Map<F, Future<T>> responses = newHashMap();
for (int i = 0; i < maxRetries; i++) {
for (F from : fromIterable) {
responses.put(from, function.apply(from));
}
exceptions = awaitCompletion(responses, exec, maxTime, logger, logPrefix);
if (exceptions.size() > 0) {
fromIterable = exceptions.keySet();
retryHandler.imposeBackoffExponentialDelay(i + 1, String.format("error %s: %s: %s", logPrefix,
fromIterable, exceptions));
} else {
break;
}
}
if (exceptions.size() > 0)
throw new RuntimeException(String.format("error %s: %s: %s", logPrefix, fromIterable, exceptions));
return unwrap(responses.values());
}
public static <T> Map<T, Exception> awaitCompletion(Map<T, ? extends Future<?>> responses, ExecutorService exec,
@Nullable Long maxTime, final Logger logger, final String logPrefix) {
if (responses.size() == 0)
return ImmutableMap.of();
final int total = responses.size();
final CountDownLatch doneSignal = new CountDownLatch(total);
final AtomicInteger complete = new AtomicInteger(0);
final AtomicInteger errors = new AtomicInteger(0);
final long start = System.currentTimeMillis();
final Map<T, Exception> errorMap = Maps.newHashMap();
for (final java.util.Map.Entry<T, ? extends Future<?>> future : responses.entrySet()) {
Futures.makeListenable(future.getValue(), exec).addListener(new Runnable() {
public void run() {
try {
future.getValue().get();
complete.incrementAndGet();
} catch (Exception e) {
errors.incrementAndGet();
logException(logger, logPrefix, total, complete.get(), errors.get(), start, e);
errorMap.put(future.getKey(), e);
}
doneSignal.countDown();
}
}, exec);
}
try {
if (maxTime != null)
doneSignal.await(maxTime, TimeUnit.MILLISECONDS);
else
doneSignal.await();
if (errors.get() > 0) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
RuntimeException exception = new RuntimeException(message);
logger.error(exception, message);
}
if (logger.isTraceEnabled()) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
logger.trace(message);
}
} catch (InterruptedException e) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
TimeoutException exception = new TimeoutException(message);
logger.error(exception, message);
Throwables.propagate(exception);
}
return errorMap;
}
public static <T> Iterable<T> unwrap(Iterable<Future<T>> values) {
return Iterables.transform(values, new Function<Future<T>, T>() {
@Override
public T apply(Future<T> from) {
try {
return from.get();
} catch (InterruptedException e) {
Throwables.propagate(e);
} catch (ExecutionException e) {
Throwables.propagate(e);
}
return null;
}
});
}
private static void logException(Logger logger, String logPrefix, int total, int complete, int errors, long start,
Exception e) {
String message = message(logPrefix, total, complete, errors, start);
logger.error(e, message);
}
private static String message(String prefix, int size, int complete, int errors, long start) {
return String.format("%s, completed: %d/%d, errors: %d, rate: %dms/op", prefix, complete, size, errors,
(long) ((System.currentTimeMillis() - start) / ((double) size)));
}
protected static boolean timeOut(long start, Long maxTime) {
return maxTime != null ? System.currentTimeMillis() < start + maxTime : false;
}
}

248
core/src/main/java/org/jclouds/concurrent/Futures.java Normal file
View File

@ -0,0 +1,248 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.concurrent;
import static com.google.common.base.Preconditions.checkNotNull;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.Beta;
import com.google.common.base.Function;
import com.google.common.collect.ForwardingObject;
import com.google.common.util.concurrent.ExecutionList;
import com.google.common.util.concurrent.ForwardingFuture;
import com.google.common.util.concurrent.ListenableFuture;
/**
* functions related to or replacing those in {@link com.google.common.util.concurrent.Futures}
*
* @author Adrian Cole
*/
@Beta
public class Futures {
public static class FutureListener<T> {
private final Future<T> future;
final ExecutorService executor;
private final ExecutionList executionList = new ExecutionList();
private final AtomicBoolean hasListeners = new AtomicBoolean(false);
static <T> FutureListener<T> create(Future<T> future, ExecutorService executor) {
return new FutureListener<T>(future, executor);
}
private FutureListener(Future<T> future, ExecutorService executor) {
this.future = checkNotNull(future, "future");
this.executor = checkNotNull(executor, "executor");
}
public void addListener(Runnable listener, Executor exec) {
// When a listener is first added, we run a task that will wait for
// the future to finish, and when it is done will run the listeners.
if (!hasListeners.get() && hasListeners.compareAndSet(false, true)) {
executor.execute(new Runnable() {
/* @Override */
public void run() {
try {
future.get();
} catch (CancellationException e) {
// The task was cancelled, so it is done, run the listeners.
} catch (InterruptedException e) {
// This thread was interrupted. This should never happen, so we
// throw an IllegalStateException.
throw new IllegalStateException("Adapter thread interrupted!", e);
} catch (ExecutionException e) {
// The task caused an exception, so it is done, run the listeners.
}
executionList.run();
}
});
}
executionList.add(listener, exec);
}
Future<T> getFuture() {
return future;
}
ExecutorService getExecutor() {
return executor;
}
}
public static class ListenableFutureAdapter<T> extends ForwardingFuture<T> implements ListenableFuture<T> {
final FutureListener<T> futureListener;
static <T> ListenableFutureAdapter<T> create(Future<T> future, ExecutorService executor) {
return new ListenableFutureAdapter<T>(future, executor);
}
private ListenableFutureAdapter(Future<T> future, ExecutorService executor) {
this.futureListener = FutureListener.create(future, executor);
}
@Override
protected Future<T> delegate() {
return futureListener.getFuture();
}
@Override
public void addListener(Runnable listener, Executor exec) {
futureListener.addListener(listener, exec);
}
}
public static class LazyListenableFutureFunctionAdapter<I, O> extends ForwardingObject implements
ListenableFuture<O> {
private final FutureListener<I> futureListener;
private final Function<? super I, ? extends O> function;
static <I, O> LazyListenableFutureFunctionAdapter<I, O> create(Future<I> future,
Function<? super I, ? extends O> function, ExecutorService executor) {
return new LazyListenableFutureFunctionAdapter<I, O>(future, function, executor);
}
static <I, O> LazyListenableFutureFunctionAdapter<I, O> create(FutureListener<I> futureListener,
Function<? super I, ? extends O> function) {
return new LazyListenableFutureFunctionAdapter<I, O>(futureListener, function);
}
private LazyListenableFutureFunctionAdapter(Future<I> future, Function<? super I, ? extends O> function,
ExecutorService executor) {
this(FutureListener.create(future, executor), function);
}
private LazyListenableFutureFunctionAdapter(FutureListener<I> futureListener,
Function<? super I, ? extends O> function) {
this.futureListener = checkNotNull(futureListener, "futureListener");
this.function = checkNotNull(function, "function");
}
/*
* Concurrency detail:
*
* <p>To preserve the idempotency of calls to this.get(*) calls to the function are only
* applied once. A lock is required to prevent multiple applications of the function. The
* calls to future.get(*) are performed outside the lock, as is required to prevent calls to
* get(long, TimeUnit) to persist beyond their timeout.
*
* <p>Calls to future.get(*) on every call to this.get(*) also provide the cancellation
* behavior for this.
*
* <p>(Consider: in thread A, call get(), in thread B call get(long, TimeUnit). Thread B may
* have to wait for Thread A to finish, which would be unacceptable.)
*
* <p>Note that each call to Future<O>.get(*) results in a call to Future<I>.get(*), but the
* function is only applied once, so Future<I>.get(*) is assumed to be idempotent.
*/
private final Object lock = new Object();
private boolean set = false;
private O value = null;
@Override
public O get() throws InterruptedException, ExecutionException {
return apply(futureListener.getFuture().get());
}
@Override
public O get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
return apply(futureListener.getFuture().get(timeout, unit));
}
private O apply(I raw) {
synchronized (lock) {
if (!set) {
value = function.apply(raw);
set = true;
}
return value;
}
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
return futureListener.getFuture().cancel(mayInterruptIfRunning);
}
@Override
public boolean isCancelled() {
return futureListener.getFuture().isCancelled();
}
@Override
public boolean isDone() {
return futureListener.getFuture().isDone();
}
@Override
public void addListener(Runnable listener, Executor exec) {
futureListener.addListener(listener, exec);
}
@Override
protected Object delegate() {
return futureListener.getFuture();
}
}
/**
* Just like {@code Futures#compose} except that we check the type of the executorService before
* creating the Future. If we are single threaded, invoke the function lazy as opposed to
* chaining, so that we don't invoke get() early.
*/
public static <I, O> ListenableFuture<O> compose(Future<I> future, final Function<? super I, ? extends O> function,
ExecutorService executorService) {
if (future instanceof Futures.ListenableFutureAdapter<?>) {
Futures.ListenableFutureAdapter<I> lf = (ListenableFutureAdapter<I>) future;
if (lf.futureListener.executor.getClass().isAnnotationPresent(SingleThreaded.class))
return Futures.LazyListenableFutureFunctionAdapter.create(
((org.jclouds.concurrent.Futures.ListenableFutureAdapter<I>) future).futureListener, function);
else
return com.google.common.util.concurrent.Futures.compose(lf, function, executorService);
} else if (executorService.getClass().isAnnotationPresent(SingleThreaded.class)) {
return Futures.LazyListenableFutureFunctionAdapter.create(future, function, executorService);
} else {
return com.google.common.util.concurrent.Futures.compose(Futures.makeListenable(future, executorService), function, executorService);
}
}
/**
* Just like {@code Futures#makeListenable} except that we pass in an executorService.
* <p/>
* Temporary hack until http://code.google.com/p/guava-libraries/issues/detail?id=317 is fixed.
*/
public static <T> ListenableFuture<T> makeListenable(Future<T> future, ExecutorService executorService) {
if (future instanceof ListenableFuture<?>) {
return (ListenableFuture<T>) future;
}
return ListenableFutureAdapter.create(future, executorService);
}
}

199
core/src/main/java/org/jclouds/concurrent/MoreExecutors.java Normal file
View File

@ -0,0 +1,199 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.concurrent;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.AbstractExecutorService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import com.google.common.annotations.Beta;
/**
* functions related to or replacing those in
* {@link com.google.common.util.concurrent.MoreExecutors}
*
* @author Adrian Cole
*/
@Beta
public class MoreExecutors {
/**
* Taken from @link com.google.common.util.concurrent.MoreExecutors} as it was hidden and
* therefore incapable of instanceof checks.
*
*
* Creates an executor service that runs each task in the thread that invokes {@code
* execute/submit}, as in {@link CallerRunsPolicy} This applies both to individually submitted
* tasks and to collections of tasks submitted via {@code invokeAll} or {@code invokeAny}. In the
* latter case, tasks will run serially on the calling thread. Tasks are run to completion before
* a {@code Future} is returned to the caller (unless the executor has been shutdown).
*
* <p>
* Although all tasks are immediately executed in the thread that submitted the task, this
* {@code ExecutorService} imposes a small locking overhead on each task submission in order to
* implement shutdown and termination behavior.
*
* <p>
* The implementation deviates from the {@code ExecutorService} specification with regards to the
* {@code shutdownNow} method. First, "best-effort" with regards to canceling running tasks is
* implemented as "no-effort". No interrupts or other attempts are made to stop threads executing
* tasks. Second, the returned list will always be empty, as any submitted task is considered to
* have started execution. This applies also to tasks given to {@code invokeAll} or {@code
* invokeAny} which are pending serial execution, even the subset of the tasks that have not yet
* started execution. It is unclear from the {@code ExecutorService} specification if these
* should be included, and it's much easier to implement the interpretation that they not be.
* Finally, a call to {@code shutdown} or {@code shutdownNow} may result in concurrent calls to
* {@code invokeAll/invokeAny} throwing RejectedExecutionException, although a subset of the
* tasks may already have been executed.
*/
public static ExecutorService sameThreadExecutor() {
return new SameThreadExecutorService();
}
// See sameThreadExecutor javadoc for behavioral notes.
@SingleThreaded
public static class SameThreadExecutorService extends AbstractExecutorService {
/**
* Lock used whenever accessing the state variables (runningTasks, shutdown,
* terminationCondition) of the executor
*/
private final Lock lock = new ReentrantLock();
/** Signaled after the executor is shutdown and running tasks are done */
private final Condition termination = lock.newCondition();
private SameThreadExecutorService() {
}
/*
* Conceptually, these two variables describe the executor being in one of three states: -
* Active: shutdown == false - Shutdown: runningTasks > 0 and shutdown == true - Terminated:
* runningTasks == 0 and shutdown == true
*/
private int runningTasks = 0;
private boolean shutdown = false;
@Override
public void execute(Runnable command) {
startTask();
try {
command.run();
} finally {
endTask();
}
}
@Override
public boolean isShutdown() {
lock.lock();
try {
return shutdown;
} finally {
lock.unlock();
}
}
@Override
public void shutdown() {
lock.lock();
try {
shutdown = true;
} finally {
lock.unlock();
}
}
// See sameThreadExecutor javadoc for unusual behavior of this method.
@Override
public List<Runnable> shutdownNow() {
shutdown();
return Collections.emptyList();
}
@Override
public boolean isTerminated() {
lock.lock();
try {
return shutdown && runningTasks == 0;
} finally {
lock.unlock();
}
}
@Override
public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
long nanos = unit.toNanos(timeout);
lock.lock();
try {
for (;;) {
if (isTerminated()) {
return true;
} else if (nanos <= 0) {
return false;
} else {
nanos = termination.awaitNanos(nanos);
}
}
} finally {
lock.unlock();
}
}
/**
* Checks if the executor has been shut down and increments the running task count.
*
* @throws RejectedExecutionException
* if the executor has been previously shutdown
*/
private void startTask() {
lock.lock();
try {
if (isShutdown()) {
throw new RejectedExecutionException("Executor already shutdown");
}
runningTasks++;
} finally {
lock.unlock();
}
}
/**
* Decrements the running task count.
*/
private void endTask() {
lock.lock();
try {
runningTasks--;
if (isTerminated()) {
termination.signalAll();
}
} finally {
lock.unlock();
}
}
}
}

6
core/src/main/java/org/jclouds/concurrent/config/ExecutorServiceModule.java
View File

@ -32,7 +32,7 @@ import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import static org.jclouds.concurrent.ConcurrentUtils.*;
import org.jclouds.concurrent.MoreExecutors;
import org.jclouds.concurrent.SingleThreaded;
import org.jclouds.lifecycle.Closer;
import org.jclouds.logging.Logger;
@ -87,8 +87,8 @@ public class ExecutorServiceModule extends AbstractModule {
&& executor.getClass().getSimpleName().indexOf("SameThread") != -1) {
Logger.CONSOLE.warn(
"please switch from %s to %s or annotate your same threaded executor with @SingleThreaded", executor
.getClass().getName(), SameThreadExecutorService.class.getName());
return sameThreadExecutor();
.getClass().getName(), MoreExecutors.SameThreadExecutorService.class.getName());
return MoreExecutors.sameThreadExecutor();
}
return executor;
}

60
core/src/main/java/org/jclouds/crypto/Crypto.java Normal file
View File

@ -0,0 +1,60 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.crypto;
import java.security.InvalidKeyException;
import java.security.KeyFactory;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateFactory;
import javax.crypto.Mac;
import org.jclouds.encryption.internal.JCECrypto;
import com.google.inject.ImplementedBy;
/**
* Allows you to access cryptographic objects and factories without adding a provider to the JCE
* runtime.
*
* @author Adrian Cole
*/
@ImplementedBy(JCECrypto.class)
public interface Crypto {
KeyFactory rsaKeyFactory();
CertificateFactory certFactory();
Mac hmac(String algorithm, byte[] key) throws NoSuchAlgorithmException, InvalidKeyException;
Mac hmacSHA256(byte[] key) throws InvalidKeyException;
Mac hmacSHA1(byte[] key) throws InvalidKeyException;
MessageDigest digest(String algorithm) throws NoSuchAlgorithmException;
MessageDigest md5();
MessageDigest sha1();
MessageDigest sha256();
}

318
core/src/main/java/org/jclouds/crypto/CryptoStreams.java Normal file
View File

@ -0,0 +1,318 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.crypto;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import javax.crypto.Mac;
import org.jclouds.encryption.internal.Base64;
import org.jclouds.io.InputSuppliers;
import com.google.common.annotations.Beta;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.io.ByteProcessor;
import com.google.common.io.ByteStreams;
import com.google.common.io.InputSupplier;
/**
* functions related to but not in {@link com.google.common.io.ByteStreams}
*
* @author Adrian Cole
*/
@Beta
public class CryptoStreams {
public static String hex(byte[] in) {
byte[] hex = new byte[2 * in.length];
int index = 0;
for (byte b : in) {
int v = b & 0xFF;
hex[index++] = HEX_CHAR_TABLE[v >>> 4];
hex[index++] = HEX_CHAR_TABLE[v & 0xF];
}
try {
return new String(hex, "ASCII");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
public static byte[] hex(String s) {
int len = s.length();
byte[] data = new byte[len / 2];
for (int i = 0; i < len; i += 2) {
data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + Character.digit(s.charAt(i + 1), 16));
}
return data;
}
public static String base64(byte[] in) {
return Base64.encodeBytes(in, Base64.DONT_BREAK_LINES);
}
public static byte[] base64(String in) {
return Base64.decode(in);
}
/**
* @see #md5
* @see #hex
*/
public static String md5Hex(InputSupplier<? extends InputStream> supplier) throws IOException {
return hex(md5(supplier));
}
/**
* @see #md5
* @see #base64
*/
public static String md5Base64(InputSupplier<? extends InputStream> supplier) throws IOException {
return base64(md5(supplier));
}
/**
* Computes and returns the MAC value for a supplied input stream. The mac object is reset when
* this method returns successfully.
*
* @param supplier
* the input stream factory
* @param mac
* the mac object
* @return the result of {@link Mac#doFinal()} after updating the mac object with all of the
* bytes in the stream and encoding in Base64
* @throws IOException
* if an I/O error occurs
*/
public static String macBase64(InputSupplier<? extends InputStream> supplier, final Mac mac) throws IOException {
return base64(mac(supplier, mac));
}
/**
* Computes and returns the Digest value for a supplied input stream. The digest object is reset
* when this method returns successfully.
*
* @param supplier
* the input stream factory
* @param md
* the digest object
* @return the result of {@link MessageDigest#digest()} after updating the digest object with all
* of the bytes in the stream
* @throws IOException
* if an I/O error occurs
*/
public static byte[] digest(InputSupplier<? extends InputStream> supplier, final MessageDigest md)
throws IOException {
return com.google.common.io.ByteStreams.readBytes(supplier, new ByteProcessor<byte[]>() {
public boolean processBytes(byte[] buf, int off, int len) {
md.update(buf, off, len);
return true;
}
public byte[] getResult() {
return md.digest();
}
});
}
/**
* Computes and returns the MD5 value for a supplied input stream. A digest object is created and
* disposed of at runtime, consider using {@link #digest} to be more efficient.
*
* @param supplier
* the input stream factory
*
* @return the result of {@link MessageDigest#digest()} after updating the md5 object with all of
* the bytes in the stream
* @throws IOException
* if an I/O error occurs
*/
public static byte[] md5(InputSupplier<? extends InputStream> supplier) throws IOException {
try {
return digest(supplier, MessageDigest.getInstance("MD5"));
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
return null;
}
}
public static byte[] md5(byte[] in) throws IOException {
return md5(ByteStreams.newInputStreamSupplier(in));
}
/**
* Computes and returns the MAC value for a supplied input stream. The mac object is reset when
* this method returns successfully.
*
* @param supplier
* the input stream factory
* @param mac
* the mac object
* @return the result of {@link Mac#doFinal()} after updating the mac object with all of the
* bytes in the stream
* @throws IOException
* if an I/O error occurs
*/
public static byte[] mac(InputSupplier<? extends InputStream> supplier, final Mac mac) throws IOException {
return com.google.common.io.ByteStreams.readBytes(checkNotNull(supplier, "supplier"),
new ByteProcessor<byte[]>() {
public boolean processBytes(byte[] buf, int off, int len) {
mac.update(buf, off, len);
return true;
}
public byte[] getResult() {
return mac.doFinal();
}
});
}
/**
* Computes and returns the base64 value for a supplied input stream.
*
* @param supplier
* the input stream factory
*
* @return the result of base 64 encoding all of the bytes in the stream
* @throws IOException
* if an I/O error occurs
*/
public static String base64Encode(InputSupplier<? extends InputStream> supplier) throws IOException {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
return com.google.common.io.ByteStreams.readBytes(InputSuppliers.base64Encoder(supplier),
new ByteProcessor<String>() {
public boolean processBytes(byte[] buf, int off, int len) {
out.write(buf, off, len);
return true;
}
public String getResult() {
return new String(out.toByteArray(), Charsets.UTF_8);
}
});
}
/**
* Computes and returns the unencoded value for an input stream which is encoded in Base64.
*
* @param supplier
* the input stream factory
*
* @return the result of base 64 decoding all of the bytes in the stream
* @throws IOException
* if an I/O error occurs
*/
public static byte[] base64Decode(InputSupplier<? extends InputStream> supplier) throws IOException {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
return com.google.common.io.ByteStreams.readBytes(InputSuppliers.base64Decoder(supplier),
new ByteProcessor<byte[]>() {
public boolean processBytes(byte[] buf, int off, int len) {
out.write(buf, off, len);
return true;
}
public byte[] getResult() {
return out.toByteArray();
}
});
}
final static byte[] HEX_CHAR_TABLE = { (byte) '0', (byte) '1', (byte) '2', (byte) '3', (byte) '4', (byte) '5',
(byte) '6', (byte) '7', (byte) '8', (byte) '9', (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e',
(byte) 'f' };
/**
* Computes and returns the hex value for a supplied input stream.
*
* @param supplier
* the input stream factory
*
* @return the result of hex encoding all of the bytes in the stream
* @throws IOException
* if an I/O error occurs
*/
public static String hexEncode(InputSupplier<? extends InputStream> supplier) throws IOException {
final StringBuilder out = new StringBuilder();
return com.google.common.io.ByteStreams.readBytes(supplier, new ByteProcessor<String>() {
public boolean processBytes(byte[] buf, int off, int len) {
char[] hex = new char[2 * len];
int index = 0;
for (int i = off; i < off + len; i++) {
byte b = buf[i];
int v = b & 0xFF;
hex[index++] = (char) HEX_CHAR_TABLE[v >>> 4];
hex[index++] = (char) HEX_CHAR_TABLE[v & 0xF];
}
out.append(hex);
return true;
}
public String getResult() {
return out.toString();
}
});
}
/**
* Computes and returns the unencoded value for an input stream which is encoded in hex.
*
* @param supplier
* the input stream factory
*
* @return the result of hex decoding all of the bytes in the stream
* @throws IOException
* if an I/O error occurs
*/
public static byte[] hexDecode(InputSupplier<? extends InputStream> supplier) throws IOException {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
return com.google.common.io.ByteStreams.readBytes(supplier, new ByteProcessor<byte[]>() {
int currentPos = 0;
public boolean processBytes(byte[] buf, int off, int len) {
try {
if (currentPos == 0 && new String(Arrays.copyOfRange(buf, off, 2), "ASCII").equals("0x")) {
off += 2;
}
byte[] decoded = hex(new String(Arrays.copyOfRange(buf, off, len), "ASCII"));
out.write(decoded, 0, decoded.length);
currentPos += len;
} catch (UnsupportedEncodingException e) {
throw new IllegalStateException("ASCII must be supported");
}
return true;
}
public byte[] getResult() {
return out.toByteArray();
}
});
}
}

252
core/src/main/java/org/jclouds/crypto/Pems.java Normal file
View File

@ -0,0 +1,252 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.crypto;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.cert.CertificateEncodingException;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.spec.KeySpec;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.RSAPrivateKeySpec;
import java.security.spec.X509EncodedKeySpec;
import java.util.Map;
import javax.annotation.Nullable;
import net.oauth.signature.pem.PEMReader;
import net.oauth.signature.pem.PKCS1EncodedKeySpec;
import org.jclouds.crypto.Pems.PemProcessor.ResultParser;
import com.google.common.annotations.Beta;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteProcessor;
import com.google.common.io.ByteStreams;
import com.google.common.io.InputSupplier;
/**
* Reads and writes PEM encoded Strings and Streams
*
* @author Adrian Cole
*/
@Beta
public class Pems {
public static class PemProcessor<T> implements com.google.common.io.ByteProcessor<T> {
public interface ResultParser<T> {
T parseResult(byte[] bytes) throws IOException;
}
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
private final Map<String, ResultParser<T>> parsers;
public PemProcessor(Map<String, ResultParser<T>> parsers) {
this.parsers = checkNotNull(parsers, "parsers");
}
public boolean processBytes(byte[] buf, int off, int len) {
out.write(buf, off, len);
return true;
}
public T getResult() {
try {
PEMReader reader = new PEMReader(out.toByteArray());
byte[] bytes = reader.getDerBytes();
if (parsers.containsKey(reader.getBeginMarker())) {
return parsers.get(reader.getBeginMarker()).parseResult(bytes);
} else {
throw new IOException(String.format("Invalid PEM file: no parsers for marker %s in %s", reader
.getBeginMarker(), parsers.keySet()));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
/**
* Returns the object of generic type {@code T} that is pem encoded in the supplier.
*
* @param supplier
* the input stream factory
* @param marker
* header that begins the PEM block
* @param processor
* how to parser the object from a byte array
* @return the object of generic type {@code T} which was PEM encoded in the stream
* @throws IOException
* if an I/O error occurs
*/
public static <T> T fromPem(InputSupplier<? extends InputStream> supplier, PemProcessor<T> processor)
throws IOException {
try {
return com.google.common.io.ByteStreams.readBytes(supplier, processor);
} catch (RuntimeException e) {
if (e.getCause() != null && e.getCause() instanceof IOException) {
throw (IOException) e.getCause();
}
throw e;
}
}
/**
* Returns the {@link RSAPrivateKeySpec} that is pem encoded in the supplier.
*
* @param supplier
* the input stream factory
*
* @return the {@link RSAPrivateKeySpec} which was PEM encoded in the stream
* @throws IOException
* if an I/O error occurs
*/
public static KeySpec privateKeySpec(InputSupplier<? extends InputStream> supplier) throws IOException {
return fromPem(supplier, new PemProcessor<KeySpec>(ImmutableMap.<String, ResultParser<KeySpec>> of(
PEMReader.PRIVATE_PKCS1_MARKER, new ResultParser<KeySpec>() {
public KeySpec parseResult(byte[] bytes) throws IOException {
return (new PKCS1EncodedKeySpec(bytes)).getKeySpec();
}
}, PEMReader.PRIVATE_PKCS8_MARKER, new ResultParser<KeySpec>() {
public KeySpec parseResult(byte[] bytes) throws IOException {
return new PKCS8EncodedKeySpec(bytes);
}
})));
}
/**
* Returns the {@link X509EncodedKeySpec} that is pem encoded in the supplier.
*
* @param supplier
* the input stream factory
*
* @return the {@link X509EncodedKeySpec} which was PEM encoded in the stream
* @throws IOException
* if an I/O error occurs
*/
public static X509EncodedKeySpec publicKeySpec(InputSupplier<? extends InputStream> supplier) throws IOException {
return fromPem(supplier, new PemProcessor<X509EncodedKeySpec>(ImmutableMap
.<String, ResultParser<X509EncodedKeySpec>> of(PEMReader.PUBLIC_X509_MARKER,
new ResultParser<X509EncodedKeySpec>() {
public X509EncodedKeySpec parseResult(byte[] bytes) throws IOException {
return new X509EncodedKeySpec(bytes);
}
})));
}
/**
* Returns the {@link X509EncodedKeySpec} that is pem encoded in the supplier.
*
* @param supplier
* the input stream factory
* @param certFactory
* or null to use default
*
* @return the {@link X509EncodedKeySpec} which was PEM encoded in the stream
* @throws IOException
* if an I/O error occurs
* @throws CertificateException
*/
public static X509Certificate x509Certificate(InputSupplier<? extends InputStream> supplier,
@Nullable CertificateFactory certFactory) throws IOException, CertificateException {
final CertificateFactory finalCertFactory = certFactory != null ? certFactory : CertificateFactory
.getInstance("X.509");
try {
return fromPem(supplier, new PemProcessor<X509Certificate>(ImmutableMap
.<String, ResultParser<X509Certificate>> of(PEMReader.CERTIFICATE_X509_MARKER,
new ResultParser<X509Certificate>() {
public X509Certificate parseResult(byte[] bytes) throws IOException {
try {
return (X509Certificate) finalCertFactory
.generateCertificate(new ByteArrayInputStream(bytes));
} catch (CertificateException e) {
throw new RuntimeException(e);
}
}
})));
} catch (RuntimeException e) {
if (e.getCause() != null && e.getCause() instanceof CertificateException) {
throw (CertificateException) e.getCause();
}
throw e;
}
}
/**
* encodes the {@link X509Certificate} to PEM format.
*
* @param cert
* what to encode
* @return the PEM encoded certificate
* @throws IOException
* @throws CertificateEncodingException
*/
public static String pem(X509Certificate cert) throws IOException, CertificateEncodingException {
return new StringBuilder("-----BEGIN CERTIFICATE-----\n").append(
CryptoStreams.base64Encode(ByteStreams.newInputStreamSupplier(cert.getEncoded()))).append(
"\n-----END CERTIFICATE-----\n").toString();
}
/**
* encodes the {@link PublicKey} to PEM format.
*
* @param cert
* what to encode
* @return the PEM encoded public key
* @throws IOException
* @throws CertificateEncodingException
*/
public static String pem(PublicKey key) throws IOException {
return new StringBuilder("-----BEGIN PUBLIC KEY-----\n").append(
CryptoStreams.base64Encode(ByteStreams.newInputStreamSupplier(key.getEncoded()))).append(
"\n-----END PUBLIC KEY-----\n").toString();
}
/**
* encodes the {@link PrivateKey} to PEM format. Note
*
* @param cert
* what to encode
* @return the PEM encoded private key
* @throws IOException
* @throws CertificateEncodingException
*/
public static String pem(PrivateKey key) throws IOException {
return new StringBuilder("-----BEGIN PRIVATE KEY-----\n").append(
CryptoStreams.base64Encode(ByteStreams.newInputStreamSupplier(key.getEncoded()))).append(
"\n-----END PRIVATE KEY-----\n").toString();
}
}

102
core/src/main/java/org/jclouds/encryption/EncryptionService.java
View File

@ -1,102 +0,0 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.encryption;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.Key;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.cert.X509Certificate;
import org.jclouds.encryption.internal.JCEEncryptionService;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.io.Payload;
import org.jclouds.io.payloads.ByteArrayPayload;
import com.google.inject.ImplementedBy;
/**
*
* @author Adrian Cole
*/
@ImplementedBy(JCEEncryptionService.class)
public interface EncryptionService {
String base64(byte[] toEncode);
byte[] fromBase64(String encoded);
String hex(byte[] toEncode);
byte[] fromHex(String encoded);
byte[] rsaEncrypt(Payload payload, Key key);
byte[] hmacSha256(String toEncode, byte[] key);
byte[] hmacSha1(String toEncode, byte[] key);
byte[] sha1(InputStream toEncode);
byte[] sha256(InputStream toEncode);
byte[] md5(InputStream toEncode);
/**
* generate an MD5 Hash for the current data.
* <p/>
* <h2>Note</h2>
* <p/>
* If this is an InputStream, it will be converted to a byte array first.
*
* @throws IOException
*/
<T extends PayloadEnclosing> T generateMD5BufferingIfNotRepeatable(T payloadEnclosing);
Payload generateMD5BufferingIfNotRepeatable(Payload in);
ByteArrayPayload generatePayloadWithMD5For(InputStream toEncode);
MD5OutputStream md5OutputStream(OutputStream out);
PrivateKey privateKeyFromPEM(byte[] pem);
public static abstract class MD5OutputStream extends FilterOutputStream {
public MD5OutputStream(OutputStream out) {
super(out);
}
public abstract byte[] getMD5();
}
PublicKey publicKeyFromPEM(byte[] pem);
X509Certificate x509CertificateFromPEM(byte[] pem);
byte[] rsaDecrypt(Payload payload, Key key);
String toPem(X509Certificate cert);
String toPem(PublicKey key);
String toPem(PrivateKey key);
}

299
core/src/main/java/org/jclouds/encryption/internal/BaseEncryptionService.java
View File

@ -1,299 +0,0 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.encryption.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Throwables.propagate;
import static com.google.common.io.Closeables.closeQuietly;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.Key;
import java.security.KeyFactory;
import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.cert.CertificateEncodingException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.spec.KeySpec;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.X509EncodedKeySpec;
import java.util.Arrays;
import javax.annotation.Resource;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.ShortBufferException;
import net.oauth.signature.pem.PEMReader;
import net.oauth.signature.pem.PKCS1EncodedKeySpec;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.io.Payload;
import org.jclouds.logging.Logger;
import com.google.common.base.Throwables;
/**
*
* @author Adrian Cole
*/
public abstract class BaseEncryptionService implements EncryptionService {
@Resource
protected Logger logger = Logger.NULL;
protected static final int BUF_SIZE = 0x2000; // 8
final byte[] HEX_CHAR_TABLE = { (byte) '0', (byte) '1', (byte) '2', (byte) '3', (byte) '4', (byte) '5', (byte) '6',
(byte) '7', (byte) '8', (byte) '9', (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f' };
private final KeyFactory rsaKeyFactory;
private final CertificateFactory certFactory;
public BaseEncryptionService(KeyFactory rsaKeyFactory, CertificateFactory certFactory) {
this.rsaKeyFactory = rsaKeyFactory;
this.certFactory = certFactory;
}
@Override
public String hex(byte[] raw) {
byte[] hex = new byte[2 * raw.length];
int index = 0;
for (byte b : raw) {
int v = b & 0xFF;
hex[index++] = HEX_CHAR_TABLE[v >>> 4];
hex[index++] = HEX_CHAR_TABLE[v & 0xF];
}
try {
return new String(hex, "ASCII");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
@Override
public byte[] fromHex(String hex) {
if (hex.startsWith("0x"))
hex = hex.substring(2);
byte[] bytes = new byte[hex.length() / 2];
for (int i = 0; i < bytes.length; i++) {
bytes[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16);
}
return bytes;
}
@Override
public Payload generateMD5BufferingIfNotRepeatable(Payload payload) {
checkNotNull(payload, "payload");
if (!payload.isRepeatable()) {
String oldContentType = payload.getContentType();
payload = generatePayloadWithMD5For(payload.getInput());
payload.setContentType(oldContentType);
} else {
payload.setContentMD5(md5(payload.getInput()));
}
return payload;
}
/**
* {@inheritDoc}
*/
@Override
public <T extends PayloadEnclosing> T generateMD5BufferingIfNotRepeatable(T payloadEnclosing) {
checkState(payloadEnclosing != null, "payloadEnclosing");
Payload newPayload = generateMD5BufferingIfNotRepeatable(payloadEnclosing.getPayload());
if (newPayload != payloadEnclosing.getPayload())
payloadEnclosing.setPayload(newPayload);
return payloadEnclosing;
}
/**
* {@inheritDoc}
*/
@Override
public PrivateKey privateKeyFromPEM(byte[] pem) {
PEMReader reader;
try {
reader = new PEMReader(pem);
byte[] bytes = reader.getDerBytes();
KeySpec keySpec;
if (PEMReader.PRIVATE_PKCS1_MARKER.equals(reader.getBeginMarker())) {
keySpec = (new PKCS1EncodedKeySpec(bytes)).getKeySpec();
} else if (PEMReader.PRIVATE_PKCS8_MARKER.equals(reader.getBeginMarker())) {
keySpec = new PKCS8EncodedKeySpec(bytes);
} else {
throw new IOException("Invalid PEM file: Unknown marker for private key " + reader.getBeginMarker());
}
return rsaKeyFactory.generatePrivate(keySpec);
} catch (Exception e) {
Throwables.propagate(e);
return null;
}
}
/**
* {@inheritDoc}
*/
@Override
public PublicKey publicKeyFromPEM(byte[] pem) {
PEMReader reader;
try {
reader = new PEMReader(pem);
byte[] bytes = reader.getDerBytes();
KeySpec keySpec;
if (PEMReader.PUBLIC_X509_MARKER.equals(reader.getBeginMarker())) {
keySpec = new X509EncodedKeySpec(bytes);
} else {
throw new IOException("Invalid PEM file: Unknown marker for public key " + reader.getBeginMarker());
}
return rsaKeyFactory.generatePublic(keySpec);
} catch (Exception e) {
Throwables.propagate(e);
return null;
}
}
/**
* {@inheritDoc}
*/
@Override
public X509Certificate x509CertificateFromPEM(byte[] pem) {
PEMReader reader;
try {
reader = new PEMReader(pem);
byte[] bytes = reader.getDerBytes();
if (PEMReader.CERTIFICATE_X509_MARKER.equals(reader.getBeginMarker())) {
return (X509Certificate) certFactory.generateCertificate(new ByteArrayInputStream(bytes));
} else {
throw new IOException("Invalid PEM file: Unknown marker for public key " + reader.getBeginMarker());
}
} catch (Exception e) {
Throwables.propagate(e);
return null;
}
}
@Override
public byte[] rsaEncrypt(Payload payload, Key key) {
// TODO convert this to BC code
Cipher cipher = null;
try {
cipher = Cipher.getInstance("RSA");
cipher.init(Cipher.ENCRYPT_MODE, key);
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
} catch (NoSuchPaddingException e) {
Throwables.propagate(e);
} catch (InvalidKeyException e) {
Throwables.propagate(e);
}
return cipherPayload(cipher, payload);
}
@Override
public byte[] rsaDecrypt(Payload payload, Key key) {
// TODO convert this to BC code
Cipher cipher = null;
try {
cipher = Cipher.getInstance("RSA");
cipher.init(Cipher.DECRYPT_MODE, key);
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
} catch (NoSuchPaddingException e) {
Throwables.propagate(e);
} catch (InvalidKeyException e) {
Throwables.propagate(e);
}
return cipherPayload(cipher, payload);
}
private byte[] cipherPayload(Cipher cipher, Payload payload) {
byte[] resBuf = new byte[cipher.getOutputSize(payload.getContentLength().intValue())];
byte[] buffer = new byte[BUF_SIZE];
long length = 0;
int numRead = -1;
InputStream plainBytes = payload.getInput();
try {
do {
numRead = plainBytes.read(buffer);
if (numRead > 0) {
length += numRead;
cipher.update(buffer, 0, numRead);
}
} while (numRead != -1);
} catch (IOException e) {
propagate(e);
} finally {
closeQuietly(plainBytes);
}
try {
int size = cipher.doFinal(resBuf, 0);
return Arrays.copyOfRange(resBuf, 0, size);
} catch (IllegalBlockSizeException e) {
Throwables.propagate(e);
} catch (ShortBufferException e) {
Throwables.propagate(e);
} catch (BadPaddingException e) {
Throwables.propagate(e);
}
assert false;
return null;
}
@Override
public String toPem(X509Certificate cert) {
try {
return new StringBuilder("-----BEGIN CERTIFICATE-----\n").append(base64(cert.getEncoded())).append(
"\n-----END CERTIFICATE-----\n").toString();
} catch (CertificateEncodingException e) {
Throwables.propagate(e);
return null;
}
}
@Override
public String toPem(PublicKey key) {
return new StringBuilder("-----BEGIN PUBLIC KEY-----\n").append(base64(key.getEncoded())).append(
"\n-----END PUBLIC KEY-----\n").toString();
}
@Override
public String toPem(PrivateKey key) {
return new StringBuilder("-----BEGIN PRIVATE KEY-----\n").append(base64(key.getEncoded())).append(
"\n-----END PRIVATE KEY-----\n").toString();
}
}

135
core/src/main/java/org/jclouds/encryption/internal/JCECrypto.java Executable file
View File

@ -0,0 +1,135 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.encryption.internal;
import java.security.InvalidKeyException;
import java.security.KeyFactory;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.Provider;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import javax.annotation.Nullable;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.crypto.Crypto;
/**
*
* @author Adrian Cole
*/
@Singleton
public class JCECrypto implements Crypto {
private final KeyFactory rsaKeyFactory;
private final CertificateFactory certFactory;
private final Provider provider;
@Inject
public JCECrypto() throws NoSuchAlgorithmException, CertificateException {
this(null);
}
public JCECrypto(@Nullable Provider provider) throws NoSuchAlgorithmException, CertificateException {
this.rsaKeyFactory = provider == null ? KeyFactory.getInstance("RSA") : KeyFactory.getInstance("RSA", provider);
this.certFactory = provider == null ? CertificateFactory.getInstance("X.509") : CertificateFactory.getInstance(
"X.509", provider);
this.provider = provider;
}
@Override
public Mac hmac(String algorithm, byte[] key) throws NoSuchAlgorithmException, InvalidKeyException {
Mac mac = provider == null ? Mac.getInstance(algorithm) : Mac.getInstance(algorithm, provider);
SecretKeySpec signingKey = new SecretKeySpec(key, algorithm);
mac.init(signingKey);
return mac;
}
@Override
public MessageDigest digest(String algorithm) throws NoSuchAlgorithmException {
return provider == null ? MessageDigest.getInstance(algorithm) : MessageDigest.getInstance(algorithm, provider);
}
public final static String MD5 = "MD5";
public final static String SHA1 = "SHA1";
public final static String SHA256 = "SHA256";
@Override
public MessageDigest md5() {
try {
return digest(MD5);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("MD5 must be supported", e);
}
}
@Override
public MessageDigest sha1() {
try {
return digest(SHA1);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("MD5 must be supported", e);
}
}
@Override
public MessageDigest sha256() {
try {
return digest(SHA256);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA256 must be supported", e);
}
}
public final static String HmacSHA256 = "HmacSHA256";
public final static String HmacSHA1 = "HmacSHA1";
@Override
public Mac hmacSHA1(byte[] key) throws InvalidKeyException {
try {
return hmac(HmacSHA1, key);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("HmacSHA1 must be supported", e);
}
}
@Override
public Mac hmacSHA256(byte[] key) throws InvalidKeyException {
try {
return hmac(HmacSHA256, key);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("HmacSHA256 must be supported", e);
}
}
@Override
public CertificateFactory certFactory() {
return certFactory;
}
@Override
public KeyFactory rsaKeyFactory() {
return rsaKeyFactory;
}
}

202
core/src/main/java/org/jclouds/encryption/internal/JCEEncryptionService.java
View File

@ -1,202 +0,0 @@
/**
*
* Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.encryption.internal;
import static com.google.common.base.Throwables.propagate;
import static com.google.common.io.Closeables.closeQuietly;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.DigestOutputStream;
import java.security.InvalidKeyException;
import java.security.KeyFactory;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.jclouds.io.payloads.ByteArrayPayload;
/**
*
* @author Adrian Cole
*/
public class JCEEncryptionService extends BaseEncryptionService {
public JCEEncryptionService(KeyFactory rsaKeyFactory, CertificateFactory certFactory) {
super(rsaKeyFactory, certFactory);
}
public JCEEncryptionService() throws NoSuchAlgorithmException, CertificateException {
this(KeyFactory.getInstance("RSA"), CertificateFactory.getInstance("X.509"));
}
@Override
public byte[] hmacSha256(String toEncode, byte[] key) {
return hmac(toEncode, key, "HmacSHA256");
}
@Override
public byte[] hmacSha1(String toEncode, byte[] key) {
return hmac(toEncode, key, "HmacSHA1");
}
public byte[] hmac(String toEncode, byte[] key, String algorithm) {
SecretKeySpec signingKey = new SecretKeySpec(key, algorithm);
Mac mac = null;
try {
mac = Mac.getInstance(algorithm);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException("Could not find the " + algorithm + " algorithm", e);
}
try {
mac.init(signingKey);
} catch (InvalidKeyException e) {
throw new RuntimeException("Could not initialize the " + algorithm + " algorithm", e);
}
return mac.doFinal(toEncode.getBytes());
}
@Override
public byte[] md5(InputStream toEncode) {
MessageDigest eTag = getDigest();
byte[] buffer = new byte[BUF_SIZE];
int numRead = -1;
try {
do {
numRead = toEncode.read(buffer);
if (numRead > 0) {
eTag.update(buffer, 0, numRead);
}
} while (numRead != -1);
} catch (IOException e) {
propagate(e);
} finally {
closeQuietly(toEncode);
}
return eTag.digest();
}
@Override
public String base64(byte[] resBuf) {
return Base64.encodeBytes(resBuf, Base64.DONT_BREAK_LINES);
}
@Override
public byte[] fromBase64(String encoded) {
return Base64.decode(encoded);
}
@Override
public ByteArrayPayload generatePayloadWithMD5For(InputStream toEncode) {
MessageDigest eTag = getDigest();
byte[] buffer = new byte[BUF_SIZE];
ByteArrayOutputStream out = new ByteArrayOutputStream();
long length = 0;
int numRead = -1;
try {
do {
numRead = toEncode.read(buffer);
if (numRead > 0) {
length += numRead;
eTag.update(buffer, 0, numRead);
out.write(buffer, 0, numRead);
}
} while (numRead != -1);
} catch (IOException e) {
propagate(e);
} finally {
closeQuietly(out);
closeQuietly(toEncode);
}
return new ByteArrayPayload(out.toByteArray(), eTag.digest());
}
private static MessageDigest getDigest() {
MessageDigest eTag;
try {
eTag = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException("Could not find the MD5 algorithm", e);
}
return eTag;
}
@Override
public MD5OutputStream md5OutputStream(OutputStream out) {
return new JCEMD5OutputStream(out);
}
private static class JCEMD5OutputStream extends MD5OutputStream {
public JCEMD5OutputStream(OutputStream out) {
super(new DigestOutputStream(out, getDigest()));
}
@Override
public byte[] getMD5() {
MessageDigest digest = ((DigestOutputStream) out).getMessageDigest();
return digest.digest();
}
}
@Override
public byte[] sha1(InputStream plainBytes) {
return digest(plainBytes, "SHA1");
}
@Override
public byte[] sha256(InputStream plainBytes) {
return digest(plainBytes, "SHA256");
}
private byte[] digest(InputStream plainBytes, String algorithm) {
MessageDigest digest;
try {
digest = MessageDigest.getInstance(algorithm);
} catch (NoSuchAlgorithmException e1) {
propagate(e1);
return null;
}
byte[] buffer = new byte[BUF_SIZE];
long length = 0;
int numRead = -1;
try {
do {
numRead = plainBytes.read(buffer);
if (numRead > 0) {
length += numRead;
digest.update(buffer, 0, numRead);
}
} while (numRead != -1);
} catch (IOException e) {
propagate(e);
} finally {
closeQuietly(plainBytes);
}
return digest.digest();
}
}

91
core/src/main/java/org/jclouds/http/HttpUtils.java
View File

@ -67,8 +67,10 @@ import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.UriBuilder;
import org.jclouds.Constants;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.io.InputSuppliers;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadEnclosing;
import org.jclouds.io.Payloads;
import org.jclouds.logging.Logger;
import org.jclouds.logging.internal.Wire;
@ -96,13 +98,12 @@ public class HttpUtils {
@Inject(optional = true)
@Named(Constants.PROPERTY_PROXY_SYSTEM)
private boolean systemProxies = System.getProperty("java.net.useSystemProxies") != null ? Boolean
.parseBoolean(System.getProperty("java.net.useSystemProxies")) : false;
.parseBoolean(System.getProperty("java.net.useSystemProxies")) : false;
private final int globalMaxConnections;
private final int globalMaxConnectionsPerHost;
private final int connectionTimeout;
private final int soTimeout;
private final EncryptionService encryptionService;
@Inject(optional = true)
@Named(Constants.PROPERTY_PROXY_HOST)
private String proxyHost;
@ -117,12 +118,10 @@ public class HttpUtils {
private String proxyPassword;
@Inject
public HttpUtils(EncryptionService encryptionService,
@Named(Constants.PROPERTY_CONNECTION_TIMEOUT) int connectionTimeout,
@Named(Constants.PROPERTY_SO_TIMEOUT) int soTimeout,
@Named(Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT) int globalMaxConnections,
@Named(Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST) int globalMaxConnectionsPerHost) {
this.encryptionService = encryptionService;
public HttpUtils(@Named(Constants.PROPERTY_CONNECTION_TIMEOUT) int connectionTimeout,
@Named(Constants.PROPERTY_SO_TIMEOUT) int soTimeout,
@Named(Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT) int globalMaxConnections,
@Named(Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST) int globalMaxConnectionsPerHost) {
this.soTimeout = soTimeout;
this.connectionTimeout = connectionTimeout;
this.globalMaxConnections = globalMaxConnections;
@ -182,8 +181,8 @@ public class HttpUtils {
}
/**
* keys to the map are only used for socket information, not path. In this
* case, you should remove any path or query details from the URI.
* keys to the map are only used for socket information, not path. In this case, you should
* remove any path or query details from the URI.
*/
public static URI createBaseEndpointFor(URI endpoint) {
if (endpoint.getPort() == -1) {
@ -194,8 +193,7 @@ public class HttpUtils {
}
/**
* Web browsers do not always handle '+' characters well, use the
* well-supported '%20' instead.
* Web browsers do not always handle '+' characters well, use the well-supported '%20' instead.
*/
public static String urlEncode(String in, char... skipEncode) {
if (isUrlEncoded(in))
@ -241,8 +239,7 @@ public class HttpUtils {
}
/**
* Content stream may need to be read. However, we should always close the
* http stream.
* Content stream may need to be read. However, we should always close the http stream.
*
* @throws IOException
*/
@ -262,11 +259,11 @@ public class HttpUtils {
String scheme = redirectURI.getScheme();
checkState(redirectURI.getScheme().startsWith("http"), String.format(
"header %s didn't parse an http scheme: [%s]", hostHeader, scheme));
"header %s didn't parse an http scheme: [%s]", hostHeader, scheme));
int port = redirectURI.getPort() > 0 ? redirectURI.getPort() : redirectURI.getScheme().equals("https") ? 443 : 80;
String host = redirectURI.getHost();
checkState(host.indexOf('/') == -1, String.format("header %s didn't parse an http host correctly: [%s]",
hostHeader, host));
hostHeader, host));
URI endPoint = URI.create(String.format("%s://%s:%d", scheme, host, port));
return endPoint;
}
@ -276,11 +273,10 @@ public class HttpUtils {
}
/**
* Used to extract the URI and authentication data from a String. Note that
* the java URI class breaks, if there are special characters like '/'
* present. Otherwise, we wouldn't need this class, and we could simply use
* URI.create("uri").getUserData(); Also, URI breaks if there are curly
* braces.
* Used to extract the URI and authentication data from a String. Note that the java URI class
* breaks, if there are special characters like '/' present. Otherwise, we wouldn't need this
* class, and we could simply use URI.create("uri").getUserData(); Also, URI breaks if there are
* curly braces.
*
*/
public static URI createUri(String uriPath) {
@ -339,8 +335,12 @@ public class HttpUtils {
if (message.getPayload().getContentLength() != null)
logger.debug("%s %s: %s", prefix, HttpHeaders.CONTENT_LENGTH, message.getPayload().getContentLength());
if (message.getPayload().getContentMD5() != null)
logger.debug("%s %s: %s", prefix, "Content-MD5", encryptionService.base64(message.getPayload()
.getContentMD5()));
try {
logger.debug("%s %s: %s", prefix, "Content-MD5", CryptoStreams.base64Encode(InputSuppliers.of(message
.getPayload().getContentMD5())));
} catch (IOException e) {
logger.warn(e, " error getting md5 for %s", message);
}
}
}
@ -363,13 +363,12 @@ public class HttpUtils {
}
/**
* change the destination of the current http command. typically used in
* handling redirects.
* change the destination of the current http command. typically used in handling redirects.
*
* @param string
*/
public static void changeSchemeHostAndPortTo(HttpRequest request, String scheme, String host, int port,
UriBuilder builder) {
UriBuilder builder) {
builder.uri(request.getEndpoint());
builder.scheme(scheme);
builder.host(host);
@ -403,7 +402,7 @@ public class HttpUtils {
}
public static void addQueryParamTo(HttpRequest request, String key, Iterable<?> values, UriBuilder builder,
char... skips) {
char... skips) {
builder.uri(request.getEndpoint());
Multimap<String, String> map = parseQueryToMap(request.getEndpoint().getQuery());
for (Object o : values)
@ -460,16 +459,16 @@ public class HttpUtils {
}
public static SortedSet<Entry<String, String>> sortEntries(Collection<Map.Entry<String, String>> in,
Comparator<Map.Entry<String, String>> sorter) {
Comparator<Map.Entry<String, String>> sorter) {
SortedSet<Entry<String, String>> entries = newTreeSet(sorter);
entries.addAll(in);
return entries;
}
public static String makeQueryLine(Multimap<String, String> params,
@Nullable Comparator<Map.Entry<String, String>> sorter, char... skips) {
@Nullable Comparator<Map.Entry<String, String>> sorter, char... skips) {
Iterator<Map.Entry<String, String>> pairs = ((sorter == null) ? params.entries() : sortEntries(params.entries(),
sorter)).iterator();
sorter)).iterator();
StringBuilder formBuilder = new StringBuilder();
while (pairs.hasNext()) {
Map.Entry<String, String> pair = pairs.next();
@ -499,7 +498,7 @@ public class HttpUtils {
payload.setContentLength(new Long(header.getValue()));
} else if ("Content-MD5".equalsIgnoreCase(header.getKey())) {
if (payload != null)
payload.setContentMD5(encryptionService.fromBase64(header.getValue()));
payload.setContentMD5(CryptoStreams.base64(header.getValue()));
} else if (CONTENT_TYPE.equalsIgnoreCase(header.getKey())) {
if (payload != null)
payload.setContentType(header.getValue());
@ -510,20 +509,20 @@ public class HttpUtils {
if (message instanceof HttpRequest) {
checkArgument(
message.getPayload() == null || message.getFirstHeaderOrNull(CONTENT_TYPE) == null,
"configuration error please use request.getPayload().setContentType(value) as opposed to adding a content type header: "
+ message);
message.getPayload() == null || message.getFirstHeaderOrNull(CONTENT_TYPE) == null,
"configuration error please use request.getPayload().setContentType(value) as opposed to adding a content type header: "
+ message);
checkArgument(
message.getPayload() == null || message.getFirstHeaderOrNull(CONTENT_LENGTH) == null,
"configuration error please use request.getPayload().setContentLength(value) as opposed to adding a content length header: "
+ message);
message.getPayload() == null || message.getFirstHeaderOrNull(CONTENT_LENGTH) == null,
"configuration error please use request.getPayload().setContentLength(value) as opposed to adding a content length header: "
+ message);
checkArgument(message.getPayload() == null || message.getPayload().getContentLength() != null
|| "chunked".equalsIgnoreCase(message.getFirstHeaderOrNull("Transfer-Encoding")),
"either chunked encoding must be set on the http request or contentlength set on the payload: "
+ message);
|| "chunked".equalsIgnoreCase(message.getFirstHeaderOrNull("Transfer-Encoding")),
"either chunked encoding must be set on the http request or contentlength set on the payload: "
+ message);
checkArgument(message.getPayload() == null || message.getFirstHeaderOrNull("Content-MD5") == null,
"configuration error please use request.getPayload().setContentMD5(value) as opposed to adding a content md5 header: "
+ message);
"configuration error please use request.getPayload().setContentMD5(value) as opposed to adding a content md5 header: "
+ message);
}
}
@ -537,7 +536,7 @@ public class HttpUtils {
}
public String valueOrEmpty(byte[] md5) {
return md5 != null ? encryptionService.base64(md5) : "";
return md5 != null ? CryptoStreams.base64(md5) : "";
}
public String valueOrEmpty(Collection<String> collection) {
@ -563,14 +562,14 @@ public class HttpUtils {
if (request.getPayload() != null && wire.enabled()) {
wire.output(request);
checkRequestHasContentLengthOrChunkedEncoding(request,
"After wiring, the request has neither chunked encoding nor content length: " + request);
"After wiring, the request has neither chunked encoding nor content length: " + request);
}
}
public static <T> T returnValueOnCodeOrNull(Exception from, T value, Predicate<Integer> codePredicate) {
Iterable<HttpResponseException> throwables = filter(getCausalChain(from), HttpResponseException.class);
if (size(throwables) >= 1 && get(throwables, 0).getResponse() != null
&& codePredicate.apply(get(throwables, 0).getResponse().getStatusCode())) {
&& codePredicate.apply(get(throwables, 0).getResponse().getStatusCode())) {
return value;
}
return null;

4
core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorServiceImpl.java
View File

@ -24,7 +24,7 @@ import javax.inject.Inject;
import javax.inject.Named;
import org.jclouds.Constants;
import static org.jclouds.concurrent.ConcurrentUtils.*;
import org.jclouds.concurrent.Futures;
import com.google.common.base.Function;
import com.google.common.util.concurrent.ListenableFuture;
@ -50,7 +50,7 @@ public class TransformingHttpCommandExecutorServiceImpl implements TransformingH
* {@inheritDoc}
*/
public <T> ListenableFuture<T> submit(HttpCommand command, Function<HttpResponse, T> responseTransformer) {
return compose(client.submit(command), responseTransformer, userThreads);
return Futures.compose(client.submit(command), responseTransformer, userThreads);
}
}

8
core/src/main/java/org/jclouds/http/filters/BasicAuthentication.java
View File

@ -30,7 +30,8 @@ import javax.inject.Named;
import javax.inject.Singleton;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
@ -50,11 +51,10 @@ public class BasicAuthentication implements HttpRequestFilter {
private final Set<String> credentialList;
@Inject
BasicAuthentication(@Named(PROPERTY_IDENTITY) String user,
@Named(PROPERTY_CREDENTIAL) String password, EncryptionService encryptionService)
BasicAuthentication(@Named(PROPERTY_IDENTITY) String user, @Named(PROPERTY_CREDENTIAL) String password, Crypto crypto)
throws UnsupportedEncodingException {
this.credentialList = ImmutableSet.of("Basic "
+ encryptionService.base64(String.format("%s:%s", checkNotNull(user, "user"),
+ CryptoStreams.base64(String.format("%s:%s", checkNotNull(user, "user"),
checkNotNull(password, "password")).getBytes("UTF-8")));
}

18
core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java
View File

@ -18,6 +18,7 @@
*/
package org.jclouds.http.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.io.ByteStreams.copy;
import static org.jclouds.http.HttpUtils.checkRequestHasContentLengthOrChunkedEncoding;
import static org.jclouds.http.HttpUtils.wirePayloadIfEnabled;
@ -35,7 +36,6 @@ import javax.inject.Inject;
import javax.inject.Named;
import org.jclouds.Constants;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.http.HttpCommand;
import org.jclouds.http.HttpCommandExecutorService;
import org.jclouds.http.HttpRequest;
@ -56,7 +56,6 @@ import com.google.common.io.NullOutputStream;
*/
public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandExecutorService {
protected final HttpUtils utils;
protected final EncryptionService encryptionService;
private final DelegatingRetryHandler retryHandler;
private final IOExceptionRetryHandler ioRetryHandler;
@ -72,17 +71,16 @@ public abstract class BaseHttpCommandExecutorService<Q> implements HttpCommandEx
protected final HttpWire wire;
@Inject
protected BaseHttpCommandExecutorService(HttpUtils utils, EncryptionService encryptionService,
protected BaseHttpCommandExecutorService(HttpUtils utils,
@Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire) {
this.utils = utils;
this.encryptionService = encryptionService;
this.retryHandler = retryHandler;
this.ioRetryHandler = ioRetryHandler;
this.errorHandler = errorHandler;
this.ioWorkerExecutor = ioWorkerExecutor;
this.wire = wire;
this.utils = checkNotNull(utils, "utils");
this.retryHandler = checkNotNull(retryHandler, "retryHandler");
this.ioRetryHandler = checkNotNull(ioRetryHandler, "ioRetryHandler");
this.errorHandler = checkNotNull(errorHandler, "errorHandler");
this.ioWorkerExecutor = checkNotNull(ioWorkerExecutor, "ioWorkerExecutor");
this.wire = checkNotNull(wire, "wire");
}
public static InputStream consumeOnClose(InputStream in) {

36
core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java
View File

@ -47,7 +47,7 @@ import javax.net.ssl.HttpsURLConnection;
import javax.ws.rs.core.HttpHeaders;
import org.jclouds.Constants;
import org.jclouds.encryption.EncryptionService;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.http.HttpCommandExecutorService;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpResponse;
@ -68,8 +68,7 @@ import com.google.common.collect.Multimap;
* @author Adrian Cole
*/
@Singleton
public class JavaUrlHttpCommandExecutorService extends
BaseHttpCommandExecutorService<HttpURLConnection> {
public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorService<HttpURLConnection> {
public static final String USER_AGENT = "jclouds/1.0 java/" + System.getProperty("java.version");
@Resource
@ -80,18 +79,15 @@ public class JavaUrlHttpCommandExecutorService extends
public JavaUrlHttpCommandExecutorService(HttpUtils utils,
@Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor,
DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler, HttpWire wire, HostnameVerifier verifier,
EncryptionService encryptionService) {
super(utils, encryptionService, ioWorkerExecutor, retryHandler, ioRetryHandler, errorHandler,
wire);
DelegatingErrorHandler errorHandler, HttpWire wire, HostnameVerifier verifier) {
super(utils, ioWorkerExecutor, retryHandler, ioRetryHandler, errorHandler, wire);
if (utils.getMaxConnections() > 0)
System.setProperty("http.maxConnections", String.valueOf(utils.getMaxConnections()));
this.verifier = verifier;
System.setProperty("http.maxConnections", String.valueOf(checkNotNull(utils, "utils").getMaxConnections()));
this.verifier = checkNotNull(verifier, "verifier");
}
@Override
protected HttpResponse invoke(HttpURLConnection connection) throws IOException,
InterruptedException {
protected HttpResponse invoke(HttpURLConnection connection) throws IOException, InterruptedException {
InputStream in = null;
try {
in = consumeOnClose(connection.getInputStream());
@ -109,8 +105,7 @@ public class JavaUrlHttpCommandExecutorService extends
}
Payload payload = in != null ? Payloads.newInputStreamPayload(in) : null;
HttpResponse response = new HttpResponse(connection.getResponseCode(), connection
.getResponseMessage(), payload);
HttpResponse response = new HttpResponse(connection.getResponseCode(), connection.getResponseMessage(), payload);
Multimap<String, String> headers = LinkedHashMultimap.create();
for (String header : connection.getHeaderFields().keySet()) {
headers.putAll(header, connection.getHeaderFields().get(header));
@ -133,8 +128,7 @@ public class JavaUrlHttpCommandExecutorService extends
}
@Override
protected HttpURLConnection convert(HttpRequest request) throws IOException,
InterruptedException {
protected HttpURLConnection convert(HttpRequest request) throws IOException, InterruptedException {
boolean chunked = "chunked".equals(request.getFirstHeaderOrNull("Transfer-Encoding"));
URL url = request.getEndpoint().toURL();
@ -150,8 +144,7 @@ public class JavaUrlHttpCommandExecutorService extends
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
Authenticator authenticator = new Authenticator() {
public PasswordAuthentication getPasswordAuthentication() {
return (new PasswordAuthentication(utils.getProxyUser(), utils.getProxyPassword()
.toCharArray()));
return (new PasswordAuthentication(utils.getProxyUser(), utils.getProxyPassword().toCharArray()));
}
};
Authenticator.setDefault(authenticator);
@ -182,16 +175,13 @@ public class JavaUrlHttpCommandExecutorService extends
OutputStream out = null;
try {
if (request.getPayload().getContentMD5() != null)
connection.setRequestProperty("Content-MD5", encryptionService.base64(request
.getPayload().getContentMD5()));
connection.setRequestProperty("Content-MD5", CryptoStreams.base64(request.getPayload().getContentMD5()));
if (request.getPayload().getContentType() != null)
connection.setRequestProperty(HttpHeaders.CONTENT_TYPE, request.getPayload()
.getContentType());
connection.setRequestProperty(HttpHeaders.CONTENT_TYPE, request.getPayload().getContentType());
if (chunked) {
connection.setChunkedStreamingMode(8196);
} else {
Long length = checkNotNull(request.getPayload().getContentLength(),
"payload.getContentLength");
Long length = checkNotNull(request.getPayload().getContentLength(), "payload.getContentLength");
connection.setRequestProperty(HttpHeaders.CONTENT_LENGTH, length.toString());
connection.setFixedLengthStreamingMode(length.intValue());
}

2
core/src/main/java/org/jclouds/http/internal/PayloadEnclosingImpl.java
View File

@ -26,8 +26,8 @@ import java.io.InputStream;
import javax.annotation.Nullable;
import org.jclouds.http.PayloadEnclosing;
import org.jclouds.io.Payload;
import org.jclouds.io.PayloadEnclosing;
/**
*

74
core/src/main/java/org/jclouds/io/InputSuppliers.java Normal file
View File

@ -0,0 +1,74 @@
package org.jclouds.io;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.IOException;
import java.io.InputStream;
import org.jclouds.encryption.internal.Base64;
import com.google.common.annotations.Beta;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.io.ByteStreams;
import com.google.common.io.InputSupplier;
/**
* functions related to or replacing those in {@link com.google.common.io.InputSupplier}
*
* @author Adrian Cole
*/
@Beta
public class InputSuppliers {
/**
* base64 encodes bytes from the supplied supplier as they are read.
*/
public static Base64InputSupplier base64Encoder(InputSupplier<? extends InputStream> supplier) throws IOException {
return new Base64InputSupplier(supplier, Base64.ENCODE + Base64.DONT_BREAK_LINES);
}
/**
* base64 decodes bytes from the supplied supplier as they are read.
*/
public static Base64InputSupplier base64Decoder(InputSupplier<? extends InputStream> supplier) throws IOException {
return new Base64InputSupplier(supplier, Base64.DECODE);
}
@VisibleForTesting
static class Base64InputSupplier implements InputSupplier<InputStream> {
private final InputSupplier<? extends InputStream> delegate;
private final int mode;
Base64InputSupplier(InputSupplier<? extends InputStream> inputSupplier, int mode) {
this.delegate = checkNotNull(inputSupplier, "delegate");
this.mode = mode;
}
@Override
public InputStream getInput() throws IOException {
return new Base64.InputStream(delegate.getInput(), mode);
}
}
public static InputSupplier<? extends InputStream> of(final InputStream in) {
checkNotNull(in, "in");
return new InputSupplier<InputStream>() {
@Override
public InputStream getInput() throws IOException {
return in;
}
};
}
public static InputSupplier<? extends InputStream> of(byte[] in) {
return ByteStreams.newInputStreamSupplier(checkNotNull(in, "in"));
}
public static InputSupplier<? extends InputStream> of(String in) {
return of(checkNotNull(in, "in").getBytes(Charsets.UTF_8));
}
}

3
core/src/main/java/org/jclouds/http/PayloadEnclosing.java → core/src/main/java/org/jclouds/io/PayloadEnclosing.java
View File

@ -16,12 +16,11 @@
* limitations under the License.
* ====================================================================
*/
package org.jclouds.http;
package org.jclouds.io;
import java.io.File;
import java.io.InputStream;
import org.jclouds.io.Payload;
/**
*

87
core/src/main/java/org/jclouds/io/Payloads.java
View File

@ -19,20 +19,27 @@
package org.jclouds.io;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.io.ByteStreams.toByteArray;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Comparator;
import java.util.Map;
import javax.annotation.Nullable;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.io.payloads.ByteArrayPayload;
import org.jclouds.io.payloads.FilePayload;
import org.jclouds.io.payloads.InputStreamPayload;
import org.jclouds.io.payloads.StringPayload;
import org.jclouds.io.payloads.UrlEncodedFormPayload;
import com.google.common.base.Throwables;
import com.google.common.collect.Multimap;
/**
@ -40,6 +47,8 @@ import com.google.common.collect.Multimap;
* @author Adrian Cole
*/
public class Payloads {
private Payloads() {
}
public static Payload newPayload(Object data) {
checkNotNull(data, "data");
@ -74,14 +83,84 @@ public class Payloads {
return new FilePayload(checkNotNull(data, "data"));
}
public static UrlEncodedFormPayload newUrlEncodedFormPayload(
Multimap<String, String> formParams, char... skips) {
public static UrlEncodedFormPayload newUrlEncodedFormPayload(Multimap<String, String> formParams, char... skips) {
return new UrlEncodedFormPayload(formParams, skips);
}
public static UrlEncodedFormPayload newUrlEncodedFormPayload(
Multimap<String, String> formParams,
public static UrlEncodedFormPayload newUrlEncodedFormPayload(Multimap<String, String> formParams,
@Nullable Comparator<Map.Entry<String, String>> sorter, char... skips) {
return new UrlEncodedFormPayload(formParams, sorter, skips);
}
/**
* Calculates and sets {@link Payload#setContentMD5} on the payload.
*
* <p/>
* note that this will rebuffer in memory if the payload is not repeatable.
*
* @param payload
* payload to calculate
* @param md5
* digester to calculate payloads with.
* @return new Payload with md5 set.
* @throws IOException
*/
public static Payload calculateMD5(Payload payload, MessageDigest md5) throws IOException {
checkNotNull(payload, "payload");
if (!payload.isRepeatable()) {
String oldContentType = payload.getContentType();
Payload oldPayload = payload;
try {
payload = newByteArrayPayload(toByteArray(payload));
} finally {
oldPayload.release();
}
payload.setContentType(oldContentType);
}
payload.setContentMD5(CryptoStreams.digest(payload, md5));
return payload;
}
/**
* Uses default md5 generator.
*
* @see #calculateMD5(Payload, MessageDigest)
*/
public static Payload calculateMD5(Payload payload) throws IOException {
try {
return calculateMD5(payload, MessageDigest.getInstance("MD5"));
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
return null;
}
}
/**
* Calculates the md5 on a payload, replacing as necessary.
*
* @see #calculateMD5(Payload, MessageDigest)
*/
public static <T extends PayloadEnclosing> T calculateMD5(T payloadEnclosing, MessageDigest md5) throws IOException {
checkState(payloadEnclosing != null, "payloadEnclosing");
Payload newPayload = calculateMD5(payloadEnclosing.getPayload(), md5);
if (newPayload != payloadEnclosing.getPayload())
payloadEnclosing.setPayload(newPayload);
return payloadEnclosing;
}
/**
* Calculates the md5 on a payload, replacing as necessary.
* <p/>
* uses default md5 generator.
*
* @see #calculateMD5(Payload, MessageDigest)
*/
public static <T extends PayloadEnclosing> T calculateMD5(T payloadEnclosing) throws IOException {
try {
return calculateMD5(payloadEnclosing, MessageDigest.getInstance("MD5"));
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
return null;
}
}
}

57
core/src/main/java/org/jclouds/io/payloads/BaseCipherPayload.java Normal file
View File

@ -0,0 +1,57 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.io.payloads;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.IOException;
import java.io.OutputStream;
import java.security.Key;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import org.jclouds.io.Payload;
/**
*
* @author Adrian Cole
*/
public abstract class BaseCipherPayload extends DelegatingPayload {
private final Key key;
public BaseCipherPayload(Payload delegate, Key key) {
super(delegate);
this.key = checkNotNull(key, "key");
}
public abstract Cipher initializeCipher(Key key);
@Override
public CipherInputStream getInput() {
return new CipherInputStream(super.getInput(), initializeCipher(key));
}
@Override
public void writeTo(OutputStream outstream) throws IOException {
super.writeTo(new CipherOutputStream(outstream, initializeCipher(key)));
}
}

58
core/src/main/java/org/jclouds/io/payloads/RSADecryptingPayload.java Normal file
View File

@ -0,0 +1,58 @@
/**
*
* Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.jclouds.io.payloads;
import java.security.InvalidKeyException;
import java.security.Key;
import java.security.NoSuchAlgorithmException;
import javax.crypto.Cipher;
import javax.crypto.NoSuchPaddingException;
import org.jclouds.io.Payload;
import com.google.common.base.Throwables;
/**
*
* @author Adrian Cole
*/
public class RSADecryptingPayload extends BaseCipherPayload {
public RSADecryptingPayload(Payload delegate, Key key) {
super(delegate, key);
}
@Override
public Cipher initializeCipher(Key key) {
Cipher cipher = null;
try {
cipher = Cipher.getInstance("RSA");
cipher.init(Cipher.DECRYPT_MODE, key);
} catch (NoSuchAlgorithmException e) {
Throwables.propagate(e);
} catch (NoSuchPaddingException e) {
Throwables.propagate(e);
} catch (InvalidKeyException e) {
Throwables.propagate(e);
}
return cipher;
}
}

Some files were not shown because too many files have changed in this diff Show More