Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-480: AWS S3 v4 signature 

This includes support for chunked uploads.
This commit is contained in:
Zhao Jin 2015-02-11 10:55:28 +08:00 committed by Andrew Gaul
parent c20fcb8cda
commit 8bddbb496a
24 changed files with 2854 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java
View File

@ -22,6 +22,7 @@ import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX;
import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX;
import static org.jclouds.reflect.Reflection2.typeToken;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_JCLOUDS_S3_CHUNKED_SIZE;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
@ -78,6 +79,9 @@ public class S3ApiMetadata extends BaseHttpApiMetadata {
properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true");
properties.setProperty(PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX, "/");
properties.setProperty(PROPERTY_USER_METADATA_PREFIX, String.format("x-${%s}-meta-", PROPERTY_HEADER_TAG));
// Chunk size must be at least 8 KB. We recommend a chunk size of a least 64 KB for better performance.
properties.setProperty(PROPERTY_JCLOUDS_S3_CHUNKED_SIZE, String.valueOf(64 * 1024));
return properties;
}

43
apis/s3/src/main/java/org/jclouds/s3/config/S3HttpApiModule.java
View File

@ -17,6 +17,7 @@
package org.jclouds.s3.config;
import java.net.URI;
import java.util.Date;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@ -33,6 +34,7 @@ import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.date.DateService;
import org.jclouds.date.TimeStamp;
import org.jclouds.http.HttpErrorHandler;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRetryHandler;
import org.jclouds.http.annotation.ClientError;
import org.jclouds.http.annotation.Redirection;
@ -46,6 +48,7 @@ import org.jclouds.s3.S3Client;
import org.jclouds.s3.blobstore.functions.BucketsToStorageMetadata;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.filters.RequestAuthorizeSignatureV2;
import org.jclouds.s3.functions.GetRegionForBucket;
import org.jclouds.s3.handlers.ParseS3ErrorFromXmlContent;
import org.jclouds.s3.handlers.S3RedirectionRetryHandler;
@ -181,13 +184,26 @@ public class S3HttpApiModule<S extends S3Client> extends AWSHttpApiModule<S> {
}
protected void bindRequestSigner() {
bind(RequestAuthorizeSignature.class).in(Scopes.SINGLETON);
bind(RequestAuthorizeSignature.class).to(RequestAuthorizeSignatureV2.class).in(Scopes.SINGLETON);
}
@Provides
@Singleton
protected final RequestSigner provideRequestSigner(RequestAuthorizeSignature in) {
return in;
if (in instanceof RequestSigner) {
return (RequestSigner) in;
}
return new RequestSigner() {
@Override
public String createStringToSign(HttpRequest input) {
return null;
}
@Override
public String sign(String toSign) {
return null;
}
};
}
@Override
@ -222,4 +238,27 @@ public class S3HttpApiModule<S extends S3Client> extends AWSHttpApiModule<S> {
}
}, seconds, TimeUnit.SECONDS);
}
@Provides
@TimeStamp
protected Date provideTimeStampDate(@TimeStamp Supplier<Date> cache) {
return cache.get();
}
/**
* borrowing concurrency code to ensure that caching takes place properly
*/
@Provides
@TimeStamp
@Singleton
protected Supplier<Date> provideTimeStampCacheDate(
@Named(Constants.PROPERTY_SESSION_INTERVAL) long seconds,
@TimeStamp final Supplier<String> timestamp,
final DateService dateService) {
return Suppliers.memoizeWithExpiration(new Supplier<Date>() {
public Date get() {
return dateService.rfc822DateParse(timestamp.get());
}
}, seconds, TimeUnit.SECONDS);
}
}

437
apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerBase.java Normal file
View File

@ -0,0 +1,437 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.io.BaseEncoding.base16;
import static com.google.common.io.ByteStreams.readBytes;
import static org.jclouds.crypto.Macs.asByteProcessor;
import static org.jclouds.http.utils.Queries.queryParser;
import static org.jclouds.util.Strings2.toInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.security.InvalidKeyException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
import java.util.TimeZone;
import javax.inject.Inject;
import javax.xml.ws.http.HTTPException;
import com.google.common.base.Joiner;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.escape.Escaper;
import com.google.common.hash.Hashing;
import com.google.common.hash.HashingInputStream;
import com.google.common.io.ByteProcessor;
import com.google.common.io.ByteSource;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import com.google.common.net.PercentEscaper;
import com.google.inject.ImplementedBy;
import org.jclouds.crypto.Crypto;
import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.io.Payload;
import org.jclouds.providers.ProviderMetadata;
/**
* Common methods and properties for all AWS4 signer variants
*/
public abstract class Aws4SignerBase {
private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
protected static final DateFormat timestampFormat;
protected static final DateFormat dateFormat;
// Do not URL-encode any of the unreserved characters that RFC 3986 defines:
// A-Z, a-z, 0-9, hyphen (-), underscore (_), period (.), and tilde (~).
private static final Escaper AWS_URL_PARAMETER_ESCAPER;
static {
timestampFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
timestampFormat.setTimeZone(GMT);
dateFormat = new SimpleDateFormat("yyyyMMdd");
dateFormat.setTimeZone(GMT);
AWS_URL_PARAMETER_ESCAPER = new PercentEscaper("-_.~", false);
}
// Specifying a default for how to parse the service and region in this way allows
// tests or other downstream services to not have to use guice overrides.
@ImplementedBy(ServiceAndRegion.AWSServiceAndRegion.class)
public interface ServiceAndRegion {
String service();
String region(String host);
final class AWSServiceAndRegion implements ServiceAndRegion {
private final String service;
@Inject
AWSServiceAndRegion(ProviderMetadata provider) {
this(provider.getEndpoint());
}
AWSServiceAndRegion(String endpoint) {
this.service = AwsHostNameUtils.parseServiceName(URI.create(checkNotNull(endpoint, "endpoint")));
}
@Override
public String service() {
return service;
}
@Override
public String region(String host) {
return AwsHostNameUtils.parseRegionName(host, service());
}
}
}
protected final String headerTag;
protected final ServiceAndRegion serviceAndRegion;
protected final SignatureWire signatureWire;
protected final Supplier<Credentials> creds;
protected final Supplier<Date> timestampProvider;
protected final Crypto crypto;
protected Aws4SignerBase(SignatureWire signatureWire, String headerTag,
Supplier<Credentials> creds, Supplier<Date> timestampProvider,
ServiceAndRegion serviceAndRegion, Crypto crypto) {
this.signatureWire = signatureWire;
this.headerTag = headerTag;
this.creds = creds;
this.timestampProvider = timestampProvider;
this.serviceAndRegion = serviceAndRegion;
this.crypto = crypto;
}
protected String getContentType(HttpRequest request) {
Payload payload = request.getPayload();
// Default Content Type
String contentType = request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE);
if (payload != null
&& payload.getContentMetadata() != null
&& payload.getContentMetadata().getContentType() != null) {
contentType = payload.getContentMetadata().getContentType();
}
return contentType;
}
protected String getContentLength(HttpRequest request) {
Payload payload = request.getPayload();
// Default Content Type
String contentLength = request.getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH);
if (payload != null
&& payload.getContentMetadata() != null
&& payload.getContentMetadata().getContentType() != null) {
Long length = payload.getContentMetadata().getContentLength();
contentLength =
length == null ? contentLength : String.valueOf(payload.getContentMetadata().getContentLength());
}
return contentLength;
}
// append all of 'x-amz-*' headers
protected void appendAmzHeaders(HttpRequest request,
ImmutableMap.Builder<String, String> signedHeadersBuilder) {
for (Map.Entry<String, String> header : request.getHeaders().entries()) {
String key = header.getKey();
if (key.startsWith("x-" + headerTag + "-")) {
signedHeadersBuilder.put(key.toLowerCase(), header.getValue());
}
}
}
/**
* caluclate AWS signature key.
* <p>
* <code>
* DateKey = hmacSHA256(datestamp, "AWS4"+ secretKey)
* <br>
* DateRegionKey = hmacSHA256(region, DateKey)
* <br>
* DateRegionServiceKey = hmacSHA256(service, DateRegionKey)
* <br>
* SigningKey = hmacSHA256("aws4_request", DateRegionServiceKey)
* <br>
* <p/>
* </code>
* </p>
*
* @param secretKey AWS access secret key
* @param datestamp date yyyyMMdd
* @param region AWS region
* @param service AWS service
* @return SigningKey
*/
protected byte[] signatureKey(String secretKey, String datestamp, String region, String service) {
byte[] kSecret = ("AWS4" + secretKey).getBytes(UTF_8);
byte[] kDate = hmacSHA256(datestamp, kSecret);
byte[] kRegion = hmacSHA256(region, kDate);
byte[] kService = hmacSHA256(service, kRegion);
byte[] kSigning = hmacSHA256("aws4_request", kService);
return kSigning;
}
/**
* hmac sha256
*
* @param toSign string to sign
* @param key hash key
*/
protected byte[] hmacSHA256(String toSign, byte[] key) {
try {
return readBytes(toInputStream(toSign), hmacSHA256(crypto, key));
} catch (IOException e) {
throw new HttpException("read sign error", e);
} catch (InvalidKeyException e) {
throw new HttpException("invalid key", e);
}
}
public static ByteProcessor<byte[]> hmacSHA256(Crypto crypto, byte[] signatureKey) throws InvalidKeyException {
return asByteProcessor(crypto.hmacSHA256(signatureKey));
}
/**
* hash input with sha256
*
* @param input
* @return hash result
* @throws HTTPException
*/
public static byte[] hash(InputStream input) throws HTTPException {
HashingInputStream his = new HashingInputStream(Hashing.sha256(), input);
try {
ByteStreams.copy(his, ByteStreams.nullOutputStream());
return his.hash().asBytes();
} catch (IOException e) {
throw new HttpException("Unable to compute hash while signing request: " + e.getMessage(), e);
}
}
/**
* hash input with sha256
*
* @param bytes input bytes
* @return hash result
* @throws HTTPException
*/
public static byte[] hash(byte[] bytes) throws HTTPException {
try {
return ByteSource.wrap(bytes).hash(Hashing.sha256()).asBytes();
} catch (IOException e) {
throw new HttpException("Unable to compute hash while signing request: " + e.getMessage(), e);
}
}
/**
* hash string (encoding UTF_8) with sha256
*
* @param input input stream
* @return hash result
* @throws HTTPException
*/
public static byte[] hash(String input) throws HTTPException {
return hash(new ByteArrayInputStream(input.getBytes(UTF_8)));
}
/**
* Examines the specified query string parameters and returns a
* canonicalized form.
* <p/>
* The canonicalized query string is formed by first sorting all the query
* string parameters, then URI encoding both the key and value and then
* joining them, in order, separating key value pairs with an '&'.
*
* @param queryString The query string parameters to be canonicalized.
* @return A canonicalized form for the specified query string parameters.
*/
protected String getCanonicalizedQueryString(String queryString) {
Multimap<String, String> params = queryParser().apply(queryString);
SortedMap<String, String> sorted = Maps.newTreeMap();
if (params == null) {
return "";
}
Iterator<Map.Entry<String, String>> pairs = params.entries().iterator();
while (pairs.hasNext()) {
Map.Entry<String, String> pair = pairs.next();
String key = pair.getKey();
String value = pair.getValue();
sorted.put(urlEncode(key), urlEncode(value));
}
return Joiner.on("&").withKeyValueSeparator("=").join(sorted);
}
/**
* Encode a string for use in the path of a URL; uses URLEncoder.encode,
* (which encodes a string for use in the query portion of a URL), then
* applies some postfilters to fix things up per the RFC. Can optionally
* handle strings which are meant to encode a path (ie include '/'es
* which should NOT be escaped).
*
* @param value the value to encode
* @return the encoded value
*/
public static String urlEncode(final String value) {
if (value == null) {
return "";
}
return AWS_URL_PARAMETER_ESCAPER.escape(value);
}
/**
* Lowercase base 16 encoding.
*
* @param bytes bytes
* @return base16 lower case hex string.
*/
public static String hex(final byte[] bytes) {
return base16().lowerCase().encode(bytes);
}
/**
* Create a Canonical Request to sign
* <h4>Canonical Request</h4>
* <p>
* <code>
* &lt;HTTPMethod>\n
* <br>
* &lt;CanonicalURI>\n
* <br>
* &lt;CanonicalQueryString>\n
* <br>
* &lt;CanonicalHeaders>\n
* <br>
* &lt;SignedHeaders>\n
* <br>
* &lt;HashedPayload>
* </code>
* </p>
* <p><b>HTTPMethod</b> is one of the HTTP methods, for example GET, PUT, HEAD, and DELETE.</p>
* <p><b>CanonicalURI</b> is the URI-encoded version of the absolute path component of the URI—everything starting
* with the "/" that follows the domain name and up to the end of the string or to the question mark character ('?')
* if you have query string parameters.</p>
* <p><b>CanonicalQueryString</b> specifies the URI-encoded query string parameters. You URI-encode name and values
* individually. You must also sort the parameters in the canonical query string alphabetically by key name.
* The sorting occurs after encoding.</p>
* <p><b>CanonicalHeaders</b> is a list of request headers with their values. Individual header name and value pairs are
* separated by the newline character ("\n"). Header names must be in lowercase. Header value must be trim space.
* <br>
* The <b>CanonicalHeaders</b> list must include the following:
* HTTP host header.
* If the Content-Type header is present in the request, it must be added to the CanonicalHeaders list.
* Any x-amz-* headers that you plan to include in your request must also be added.</p>
* <p><b>SignedHeaders</b> is an alphabetically sorted, semicolon-separated list of lowercase request header names.
* The request headers in the list are the same headers that you included in the CanonicalHeaders string.</p>
* <p><b>HashedPayload</b> is the hexadecimal value of the SHA256 hash of the request payload. </p>
* <p>If there is no payload in the request, you compute a hash of the empty string as follows:
* <code>Hex(SHA256Hash(""))</code> The hash returns the following value:
* e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 </p>
*
* @param method http request method
* @param endpoint http request endpoing
* @param signedHeaders signed headers
* @param timestamp ISO8601 timestamp
* @param credentialScope credential scope
* @return string to sign
*/
protected String createStringToSign(String method, URI endpoint, Map<String, String> signedHeaders,
String timestamp, String credentialScope, String hashedPayload) {
// lower case header keys
Map<String, String> lowerCaseHeaders = lowerCaseNaturalOrderKeys(signedHeaders);
StringBuilder canonicalRequest = new StringBuilder();
// HTTPRequestMethod + '\n' +
canonicalRequest.append(method).append("\n");
// CanonicalURI + '\n' +
canonicalRequest.append(endpoint.getPath()).append("\n");
// CanonicalQueryString + '\n' +
if (endpoint.getQuery() != null) {
canonicalRequest.append(getCanonicalizedQueryString(endpoint.getQuery()));
}
canonicalRequest.append("\n");
// CanonicalHeaders + '\n' +
for (Map.Entry<String, String> entry : lowerCaseHeaders.entrySet()) {
canonicalRequest.append(entry.getKey()).append(':').append(entry.getValue()).append('\n');
}
canonicalRequest.append("\n");
// SignedHeaders + '\n' +
canonicalRequest.append(Joiner.on(';').join(lowerCaseHeaders.keySet())).append('\n');
// HexEncode(Hash(Payload))
canonicalRequest.append(hashedPayload);
signatureWire.getWireLog().debug("<<", canonicalRequest);
// Create a String to Sign
StringBuilder toSign = new StringBuilder();
// Algorithm + '\n' +
toSign.append("AWS4-HMAC-SHA256").append('\n');
// RequestDate + '\n' +
toSign.append(timestamp).append('\n');
// CredentialScope + '\n' +
toSign.append(credentialScope).append('\n');
// HexEncode(Hash(CanonicalRequest))
toSign.append(hex(hash(canonicalRequest.toString())));
return toSign.toString();
}
/**
* change the keys but keep the values in-tact.
*
* @param in input map to transform
* @return immutableSortedMap with the new lowercase keys.
*/
protected static Map<String, String> lowerCaseNaturalOrderKeys(Map<String, String> in) {
checkNotNull(in, "input map");
ImmutableSortedMap.Builder<String, String> returnVal = ImmutableSortedMap.<String, String>naturalOrder();
for (Map.Entry<String, String> entry : in.entrySet())
returnVal.put(entry.getKey().toLowerCase(Locale.US), entry.getValue());
return returnVal.build();
}
}

204
apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForAuthorizationHeader.java Normal file
View File

@ -0,0 +1,204 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import com.google.common.base.Joiner;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.net.HttpHeaders;
import com.google.inject.Inject;
import org.jclouds.aws.domain.SessionCredentials;
import org.jclouds.crypto.Crypto;
import org.jclouds.date.TimeStamp;
import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.io.Payload;
import org.jclouds.location.Provider;
import org.jclouds.util.Closeables2;
import javax.inject.Named;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.io.BaseEncoding.base16;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static com.google.common.net.HttpHeaders.CONTENT_MD5;
import static com.google.common.net.HttpHeaders.DATE;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_ALGORITHM_HMAC_SHA256;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_CONTENT_SHA256_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DATE_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SECURITY_TOKEN_HEADER;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
/**
* AWS4 signer sign requests to Amazon S3 using an 'Authorization' header.
*/
public class Aws4SignerForAuthorizationHeader extends Aws4SignerBase {
@Inject
public Aws4SignerForAuthorizationHeader(SignatureWire signatureWire,
@Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
@Named(PROPERTY_HEADER_TAG) String headerTag,
@Provider Supplier<Credentials> creds, @TimeStamp Supplier<Date> timestampProvider,
ServiceAndRegion serviceAndRegion, Crypto crypto) {
super(signatureWire, headerTag, creds, timestampProvider, serviceAndRegion, crypto);
}
protected HttpRequest sign(HttpRequest request) throws HttpException {
checkNotNull(request, "request is not ready to sign");
checkNotNull(request.getEndpoint(), "request is not ready to sign, request.endpoint not present.");
// get host from request endpoint.
String host = request.getEndpoint().getHost();
Date date = timestampProvider.get();
String timestamp = timestampFormat.format(date);
String datestamp = dateFormat.format(date);
String service = serviceAndRegion.service();
String region = serviceAndRegion.region(host);
String credentialScope = Joiner.on('/').join(datestamp, region, service, "aws4_request");
HttpRequest.Builder<?> requestBuilder = request.toBuilder() //
.removeHeader(AUTHORIZATION) // remove Authorization
.removeHeader(CONTENT_MD5) // aws s3 not allowed Content-MD5, use specs x-amz-content-sha256
.removeHeader(DATE); // remove date
ImmutableMap.Builder<String, String> signedHeadersBuilder = ImmutableSortedMap.<String, String>naturalOrder();
// Content Type
// content-type is not a required signing param. However, examples use this, so we include it to ease testing.
String contentType = getContentType(request);
if (!Strings.isNullOrEmpty(contentType)) {
requestBuilder.replaceHeader(HttpHeaders.CONTENT_TYPE, contentType);
signedHeadersBuilder.put(HttpHeaders.CONTENT_TYPE.toLowerCase(), contentType);
}
// Content-Length for PUT or POST request http method
String contentLength = getContentLength(request);
if (!Strings.isNullOrEmpty(contentLength)) {
requestBuilder.replaceHeader(HttpHeaders.CONTENT_LENGTH, contentLength);
signedHeadersBuilder.put(HttpHeaders.CONTENT_LENGTH.toLowerCase(), contentLength);
}
// host
requestBuilder.replaceHeader(HttpHeaders.HOST, host);
signedHeadersBuilder.put(HttpHeaders.HOST.toLowerCase(), host);
// user-agent
if (request.getHeaders().containsKey(HttpHeaders.USER_AGENT)) {
signedHeadersBuilder.put(HttpHeaders.USER_AGENT.toLowerCase(),
request.getFirstHeaderOrNull(HttpHeaders.USER_AGENT));
}
// all x-amz-* headers
appendAmzHeaders(request, signedHeadersBuilder);
// x-amz-security-token
Credentials credentials = creds.get();
if (credentials instanceof SessionCredentials) {
String token = SessionCredentials.class.cast(credentials).getSessionToken();
requestBuilder.replaceHeader(AMZ_SECURITY_TOKEN_HEADER, token);
signedHeadersBuilder.put(AMZ_SECURITY_TOKEN_HEADER.toLowerCase(), token);
}
// x-amz-content-sha256
String contentSha256 = getPayloadHash(request);
requestBuilder.replaceHeader(AMZ_CONTENT_SHA256_HEADER, contentSha256);
signedHeadersBuilder.put(AMZ_CONTENT_SHA256_HEADER.toLowerCase(), contentSha256);
// put x-amz-date
requestBuilder.replaceHeader(AMZ_DATE_HEADER, timestamp);
signedHeadersBuilder.put(AMZ_DATE_HEADER.toLowerCase(), timestamp);
ImmutableMap<String, String> signedHeaders = signedHeadersBuilder.build();
String stringToSign = createStringToSign(request.getMethod(), request.getEndpoint(), signedHeaders, timestamp,
credentialScope, contentSha256);
signatureWire.getWireLog().debug("<< " + stringToSign);
byte[] signatureKey = signatureKey(credentials.credential, datestamp, region, service);
String signature = base16().lowerCase().encode(hmacSHA256(stringToSign, signatureKey));
StringBuilder authorization = new StringBuilder(AMZ_ALGORITHM_HMAC_SHA256).append(" ");
authorization.append("Credential=").append(Joiner.on("/").join(credentials.identity, credentialScope))
.append(", ");
authorization.append("SignedHeaders=").append(Joiner.on(";").join(signedHeaders.keySet()))
.append(", ");
authorization.append("Signature=").append(signature);
return requestBuilder.replaceHeader(HttpHeaders.AUTHORIZATION, authorization.toString()).build();
}
protected String getPayloadHash(HttpRequest request) {
Payload payload = request.getPayload();
if (payload == null) {
// when payload is null.
return getEmptyPayloadContentHash();
}
return calculatePayloadContentHash(payload);
}
/**
* The hash returns the following value: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
*/
protected String getEmptyPayloadContentHash() {
return base16().lowerCase().encode(hash(new ByteArrayInputStream(new byte[0])));
}
/**
* in this time, payload ContentMetadata provided content hash md5, but aws required sha256.
*/
protected String calculatePayloadContentHash(Payload payload) {
// use payload stream calculate content sha256
InputStream payloadStream;
try {
payloadStream = payload.openStream();
} catch (IOException e) {
throw new HttpException("unable to open payload stream to calculate AWS4 signature.");
}
try {
return base16().lowerCase().encode(hash(payloadStream));
} finally {
closeOrResetPayloadStream(payloadStream, payload.isRepeatable());
}
}
// some times, when use Multipart Payload and a part can not be repeatable, will happen some error...
void closeOrResetPayloadStream(InputStream payloadStream, boolean repeatable) {
// if payload stream can repeatable.
if (repeatable) {
Closeables2.closeQuietly(payloadStream);
} else {
try {
// reset unrepeatable payload stream
payloadStream.reset();
} catch (IOException e) {
// reset payload stream
throw new HttpException(
"unable to reset unrepeatable payload stream after calculating AWS4 signature.");
}
}
}
}

254
apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForChunkedUpload.java Normal file
View File

@ -0,0 +1,254 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_ALGORITHM_HMAC_SHA256;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_CONTENT_SHA256_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DATE_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DECODED_CONTENT_LENGTH_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SECURITY_TOKEN_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.CHUNK_SIGNATURE_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.CLRF;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.CONTENT_ENCODING_HEADER_AWS_CHUNKED;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.SIGNATURE_LENGTH;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.STREAMING_BODY_SHA256;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_JCLOUDS_S3_CHUNKED_SIZE;
import static org.jclouds.util.Strings2.toInputStream;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.io.ByteStreams.readBytes;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
import static com.google.common.net.HttpHeaders.CONTENT_MD5;
import static com.google.common.net.HttpHeaders.DATE;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.util.Date;
import javax.inject.Named;
import org.jclouds.aws.domain.SessionCredentials;
import org.jclouds.crypto.Crypto;
import org.jclouds.date.TimeStamp;
import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.io.Payload;
import org.jclouds.location.Provider;
import com.google.common.base.Joiner;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.io.ByteProcessor;
import com.google.common.net.HttpHeaders;
import com.google.inject.Inject;
/**
* AWS4 signer sign 'chunked' uploads.
*/
public class Aws4SignerForChunkedUpload extends Aws4SignerBase {
private final int userDataBlockSize;
@Inject
public Aws4SignerForChunkedUpload(SignatureWire signatureWire,
@Named(PROPERTY_HEADER_TAG) String headerTag,
@Named(PROPERTY_JCLOUDS_S3_CHUNKED_SIZE) int userDataBlockSize,
@Provider Supplier<Credentials> creds, @TimeStamp Supplier<Date> timestampProvider,
ServiceAndRegion serviceAndRegion, Crypto crypto) {
super(signatureWire, headerTag, creds, timestampProvider, serviceAndRegion, crypto);
this.userDataBlockSize = userDataBlockSize;
}
protected HttpRequest sign(HttpRequest request) throws HttpException {
checkNotNull(request, "request is not ready to sign");
checkNotNull(request.getEndpoint(), "request is not ready to sign, request.endpoint not present.");
Payload payload = request.getPayload();
// chunked upload required content-length.
Long contentLength = payload.getContentMetadata().getContentLength();
// check contentLength not null
checkNotNull(contentLength, "request is not ready to sign, payload contentLength not present.");
// get host from request endpoint.
String host = request.getEndpoint().getHost();
Date date = timestampProvider.get();
String timestamp = timestampFormat.format(date);
String datestamp = dateFormat.format(date);
String service = serviceAndRegion.service();
String region = serviceAndRegion.region(host);
String credentialScope = Joiner.on('/').join(datestamp, region, service, "aws4_request");
HttpRequest.Builder<?> requestBuilder = request.toBuilder() //
.removeHeader(AUTHORIZATION) // remove Authorization
.removeHeader(CONTENT_MD5) // aws s3 not allowed Content-MD5, use aws specs x-amz-content-sha256
.removeHeader(DATE) // remove Date
.removeHeader(CONTENT_LENGTH); // remove Content-Length
ImmutableMap.Builder<String, String> signedHeadersBuilder = ImmutableSortedMap.<String, String>naturalOrder();
// content-encoding
requestBuilder.replaceHeader(HttpHeaders.CONTENT_ENCODING, CONTENT_ENCODING_HEADER_AWS_CHUNKED);
signedHeadersBuilder.put(HttpHeaders.CONTENT_ENCODING.toLowerCase(), CONTENT_ENCODING_HEADER_AWS_CHUNKED);
// x-amz-decoded-content-length
requestBuilder.replaceHeader(AMZ_DECODED_CONTENT_LENGTH_HEADER, contentLength.toString());
signedHeadersBuilder.put(AMZ_DECODED_CONTENT_LENGTH_HEADER.toLowerCase(), contentLength.toString());
// how big is the overall request stream going to be once we add the signature
// 'headers' to each chunk?
long totalLength = calculateChunkedContentLength(contentLength, userDataBlockSize);
requestBuilder.replaceHeader(CONTENT_LENGTH, Long.toString(totalLength));
signedHeadersBuilder.put(CONTENT_LENGTH.toLowerCase(), Long.toString(totalLength));
// Content Type
// content-type is not a required signing param. However, examples use this, so we include it to ease testing.
String contentType = getContentType(request);
if (!Strings.isNullOrEmpty(contentType)) {
requestBuilder.replaceHeader(HttpHeaders.CONTENT_TYPE, contentType);
signedHeadersBuilder.put(HttpHeaders.CONTENT_TYPE.toLowerCase(), contentType);
} else {
requestBuilder.removeHeader(HttpHeaders.CONTENT_TYPE);
}
// host
requestBuilder.replaceHeader(HttpHeaders.HOST, host);
signedHeadersBuilder.put(HttpHeaders.HOST.toLowerCase(), host);
// user-agent, not a required signing param
if (request.getHeaders().containsKey(HttpHeaders.USER_AGENT)) {
signedHeadersBuilder.put(HttpHeaders.USER_AGENT.toLowerCase(),
request.getFirstHeaderOrNull(HttpHeaders.USER_AGENT));
}
// all x-amz-* headers
appendAmzHeaders(request, signedHeadersBuilder);
// x-amz-security-token
Credentials credentials = creds.get();
if (credentials instanceof SessionCredentials) {
String token = SessionCredentials.class.cast(credentials).getSessionToken();
requestBuilder.replaceHeader(AMZ_SECURITY_TOKEN_HEADER, token);
signedHeadersBuilder.put(AMZ_SECURITY_TOKEN_HEADER.toLowerCase(), token);
}
// x-amz-content-sha256
String contentSha256 = getPayloadHash();
requestBuilder.replaceHeader(AMZ_CONTENT_SHA256_HEADER, contentSha256);
signedHeadersBuilder.put(AMZ_CONTENT_SHA256_HEADER.toLowerCase(), contentSha256);
// put x-amz-date
requestBuilder.replaceHeader(AMZ_DATE_HEADER, timestamp);
signedHeadersBuilder.put(AMZ_DATE_HEADER.toLowerCase(), timestamp);
ImmutableMap<String, String> signedHeaders = signedHeadersBuilder.build();
String stringToSign = createStringToSign(request.getMethod(), request.getEndpoint(), signedHeaders, timestamp,
credentialScope, contentSha256);
signatureWire.getWireLog().debug("<< " + stringToSign);
byte[] signatureKey = signatureKey(credentials.credential, datestamp, region, service);
// init hmacSHA256 processor for seed signature and chunked block signature
ByteProcessor<byte[]> hmacSHA256;
try {
hmacSHA256 = hmacSHA256(crypto, signatureKey);
} catch (InvalidKeyException e) {
throw new ChunkedUploadException("invalid key", e);
}
// Calculating the Seed Signature
String signature;
try {
signature = hex(readBytes(toInputStream(stringToSign), hmacSHA256));
} catch (IOException e) {
throw new ChunkedUploadException("hmac sha256 seed signature error", e);
}
StringBuilder authorization = new StringBuilder(AMZ_ALGORITHM_HMAC_SHA256).append(" ");
authorization.append("Credential=").append(Joiner.on("/").join(credentials.identity, credentialScope))
.append(", ");
authorization.append("SignedHeaders=").append(Joiner.on(";").join(signedHeaders.keySet()))
.append(", ");
authorization.append("Signature=").append(signature);
// replace request payload with chunked upload payload
ChunkedUploadPayload chunkedPayload = new ChunkedUploadPayload(payload, userDataBlockSize, timestamp,
credentialScope, hmacSHA256, signature);
return request.toBuilder()
.replaceHeader(HttpHeaders.AUTHORIZATION, authorization.toString())
.payload(chunkedPayload)
.build();
}
// for seed signature, value: STREAMING-AWS4-HMAC-SHA256-PAYLOAD
protected String getPayloadHash() {
return STREAMING_BODY_SHA256;
}
/**
* Calculates the expanded payload size of our data when it is chunked
*
* @param originalLength The true size of the data payload to be uploaded
* @param chunkSize The size of each chunk we intend to send; each chunk will be
* prefixed with signed header data, expanding the overall size
* by a determinable amount
* @return The overall payload size to use as content-length on a chunked
* upload
*/
public static long calculateChunkedContentLength(long originalLength, long chunkSize) {
checkArgument(originalLength > 0, "Nonnegative content length expected.");
long maxSizeChunks = originalLength / chunkSize;
long remainingBytes = originalLength % chunkSize;
return maxSizeChunks * calculateChunkHeaderLength(chunkSize)
+ (remainingBytes > 0 ? calculateChunkHeaderLength(remainingBytes) : 0)
+ calculateChunkHeaderLength(0);
}
/**
* Returns the size of a chunk header, which only varies depending on the
* selected chunk size
*
* @param chunkDataSize The intended size of each chunk; this is placed into the chunk
* header
* @return The overall size of the header that will prefix the user data in
* each chunk
*/
private static long calculateChunkHeaderLength(long chunkDataSize) {
return Long.toHexString(chunkDataSize).length()
+ CHUNK_SIGNATURE_HEADER.length()
+ SIGNATURE_LENGTH
+ CLRF.length()
+ chunkDataSize
+ CLRF.length();
}
}

150
apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForQueryString.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_ALGORITHM_PARAM;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_CONTENT_SHA256_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_CREDENTIAL_PARAM;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DATE_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DATE_PARAM;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_EXPIRES_PARAM;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SECURITY_TOKEN_PARAM;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SIGNATURE_PARAM;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SIGNEDHEADERS_PARAM;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AUTHORIZATION_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.UNSIGNED_PAYLOAD;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.io.BaseEncoding.base16;
import java.util.Date;
import javax.inject.Named;
import org.jclouds.aws.domain.SessionCredentials;
import org.jclouds.crypto.Crypto;
import org.jclouds.date.TimeStamp;
import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.Uris;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.location.Provider;
import com.google.common.base.Joiner;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.inject.Inject;
/**
* AWS4 signer sign requests to Amazon S3 using query string parameters.
*/
public class Aws4SignerForQueryString extends Aws4SignerBase {
@Inject
public Aws4SignerForQueryString(SignatureWire signatureWire,
@Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
@Named(PROPERTY_HEADER_TAG) String headerTag,
@Provider Supplier<Credentials> creds, @TimeStamp Supplier<Date> timestampProvider,
ServiceAndRegion serviceAndRegion, Crypto crypto) {
super(signatureWire, headerTag, creds, timestampProvider, serviceAndRegion, crypto);
}
protected HttpRequest sign(HttpRequest request, long timeInSeconds) throws HttpException {
checkNotNull(request, "request is not ready to sign");
checkNotNull(request.getEndpoint(), "request is not ready to sign, request.endpoint not present.");
// get host from request endpoint.
String host = request.getEndpoint().getHost();
Date date = timestampProvider.get();
String timestamp = timestampFormat.format(date);
String datestamp = dateFormat.format(date);
String service = serviceAndRegion.service();
String region = serviceAndRegion.region(host);
String credentialScope = Joiner.on('/').join(datestamp, region, service, "aws4_request");
// different with signature with Authorization header
HttpRequest.Builder<?> requestBuilder = request.toBuilder() //
// sign for temporary access use query string parameter:
// X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, X-Amz-Expires, X-Amz-SignedHeaders, X-Amz-Signature
// remove Authorization, x-amz-content-sha256, X-Amz-Date headers
.removeHeader(AUTHORIZATION_HEADER)
.removeHeader(AMZ_CONTENT_SHA256_HEADER)
.removeHeader(AMZ_DATE_HEADER);
ImmutableMap.Builder<String, String> signedHeadersBuilder = ImmutableSortedMap.<String, String>naturalOrder(); //
Uris.UriBuilder endpointBuilder = Uris.uriBuilder(request.getEndpoint());
// Canonical Headers
// must include the HTTP host header.
// If you plan to include any of the x-amz-* headers, these headers must also be added for signature calculation.
// You can optionally add all other headers that you plan to include in your request.
// For added security, you should sign as many headers as possible.
// HOST
signedHeadersBuilder.put("host", host);
ImmutableMap<String, String> signedHeaders = signedHeadersBuilder.build();
Credentials credentials = creds.get();
if (credentials instanceof SessionCredentials) {
String token = SessionCredentials.class.cast(credentials).getSessionToken();
// different with signature with Authorization header
endpointBuilder.replaceQuery(AMZ_SECURITY_TOKEN_PARAM, token);
}
// X-Amz-Algorithm=HMAC-SHA256
endpointBuilder.replaceQuery(AMZ_ALGORITHM_PARAM, AwsSignatureV4Constants.AMZ_ALGORITHM_HMAC_SHA256);
// X-Amz-Credential=<your-access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request.
String credential = Joiner.on("/").join(credentials.identity, credentialScope);
endpointBuilder.replaceQuery(AMZ_CREDENTIAL_PARAM, credential);
// X-Amz-Date=ISO 8601 format, for example, 20130721T201207Z
endpointBuilder.replaceQuery(AMZ_DATE_PARAM, timestamp);
// X-Amz-Expires=time in seconds
endpointBuilder.replaceQuery(AMZ_EXPIRES_PARAM, String.valueOf(timeInSeconds));
// X-Amz-SignedHeaders=HTTP host header is required.
endpointBuilder.replaceQuery(AMZ_SIGNEDHEADERS_PARAM, Joiner.on(';').join(signedHeaders.keySet()));
String stringToSign = createStringToSign(request.getMethod(), endpointBuilder.build(), signedHeaders, timestamp, credentialScope,
getPayloadHash());
signatureWire.getWireLog().debug("<< " + stringToSign);
byte[] signatureKey = signatureKey(credentials.credential, datestamp, region, service);
String signature = base16().lowerCase().encode(hmacSHA256(stringToSign, signatureKey));
// X-Amz-Signature=Signature
endpointBuilder.replaceQuery(AMZ_SIGNATURE_PARAM, signature);
return requestBuilder.endpoint(endpointBuilder.build()).build();
}
protected String getPayloadHash() {
return UNSIGNED_PAYLOAD;
}
}

186
apis/s3/src/main/java/org/jclouds/s3/filters/AwsHostNameUtils.java Normal file
View File

@ -0,0 +1,186 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import com.google.common.collect.ImmutableMap;
import java.net.URI;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class AwsHostNameUtils {
private static final Pattern S3_ENDPOINT_PATTERN = Pattern.compile("^(?:.+\\.)?s3[.-]([a-z0-9-]+)$");
private static final Pattern STANDARD_CLOUDSEARCH_ENDPOINT_PATTERN = Pattern.compile("^(?:.+\\.)?([a-z0-9-]+)\\.cloudsearch$");
private static final Pattern EXTENDED_CLOUDSEARCH_ENDPOINT_PATTERN = Pattern.compile("^(?:.+\\.)?([a-z0-9-]+)\\.cloudsearch\\..+");
private static final ImmutableMap<String, String> HOST_REGEX_TO_REGION_MAPPINGS = new ImmutableMap.Builder<String, String>()
.put("(.+\\.)?s3\\.amazonaws\\.com", "us-east-1")
.put("(.+\\.)?s3-external-1\\.amazonaws\\.com", "us-east-1")
.put("(.+\\.)?s3-fips-us-gov-west-1\\.amazonaws\\.com", "us-gov-west-1")
.build();
/**
* Attempts to parse the region name from an endpoint based on conventions
* about the endpoint format.
*
* @param host the hostname to parse
* @param serviceHint an optional hint about the service for the endpoint
* @return the region parsed from the hostname, or
* &quot;us-east-1&quot; if no region information
* could be found
*/
public static String parseRegionName(final String host, final String serviceHint) {
String regionNameInInternalConfig = parseRegionNameByInternalConfig(host);
if (regionNameInInternalConfig != null) {
return regionNameInInternalConfig;
}
if (host.endsWith(".amazonaws.com")) {
int index = host.length() - ".amazonaws.com".length();
return parseStandardRegionName(host.substring(0, index));
}
if (serviceHint != null) {
if (serviceHint.equals("cloudsearch")
&& !host.startsWith("cloudsearch.")) {
// CloudSearch domains use the nonstandard domain format
// [domain].[region].cloudsearch.[suffix].
Matcher matcher = EXTENDED_CLOUDSEARCH_ENDPOINT_PATTERN
.matcher(host);
if (matcher.matches()) {
return matcher.group(1);
}
}
// If we have a service hint, look for 'service.[region]' or
// 'service-[region]' in the endpoint's hostname.
Pattern pattern = Pattern.compile(
"^(?:.+\\.)?"
+ Pattern.quote(serviceHint)
+ "[.-]([a-z0-9-]+)\\."
);
Matcher matcher = pattern.matcher(host);
if (matcher.find()) {
return matcher.group(1);
}
}
// Endpoint is totally non-standard; guess us-east-1 for lack of a
// better option.
return "us-east-1";
}
/**
* Parses the region name from a standard (*.amazonaws.com) endpoint.
*
* @param fragment the portion of the endpoint excluding
* &quot;.amazonaws.com&quot;
* @return the parsed region name (or &quot;us-east-1&quot; as a
* best guess if we can't tell for sure)
*/
private static String parseStandardRegionName(final String fragment) {
Matcher matcher = S3_ENDPOINT_PATTERN.matcher(fragment);
if (matcher.matches()) {
// host was 'bucket.s3-[region].amazonaws.com'.
return matcher.group(1);
}
matcher = STANDARD_CLOUDSEARCH_ENDPOINT_PATTERN.matcher(fragment);
if (matcher.matches()) {
// host was 'domain.[region].cloudsearch.amazonaws.com'.
return matcher.group(1);
}
int index = fragment.lastIndexOf('.');
if (index == -1) {
// host was 'service.amazonaws.com', guess us-east-1
// for lack of a better option.
return "us-east-1";
}
// host was 'service.[region].amazonaws.com'.
String region = fragment.substring(index + 1);
// Special case for iam.us-gov.amazonaws.com, which is actually
// us-gov-west-1.
if ("us-gov".equals(region)) {
region = "us-gov-west-1";
}
return region;
}
/**
* @return the configured region name if the given host name matches any of
* the host-to-region mappings in the internal config; otherwise
* return null.
*/
private static String parseRegionNameByInternalConfig(String host) {
for (Map.Entry<String, String> mapping : HOST_REGEX_TO_REGION_MAPPINGS.entrySet()) {
String hostNameRegex = mapping.getKey();
if (host.matches(hostNameRegex)) {
return mapping.getValue();
}
}
return null;
}
/**
* Parses the service name from an endpoint. Can only handle endpoints of
* the form 'service.[region.]amazonaws.com'.
* or
* bucket.s3.[region.]awazonaws.com
*/
public static String parseServiceName(URI endpoint) {
String host = endpoint.getHost();
if (!host.endsWith(".amazonaws.com") && !host.endsWith(".amazonaws.com.cn")) {
return "s3"; // cannot parse name, assume s3
}
String serviceAndRegion = host.substring(0, host.indexOf(".amazonaws.com"));
// Special cases for S3 endpoints with bucket names embedded.
if (serviceAndRegion.endsWith(".s3") || S3_ENDPOINT_PATTERN.matcher(serviceAndRegion).matches()) {
return "s3";
}
char separator = '.';
// If we don't detect a separator between service name and region, then
// assume that the region is not included in the hostname, and it's only
// the service name (ex: "http://iam.amazonaws.com").
if (serviceAndRegion.indexOf(separator) == -1) {
return serviceAndRegion;
}
return serviceAndRegion.substring(0, serviceAndRegion.indexOf(separator));
}
}

151
apis/s3/src/main/java/org/jclouds/s3/filters/AwsSignatureV4Constants.java Normal file
View File

@ -0,0 +1,151 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
/**
* AWS Signature Version 4 Constants.
*/
public abstract class AwsSignatureV4Constants {
/**
* AWS authorization header key
*/
public static final String AUTHORIZATION_HEADER = "Authorization";
/**
* AWS content sha256 header key
*/
public static final String AMZ_CONTENT_SHA256_HEADER = "x-amz-content-sha256";
/**
* AWS date header key
*/
public static final String AMZ_DATE_HEADER = "X-Amz-Date";
/**
* AWS security token key
*/
public static final String AMZ_SECURITY_TOKEN_HEADER = "X-Amz-Security-Token";
/**
* For AWS Signature Version 4, you set this parameter value to "AWS4-HMAC-SHA256".
*/
public static final String AMZ_ALGORITHM_PARAM = "X-Amz-Algorithm";
/**
* This string identifies AWS Signature Version 4 (AWS4) and the HMAC-SHA256 algorithm (HMAC-SHA256).
*/
public static final String AMZ_ALGORITHM_HMAC_SHA256 = "AWS4-HMAC-SHA256";
/**
* In addition to your access key ID, this parameter also provides scope information identifying the region and
* service for which the signature is valid.
* <p>This value should match the scope that you use to calculate the signing key, as discussed in the following section.</p>
* <p>The general form for this parameter value is as follows:</p>
* <code> &lt;your-access-key-id>/&lt;date>/&lt;AWS-region>/&lt;AWS-service>/aws4_request.</code>
* <p>
* For example:
* <code>AKIAIOSFODNN7EXAMPLE/20130721/us-east-1/s3/aws4_request.</code><br>
* For Amazon S3, the AWS-service string is "s3". For a list of AWS-region strings, go to Regions and Endpoints
* in the Amazon Web Services General Reference
* </p>
*/
public static final String AMZ_CREDENTIAL_PARAM = "X-Amz-Credential";
/**
* This header can be used in the following scenarios:
* <ul>
* <li>Provide security tokens for Amazon DevPay operations—Each request that uses Amazon DevPay requires two
* x-amz-security-token headers: one for the product token and one for the user token. When Amazon S3 receives
* an authenticated request, it compares the computed signature with the provided signature.
* Improperly formatted multi-value headers used to calculate a signature can cause authentication issues</li>
* <li>Provide security token when using temporary security credentials—When making requests using temporary
* security credentials you obtained from IAM you must provide a security token using this header.
* To learn more about temporary security credentials, go to Making Requests.</li>
* </ul>
* This header is required for requests that use Amazon DevPay and requests that are signed using temporary security credentials.
*/
public static final String AMZ_SECURITY_TOKEN_PARAM = AMZ_SECURITY_TOKEN_HEADER;
/**
* The date in ISO 8601 format, for example, 20130721T201207Z. This value must match the date value used to
* calculate the signature.
*/
public static final String AMZ_DATE_PARAM = AMZ_DATE_HEADER;
/**
* Provides the time period, in seconds, for which the generated presigned URL is valid.
* <p> For example, 86400 (24 hours). This value is an integer. The minimum value you can set is 1,
* and the maximum is 604800 (seven days). </p>
* <p> A presigned URL can be valid for a maximum of seven days because the signing key you use in signature
* calculation is valid for up to seven days.</p>
*/
public static final String AMZ_EXPIRES_PARAM = "X-Amz-Expires";
/**
* Lists the headers that you used to calculate the signature.
* <p> The HTTP host header is required. Any x-amz-* headers that you plan to add to the request are also required
* for signature calculation. </p>
* <p> In general, for added security, you should sign all the request headers that you plan to include in your
* request.</p>
*/
public static final String AMZ_SIGNEDHEADERS_PARAM = "X-Amz-SignedHeaders";
/**
* X-Amz-Signature Provides the signature to authenticate your request.
* <p>This signature must match the signature Amazon S3 calculates; otherwise, Amazon S3 denies the request.
* For example, 733255ef022bec3f2a8701cd61d4b371f3f28c9f193a1f02279211d48d5193d7</p>
*/
public static final String AMZ_SIGNATURE_PARAM = "X-Amz-Signature";
/**
* You don't include a payload hash in the Canonical Request, because when you create a presigned URL,
* <p> you don't know anything about the payload. Instead, you use a constant string "UNSIGNED-PAYLOAD".</p>
*/
public static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
/**
* SHA256 substitute marker used in place of x-amz-content-sha256 when
* employing chunked uploads
*/
public static final String STREAMING_BODY_SHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
public static final String CHUNK_STRING_TO_SIGN_PREFIX = "AWS4-HMAC-SHA256-PAYLOAD";
public static final String CLRF = "\r\n";
public static final String CHUNK_SIGNATURE_HEADER = ";chunk-signature=";
public static final int SIGNATURE_LENGTH = 64;
public static final byte[] FINAL_CHUNK = new byte[0];
/**
* Content-Encoding
* <p>
* Set the value to aws-chunked.<br>
* Amazon S3 supports multiple content encodings, for example,<br>
* Content-Encoding : aws-chunked, gzip<br>
* That is, you can specify your custom content-encoding when using Signature Version 4 streaming API.
* </p>
*/
public static final String CONTENT_ENCODING_HEADER_AWS_CHUNKED = "aws-chunked";
/**
* 'x-amz-decoded-content-length' is used to transmit the actual
*/
public static final String AMZ_DECODED_CONTENT_LENGTH_HEADER = "x-amz-decoded-content-length";
private AwsSignatureV4Constants() {
}
}

23
apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadException.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
public class ChunkedUploadException extends RuntimeException {
public ChunkedUploadException(String error, Exception e) {
super(error, e);
}
}

217
apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadPayload.java Normal file
View File

@ -0,0 +1,217 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import com.google.common.io.ByteProcessor;
import com.google.common.io.ByteStreams;
import org.jclouds.http.HttpException;
import org.jclouds.io.MutableContentMetadata;
import org.jclouds.io.Payload;
import org.jclouds.io.payloads.BaseMutableContentMetadata;
import org.jclouds.io.payloads.BasePayload;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.SequenceInputStream;
import java.util.Enumeration;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.io.ByteStreams.readBytes;
import static org.jclouds.s3.filters.Aws4SignerBase.hash;
import static org.jclouds.s3.filters.Aws4SignerBase.hex;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.CHUNK_SIGNATURE_HEADER;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.CHUNK_STRING_TO_SIGN_PREFIX;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.CLRF;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.FINAL_CHUNK;
import static org.jclouds.util.Strings2.toInputStream;
public class ChunkedUploadPayload extends BasePayload<Payload> {
private static final byte[] TRAILER = CLRF.getBytes(UTF_8);
private final Payload payload;
private final int chunkedBlockSize;
private final String timestamp;
private final String scope;
private final ByteProcessor<byte[]> hmacSHA256;
private String lastComputedSignature;
public ChunkedUploadPayload(Payload payload, int blockSize, String timestamp, String scope,
ByteProcessor<byte[]> hmacSHA256, String seedSignature) {
super(payload);
this.payload = payload;
this.chunkedBlockSize = blockSize;
this.timestamp = timestamp;
this.scope = scope;
this.hmacSHA256 = hmacSHA256;
this.lastComputedSignature = seedSignature;
// init content metadata
MutableContentMetadata contentMetadata = BaseMutableContentMetadata.fromContentMetadata(
payload.getContentMetadata());
long totalLength = Aws4SignerForChunkedUpload.calculateChunkedContentLength(
payload.getContentMetadata().getContentLength(),
chunkedBlockSize);
contentMetadata.setContentLength(totalLength);
this.setContentMetadata(contentMetadata);
}
/**
* Returns a chunk for upload consisting of the signed 'header' or chunk
* prefix plus the user data. The signature of the chunk incorporates the
* signature of the previous chunk (or, if the first chunk, the signature of
* the headers portion of the request).
*
* @param userDataLen The length of the user data contained in userData
* @param userData Contains the user data to be sent in the upload chunk
* @return A new buffer of data for upload containing the chunk header plus
* user data
*/
protected byte[] constructSignedChunk(int userDataLen, byte[] userData) {
// to keep our computation routine signatures simple, if the userData
// buffer contains less data than it could, shrink it. Note the special case
// to handle the requirement that we send an empty chunk to complete
// our chunked upload.
byte[] dataToChunk;
if (userDataLen == 0) {
dataToChunk = FINAL_CHUNK;
} else {
if (userDataLen < userData.length) {
// shrink the chunkdata to fit
dataToChunk = new byte[userDataLen];
System.arraycopy(userData, 0, dataToChunk, 0, userDataLen);
} else {
dataToChunk = userData;
}
}
// string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n
StringBuilder chunkHeader = new StringBuilder();
// start with size of user data
// IntHexBase(chunk-size)
chunkHeader.append(Integer.toHexString(dataToChunk.length));
// chunk-signature
// nonsig-extension; we have none in these samples
String nonsigExtension = "";
// if this is the first chunk, we package it with the signing result
// of the request headers, otherwise we use the cached signature
// of the previous chunk
// sig-extension
StringBuilder buffer = new StringBuilder();
buffer.append(CHUNK_STRING_TO_SIGN_PREFIX);
buffer.append("\n");
buffer.append(timestamp).append("\n");
buffer.append(scope).append("\n");
buffer.append(lastComputedSignature).append("\n");
buffer.append(hex(hash(nonsigExtension))).append("\n");
buffer.append(hex(hash(dataToChunk)));
String chunkStringToSign = buffer.toString();
// compute the V4 signature for the chunk
String chunkSignature;
try {
chunkSignature = hex(readBytes(toInputStream(chunkStringToSign), hmacSHA256));
} catch (IOException e) {
throw new HttpException("hmac sha256 chunked signature error");
}
// cache the signature to include with the next chunk's signature computation
lastComputedSignature = chunkSignature;
// construct the actual chunk, comprised of the non-signed extensions, the
// 'headers' we just signed and their signature, plus a newline then copy
// that plus the user's data to a payload to be written to the request stream
chunkHeader.append(nonsigExtension + CHUNK_SIGNATURE_HEADER + chunkSignature);
chunkHeader.append(CLRF);
byte[] header = chunkHeader.toString().getBytes(UTF_8);
byte[] signedChunk = new byte[header.length + dataToChunk.length + TRAILER.length];
System.arraycopy(header, 0, signedChunk, 0, header.length);
// chunk-data
System.arraycopy(dataToChunk, 0, signedChunk, header.length, dataToChunk.length);
System.arraycopy(TRAILER, 0, signedChunk, header.length + dataToChunk.length, TRAILER.length);
// this is the total data for the chunk that will be sent to the request stream
return signedChunk;
}
@Override
public void release() {
this.payload.release();
}
@Override
public boolean isRepeatable() {
return this.payload.isRepeatable();
}
@Override
public InputStream openStream() throws IOException {
return new SequenceInputStream(new ChunkedInputStreamEnumeration(this.payload.openStream(), chunkedBlockSize));
}
private class ChunkedInputStreamEnumeration implements Enumeration<InputStream> {
private final InputStream inputStream;
private boolean lastChunked;
private byte[] buffer;
ChunkedInputStreamEnumeration(InputStream inputStream, int chunkedBlockSize) {
this.inputStream = new BufferedInputStream(inputStream, chunkedBlockSize);
buffer = new byte[chunkedBlockSize];
lastChunked = false;
}
@Override
public boolean hasMoreElements() {
return !lastChunked;
}
@Override
public InputStream nextElement() {
int bytesRead;
try {
bytesRead = ByteStreams.read(inputStream, buffer, 0, buffer.length);
} catch (IOException e) {
// IO EXCEPTION
throw new ChunkedUploadException("read from input stream error", e);
}
// buffer
byte[] chunk;
// ByteStreams.read(InputStream, byte[], int, int) returns the number of bytes read
// InputStream.read(byte[], int, int) returns -1 if the end of the stream has been reached.
if (bytesRead > 0) {
// process into a chunk
chunk = constructSignedChunk(bytesRead, buffer);
} else {
// construct last chunked block
chunk = constructSignedChunk(0, buffer);
lastChunked = true;
}
return new ByteArrayInputStream(chunk);
}
}
}

234
apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java
View File

@ -16,243 +16,11 @@
*/
package org.jclouds.s3.filters;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.collect.Iterables.get;
import static com.google.common.io.BaseEncoding.base64;
import static com.google.common.io.ByteStreams.readBytes;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.crypto.Macs.asByteProcessor;
import static org.jclouds.http.utils.Queries.queryParser;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
import static org.jclouds.util.Strings2.toInputStream;
import java.util.Collection;
import java.util.Locale;
import java.util.Map.Entry;
import java.util.Set;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.aws.domain.SessionCredentials;
import org.jclouds.crypto.Crypto;
import org.jclouds.date.TimeStamp;
import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.logging.Logger;
import org.jclouds.rest.RequestSigner;
import org.jclouds.s3.util.S3Utils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.common.collect.Ordering;
import com.google.common.collect.SortedSetMultimap;
import com.google.common.collect.TreeMultimap;
import com.google.common.io.ByteProcessor;
import com.google.common.net.HttpHeaders;
/**
* Signs the S3 request.
*/
@Singleton
public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSigner {
private static final Collection<String> FIRST_HEADERS_TO_SIGN = ImmutableList.of(HttpHeaders.DATE);
private static final Set<String> SIGNED_PARAMETERS = ImmutableSet.of("acl", "torrent", "logging", "location", "policy",
"requestPayment", "versioning", "versions", "versionId", "notification", "uploadId", "uploads",
"partNumber", "website", "response-content-type", "response-content-language", "response-expires",
"response-cache-control", "response-content-disposition", "response-content-encoding", "delete");
private final SignatureWire signatureWire;
private final Supplier<Credentials> creds;
private final Provider<String> timeStampProvider;
private final Crypto crypto;
private final HttpUtils utils;
@Resource
@Named(Constants.LOGGER_SIGNATURE)
Logger signatureLog = Logger.NULL;
private final String authTag;
private final String headerTag;
private final String servicePath;
private final boolean isVhostStyle;
@Inject
public RequestAuthorizeSignature(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,
@Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
@Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
@org.jclouds.location.Provider Supplier<Credentials> creds,
@TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
this.isVhostStyle = isVhostStyle;
this.servicePath = servicePath;
this.headerTag = headerTag;
this.authTag = authTag;
this.signatureWire = signatureWire;
this.creds = creds;
this.timeStampProvider = timeStampProvider;
this.crypto = crypto;
this.utils = utils;
}
public HttpRequest filter(HttpRequest request) throws HttpException {
request = replaceDateHeader(request);
Credentials current = creds.get();
if (current instanceof SessionCredentials) {
request = replaceSecurityTokenHeader(request, SessionCredentials.class.cast(current));
}
String signature = calculateSignature(createStringToSign(request));
request = replaceAuthorizationHeader(request, signature);
utils.logRequest(signatureLog, request, "<<");
return request;
}
HttpRequest replaceSecurityTokenHeader(HttpRequest request, SessionCredentials current) {
return request.toBuilder().replaceHeader("x-amz-security-token", current.getSessionToken()).build();
}
protected HttpRequest replaceAuthorizationHeader(HttpRequest request, String signature) {
request = request.toBuilder()
.replaceHeader(HttpHeaders.AUTHORIZATION, authTag + " " + creds.get().identity + ":" + signature).build();
return request;
}
HttpRequest replaceDateHeader(HttpRequest request) {
request = request.toBuilder().replaceHeader(HttpHeaders.DATE, timeStampProvider.get()).build();
return request;
}
public String createStringToSign(HttpRequest request) {
utils.logRequest(signatureLog, request, ">>");
SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
StringBuilder buffer = new StringBuilder();
// re-sign the request
appendMethod(request, buffer);
appendPayloadMetadata(request, buffer);
appendHttpHeaders(request, canonicalizedHeaders);
// Remove default date timestamp if "x-amz-date" is set.
if (canonicalizedHeaders.containsKey("x-" + headerTag + "-date")) {
canonicalizedHeaders.removeAll("date");
}
appendAmzHeaders(canonicalizedHeaders, buffer);
appendBucketName(request, buffer);
appendUriPath(request, buffer);
if (signatureWire.enabled())
signatureWire.output(buffer.toString());
return buffer.toString();
}
String calculateSignature(String toSign) throws HttpException {
String signature = sign(toSign);
if (signatureWire.enabled())
signatureWire.input(toInputStream(signature));
return signature;
}
public String sign(String toSign) {
try {
ByteProcessor<byte[]> hmacSHA1 = asByteProcessor(crypto.hmacSHA1(creds.get().credential.getBytes(UTF_8)));
return base64().encode(readBytes(toInputStream(toSign), hmacSHA1));
} catch (Exception e) {
throw new HttpException("error signing request", e);
}
}
void appendMethod(HttpRequest request, StringBuilder toSign) {
toSign.append(request.getMethod()).append("\n");
}
@VisibleForTesting
void appendAmzHeaders(SortedSetMultimap<String, String> canonicalizedHeaders, StringBuilder toSign) {
for (Entry<String, String> header : canonicalizedHeaders.entries()) {
String key = header.getKey();
if (key.startsWith("x-" + headerTag + "-")) {
toSign.append(String.format("%s:%s\n", key.toLowerCase(), header.getValue()));
}
}
}
void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
// note that we fall back to headers, and some requests such as ?uploads do not have a
// payload, yet specify payload related parameters
buffer.append(
request.getPayload() == null ? Strings.nullToEmpty(request.getFirstHeaderOrNull("Content-MD5")) :
HttpUtils.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
.getContentMD5())).append("\n");
buffer.append(
Strings.nullToEmpty(request.getPayload() == null ? request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE)
: request.getPayload().getContentMetadata().getContentType())).append("\n");
for (String header : FIRST_HEADERS_TO_SIGN)
buffer.append(HttpUtils.nullToEmpty(request.getHeaders().get(header))).append("\n");
}
@VisibleForTesting
void appendHttpHeaders(HttpRequest request, SortedSetMultimap<String, String> canonicalizedHeaders) {
Multimap<String, String> headers = request.getHeaders();
for (Entry<String, String> header : headers.entries()) {
if (header.getKey() == null)
continue;
String key = header.getKey().toString().toLowerCase(Locale.getDefault());
// Ignore any headers that are not particularly interesting.
if (key.equalsIgnoreCase(HttpHeaders.CONTENT_TYPE) || key.equalsIgnoreCase("Content-MD5")
|| key.equalsIgnoreCase(HttpHeaders.DATE) || key.startsWith("x-" + headerTag + "-")) {
canonicalizedHeaders.put(key, header.getValue());
}
}
}
@VisibleForTesting
void appendBucketName(HttpRequest req, StringBuilder toSign) {
String bucketName = S3Utils.getBucketName(req);
// If we have a payload/bucket/container that is not all lowercase, vhost-style URLs are not an option and must be
// automatically converted to their path-based equivalent. This should only be possible for AWS-S3 since it is
// the only S3 implementation configured to allow uppercase payload/bucket/container names.
//
// http://code.google.com/p/jclouds/issues/detail?id=992
if (isVhostStyle && bucketName != null && bucketName.equals(bucketName.toLowerCase()))
toSign.append(servicePath).append(bucketName);
}
@VisibleForTesting
void appendUriPath(HttpRequest request, StringBuilder toSign) {
toSign.append(request.getEndpoint().getRawPath());
// ...however, there are a few exceptions that must be included in the
// signed URI.
if (request.getEndpoint().getQuery() != null) {
Multimap<String, String> params = queryParser().apply(request.getEndpoint().getQuery());
char separator = '?';
for (String paramName : Ordering.natural().sortedCopy(params.keySet())) {
// Skip any parameters that aren't part of the canonical signed string
if (!SIGNED_PARAMETERS.contains(paramName))
continue;
toSign.append(separator).append(paramName);
String paramValue = get(params.get(paramName), 0);
if (paramValue != null) {
toSign.append("=").append(paramValue);
}
separator = '&';
}
}
}
public interface RequestAuthorizeSignature extends HttpRequestFilter {
}

264
apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2.java Normal file
View File

@ -0,0 +1,264 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.collect.Iterables.get;
import static com.google.common.io.BaseEncoding.base64;
import static com.google.common.io.ByteStreams.readBytes;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.crypto.Macs.asByteProcessor;
import static org.jclouds.http.utils.Queries.queryParser;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
import static org.jclouds.util.Strings2.toInputStream;
import java.util.Collection;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Provider;
import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.aws.domain.SessionCredentials;
import org.jclouds.crypto.Crypto;
import org.jclouds.date.TimeStamp;
import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.logging.Logger;
import org.jclouds.rest.RequestSigner;
import org.jclouds.s3.util.S3Utils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.common.collect.Ordering;
import com.google.common.collect.SortedSetMultimap;
import com.google.common.collect.TreeMultimap;
import com.google.common.io.ByteProcessor;
import com.google.common.net.HttpHeaders;
/**
* AWS Sign V2
*/
@Singleton
public class RequestAuthorizeSignatureV2 implements RequestAuthorizeSignature, RequestSigner {
private static final Collection<String> FIRST_HEADERS_TO_SIGN = ImmutableList.of(HttpHeaders.DATE);
private static final Set<String> SIGNED_PARAMETERS = ImmutableSet.of("acl", "torrent", "logging", "location",
"policy", "requestPayment", "versioning", "versions", "versionId", "notification", "uploadId", "uploads",
"partNumber", "website", "response-content-type", "response-content-language", "response-expires",
"response-cache-control", "response-content-disposition", "response-content-encoding", "delete");
private final SignatureWire signatureWire;
private final Supplier<Credentials> creds;
private final Provider<String> timeStampProvider;
private final Crypto crypto;
private final HttpUtils utils;
@Resource
@Named(Constants.LOGGER_SIGNATURE)
Logger signatureLog = Logger.NULL;
private final String authTag;
private final String headerTag;
private final String servicePath;
private final boolean isVhostStyle;
@Inject
public RequestAuthorizeSignatureV2(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,
@Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
@Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
@org.jclouds.location.Provider Supplier<Credentials> creds,
@TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
this.isVhostStyle = isVhostStyle;
this.servicePath = servicePath;
this.headerTag = headerTag;
this.authTag = authTag;
this.signatureWire = signatureWire;
this.creds = creds;
this.timeStampProvider = timeStampProvider;
this.crypto = crypto;
this.utils = utils;
}
public HttpRequest filter(HttpRequest request) throws HttpException {
request = replaceDateHeader(request);
Credentials current = creds.get();
if (current instanceof SessionCredentials) {
request = replaceSecurityTokenHeader(request, SessionCredentials.class.cast(current));
}
String signature = calculateSignature(createStringToSign(request));
request = replaceAuthorizationHeader(request, signature);
utils.logRequest(signatureLog, request, "<<");
return request;
}
HttpRequest replaceSecurityTokenHeader(HttpRequest request, SessionCredentials current) {
return request.toBuilder().replaceHeader("x-amz-security-token", current.getSessionToken()).build();
}
protected HttpRequest replaceAuthorizationHeader(HttpRequest request, String signature) {
request = request.toBuilder()
.replaceHeader(HttpHeaders.AUTHORIZATION,
authTag + " " + creds.get().identity + ":" + signature).build();
return request;
}
HttpRequest replaceDateHeader(HttpRequest request) {
request = request.toBuilder().replaceHeader(HttpHeaders.DATE, timeStampProvider.get()).build();
return request;
}
public String createStringToSign(HttpRequest request) {
utils.logRequest(signatureLog, request, ">>");
SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
StringBuilder buffer = new StringBuilder();
// re-sign the request
appendMethod(request, buffer);
appendPayloadMetadata(request, buffer);
appendHttpHeaders(request, canonicalizedHeaders);
// Remove default date timestamp if "x-amz-date" is set.
if (canonicalizedHeaders.containsKey("x-" + headerTag + "-date")) {
canonicalizedHeaders.removeAll("date");
}
appendAmzHeaders(canonicalizedHeaders, buffer);
appendBucketName(request, buffer);
appendUriPath(request, buffer);
if (signatureWire.enabled()) {
signatureWire.output(buffer.toString());
}
return buffer.toString();
}
String calculateSignature(String toSign) throws HttpException {
String signature = sign(toSign);
if (signatureWire.enabled()) {
signatureWire.input(toInputStream(signature));
}
return signature;
}
public String sign(String toSign) {
try {
ByteProcessor<byte[]> hmacSHA1 = asByteProcessor(
crypto.hmacSHA1(creds.get().credential.getBytes(UTF_8)));
return base64().encode(readBytes(toInputStream(toSign), hmacSHA1));
} catch (Exception e) {
throw new HttpException("error signing request", e);
}
}
void appendMethod(HttpRequest request, StringBuilder toSign) {
toSign.append(request.getMethod()).append("\n");
}
@VisibleForTesting
void appendAmzHeaders(SortedSetMultimap<String, String> canonicalizedHeaders, StringBuilder toSign) {
for (Map.Entry<String, String> header : canonicalizedHeaders.entries()) {
String key = header.getKey();
if (key.startsWith("x-" + headerTag + "-")) {
toSign.append(String.format("%s:%s\n", key.toLowerCase(), header.getValue()));
}
}
}
void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
// note that we fall back to headers, and some requests such as ?uploads do not have a
// payload, yet specify payload related parameters
buffer.append(
request.getPayload() == null ? Strings.nullToEmpty(request.getFirstHeaderOrNull("Content-MD5")) :
HttpUtils.nullToEmpty(
request.getPayload() == null ? null : request.getPayload().getContentMetadata()
.getContentMD5())).append("\n");
buffer.append(
Strings.nullToEmpty(
request.getPayload() == null ? request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE)
: request.getPayload().getContentMetadata().getContentType())).append("\n");
for (String header : FIRST_HEADERS_TO_SIGN)
buffer.append(HttpUtils.nullToEmpty(request.getHeaders().get(header))).append("\n");
}
@VisibleForTesting
void appendHttpHeaders(HttpRequest request, SortedSetMultimap<String, String> canonicalizedHeaders) {
Multimap<String, String> headers = request.getHeaders();
for (Map.Entry<String, String> header : headers.entries()) {
if (header.getKey() == null) {
continue;
}
String key = header.getKey().toString().toLowerCase(Locale.getDefault());
// Ignore any headers that are not particularly interesting.
if (key.equalsIgnoreCase(HttpHeaders.CONTENT_TYPE) || key.equalsIgnoreCase("Content-MD5")
|| key.equalsIgnoreCase(HttpHeaders.DATE) || key.startsWith("x-" + headerTag + "-")) {
canonicalizedHeaders.put(key, header.getValue());
}
}
}
@VisibleForTesting
void appendBucketName(HttpRequest req, StringBuilder toSign) {
String bucketName = S3Utils.getBucketName(req);
// If we have a payload/bucket/container that is not all lowercase, vhost-style URLs are not an option and must be
// automatically converted to their path-based equivalent. This should only be possible for AWS-S3 since it is
// the only S3 implementation configured to allow uppercase payload/bucket/container names.
//
// http://code.google.com/p/jclouds/issues/detail?id=992
if (isVhostStyle && bucketName != null && bucketName.equals(bucketName.toLowerCase())) {
toSign.append(servicePath).append(bucketName);
}
}
@VisibleForTesting
void appendUriPath(HttpRequest request, StringBuilder toSign) {
toSign.append(request.getEndpoint().getRawPath());
// ...however, there are a few exceptions that must be included in the
// signed URI.
if (request.getEndpoint().getQuery() != null) {
Multimap<String, String> params = queryParser().apply(request.getEndpoint().getQuery());
char separator = '?';
for (String paramName : Ordering.natural().sortedCopy(params.keySet())) {
// Skip any parameters that aren't part of the canonical signed string
if (!SIGNED_PARAMETERS.contains(paramName)) {
continue;
}
toSign.append(separator).append(paramName);
String paramValue = get(params.get(paramName), 0);
if (paramValue != null) {
toSign.append("=").append(paramValue);
}
separator = '&';
}
}
}
}

114
apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import com.google.common.reflect.TypeToken;
import com.google.inject.Singleton;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpRequest;
import org.jclouds.io.Payload;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.s3.S3Client;
import javax.inject.Inject;
@Singleton
public class RequestAuthorizeSignatureV4 implements RequestAuthorizeSignature {
private static final String PUT_OBJECT_METHOD = "putObject";
private static final TypeToken<S3Client> S3_CLIENT_TYPE = new TypeToken<S3Client>() {
};
private final Aws4SignerForAuthorizationHeader signerForAuthorizationHeader;
private final Aws4SignerForChunkedUpload signerForChunkedUpload;
private final Aws4SignerForQueryString signerForQueryString;
@Inject
public RequestAuthorizeSignatureV4(Aws4SignerForAuthorizationHeader signerForAuthorizationHeader,
Aws4SignerForChunkedUpload signerForChunkedUpload,
Aws4SignerForQueryString signerForQueryString) {
this.signerForAuthorizationHeader = signerForAuthorizationHeader;
this.signerForChunkedUpload = signerForChunkedUpload;
this.signerForQueryString = signerForQueryString;
}
@Override
public HttpRequest filter(HttpRequest request) throws HttpException {
// request use chunked upload
if (useChunkedUpload(request)) {
return signForChunkedUpload(request);
}
return signForAuthorizationHeader(request);
}
/**
* returns true, if use AWS S3 chunked upload.
*/
protected boolean useChunkedUpload(HttpRequest request) {
// only S3Client putObject method, payload not null, content-length > 0 and cannot repeatable
if (!GeneratedHttpRequest.class.isAssignableFrom(request.getClass())) {
return false;
}
GeneratedHttpRequest req = GeneratedHttpRequest.class.cast(request);
// s3 client type and method name is putObject
if (S3_CLIENT_TYPE.equals(req.getInvocation().getInvokable().getOwnerType()) &&
!PUT_OBJECT_METHOD.equals(req.getInvocation().getInvokable().getName())) {
return false;
}
Payload payload = req.getPayload();
// check payload null or payload.contentMetadata null
if (payload == null || payload.getContentMetadata() == null) {
return false;
}
Long contentLength = payload.getContentMetadata().getContentLength();
if (contentLength == null) {
return false;
}
return contentLength > 0l && !payload.isRepeatable();
}
protected HttpRequest signForAuthorizationHeader(HttpRequest request) {
return signerForAuthorizationHeader.sign(request);
}
protected HttpRequest signForChunkedUpload(HttpRequest request) {
return signerForChunkedUpload.sign(request);
}
// Authenticating Requests by Using Query Parameters (AWS Signature Version 4)
/**
* Using query parameters to authenticate requests is useful when you want to express a request entirely in a URL.
* This method is also referred as presigning a URL. Presigned URLs enable you to grant temporary access to your
* Amazon S3 resources. The end user can then enter the presigned URL in his or her browser to access the specific
* Amazon S3 resource. You can also use presigned URLs to embed clickable links in HTML.
* <p/>
* For example, you might store videos in an Amazon S3 bucket and make them available on your website by using presigned URLs.
* Identifies the version of AWS Signature and the algorithm that you used to calculate the signature.
*/
public HttpRequest signForTemporaryAccess(HttpRequest request, long timeInSeconds) {
return signerForQueryString.sign(request, timeInSeconds);
}
}

1
apis/s3/src/main/java/org/jclouds/s3/reference/S3Constants.java
View File

@ -31,6 +31,7 @@ public final class S3Constants {
public static final String DELIMITER = "delimiter";
public static final String PROPERTY_S3_SERVICE_PATH = "jclouds.s3.service-path";
public static final String PROPERTY_S3_VIRTUAL_HOST_BUCKETS = "jclouds.s3.virtual-host-buckets";
public static final String PROPERTY_JCLOUDS_S3_CHUNKED_SIZE = "jclouds.s3.chunked.size";
private S3Constants() {
throw new AssertionError("intentionally unimplemented");

61
apis/s3/src/test/java/org/jclouds/s3/filters/AwsHostNameUtilsTest.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.net.URI;
/**
* Tests parser region and service
*/
public class AwsHostNameUtilsTest {
@Test
public void testParseRegion() {
Assert.assertEquals(
AwsHostNameUtils.parseRegionName("test.s3.cn-north-1.amazonaws.com.cn", "s3"),
"cn-north-1"
);
}
@Test
// default region
public void testParseDefaultRegion() {
Assert.assertEquals(
AwsHostNameUtils.parseRegionName("s3.amazonaws.com", "s3"),
"us-east-1"
);
}
@Test
// test s3 service
public void testParseService() {
Assert.assertEquals(
AwsHostNameUtils.parseServiceName(URI.create("https://s3.amazonaws.com")),
"s3"
);
Assert.assertEquals(
AwsHostNameUtils.parseServiceName(URI.create("https://test-bucket.s3.cn-north-1.amazonaws.com.cn")),
"s3"
);
}
}

43
apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java → apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2Test.java
View File

@ -40,17 +40,18 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.SortedSetMultimap;
import com.google.common.collect.TreeMultimap;
import com.google.common.net.HttpHeaders;
/**
* Tests behavior of {@code RequestAuthorizeSignature}
* Tests behavior of {@code RequestAuthorizeSignatureV2}
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "RequestAuthorizeSignatureTest")
public class RequestAuthorizeSignatureTest extends BaseS3ClientTest<S3Client> {
@Test(groups = "unit", testName = "RequestAuthorizeSignatureV2Test")
public class RequestAuthorizeSignatureV2Test extends BaseS3ClientTest<S3Client> {
String bucketName = "bucket";
@DataProvider(parallel = true)
public Object[][] dataProvider() throws NoSuchMethodException {
return new Object[][] { { listOwnedBuckets() }, { putObject() }, { putBucketAcl() }
return new Object[][]{{listOwnedBuckets()}, {putObject()}, {putBucketAcl()}
};
}
@ -71,23 +72,23 @@ public class RequestAuthorizeSignatureTest extends BaseS3ClientTest<S3Client> {
request = filter.filter(request);
if (request.getFirstHeaderOrNull(HttpHeaders.DATE).equals(date))
assert signature.equals(request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION)) : String.format(
"sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION),
iterations);
"sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION),
iterations);
else
iterations++;
}
System.out.printf("%s: %d iterations before the timestamp updated %n", Thread.currentThread().getName(),
iterations);
iterations);
}
@Test
void testAppendBucketNameHostHeader() throws SecurityException, NoSuchMethodException {
GeneratedHttpRequest request = processor.createRequest(
method(S3Client.class, "getBucketLocation", String.class),
ImmutableList.<Object> of("bucket"));
method(S3Client.class, "getBucketLocation", String.class),
ImmutableList.<Object>of("bucket"));
StringBuilder builder = new StringBuilder();
filter.appendBucketName(request, builder);
((RequestAuthorizeSignatureV2) filter).appendBucketName(request, builder);
assertEquals(builder.toString(), "");
}
@ -95,15 +96,15 @@ public class RequestAuthorizeSignatureTest extends BaseS3ClientTest<S3Client> {
void testAclQueryString() throws SecurityException, NoSuchMethodException {
HttpRequest request = putBucketAcl();
StringBuilder builder = new StringBuilder();
filter.appendUriPath(request, builder);
((RequestAuthorizeSignatureV2) filter).appendUriPath(request, builder);
assertEquals(builder.toString(), "/" + bucketName + "?acl");
}
private GeneratedHttpRequest putBucketAcl() throws NoSuchMethodException {
return processor.createRequest(
method(S3Client.class, "putBucketACL", String.class, AccessControlList.class),
ImmutableList.<Object> of("bucket",
AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, "1234")));
method(S3Client.class, "putBucketACL", String.class, AccessControlList.class),
ImmutableList.<Object>of("bucket",
AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, "1234")));
}
// "?acl", "?location", "?logging", "?uploads", or "?torrent"
@ -112,22 +113,22 @@ public class RequestAuthorizeSignatureTest extends BaseS3ClientTest<S3Client> {
void testAppendBucketNameHostHeaderService() throws SecurityException, NoSuchMethodException {
HttpRequest request = listOwnedBuckets();
StringBuilder builder = new StringBuilder();
filter.appendBucketName(request, builder);
((RequestAuthorizeSignatureV2) filter).appendBucketName(request, builder);
assertEquals(builder.toString(), "");
}
private GeneratedHttpRequest listOwnedBuckets() throws NoSuchMethodException {
return processor.createRequest(method(S3Client.class, "listOwnedBuckets"),
ImmutableList.of());
ImmutableList.of());
}
@Test
void testHeadersGoLowercase() throws SecurityException, NoSuchMethodException {
HttpRequest request = putObject();
SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
filter.appendHttpHeaders(request, canonicalizedHeaders);
((RequestAuthorizeSignatureV2) filter).appendHttpHeaders(request, canonicalizedHeaders);
StringBuilder builder = new StringBuilder();
filter.appendAmzHeaders(canonicalizedHeaders, builder);
((RequestAuthorizeSignatureV2) filter).appendAmzHeaders(canonicalizedHeaders, builder);
assertEquals(builder.toString(), S3Headers.USER_METADATA_PREFIX + "adrian:foo\n");
}
@ -135,14 +136,14 @@ public class RequestAuthorizeSignatureTest extends BaseS3ClientTest<S3Client> {
S3Object object = blobToS3Object.apply(BindBlobToMultipartFormTest.TEST_BLOB);
object.getMetadata().getUserMetadata().put("Adrian", "foo");
return processor.createRequest(method(S3Client.class, "putObject", String.class,
S3Object.class, PutObjectOptions[].class), ImmutableList.<Object> of("bucket", object));
S3Object.class, PutObjectOptions[].class), ImmutableList.<Object>of("bucket", object));
}
@Test
void testAppendBucketNameInURIPath() throws SecurityException, NoSuchMethodException {
GeneratedHttpRequest request = processor.createRequest(
method(S3Client.class, "getBucketLocation", String.class),
ImmutableList.<Object> of(bucketName));
method(S3Client.class, "getBucketLocation", String.class),
ImmutableList.<Object>of(bucketName));
URI uri = request.getEndpoint();
assertEquals(uri.getHost(), "localhost");
assertEquals(uri.getPath(), "/" + bucketName);

199
apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4ChunkedUploadTest.java Normal file
View File

@ -0,0 +1,199 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import com.google.common.net.HttpHeaders;
import com.google.inject.Injector;
import com.google.inject.Module;
import org.jclouds.Constants;
import org.jclouds.ContextBuilder;
import org.jclouds.date.DateService;
import org.jclouds.date.TimeStamp;
import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpRequest;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.reflect.Invocation;
import org.jclouds.rest.ConfiguresHttpApi;
import org.jclouds.rest.internal.BaseRestApiTest;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.s3.S3ApiMetadata;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.config.S3HttpApiModule;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.s3.options.PutObjectOptions;
import org.jclouds.util.Closeables2;
import org.testng.annotations.Test;
import javax.inject.Named;
import javax.xml.ws.http.HTTPException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Date;
import static com.google.common.io.BaseEncoding.base16;
import static org.jclouds.reflect.Reflection2.method;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;
/**
* Tests behavior of {@code RequestAuthorizeSignature}
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "RequestAuthorizeSignatureV4ChunkedUploadTest")
public class RequestAuthorizeSignatureV4ChunkedUploadTest {
private static final String CONTENT_SEED =
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc tortor metus, sagittis eget augue ut,\n"
+ "feugiat vehicula risus. Integer tortor mauris, vehicula nec mollis et, consectetur eget tortor. In ut\n"
+ "elit sagittis, ultrices est ut, iaculis turpis. In hac habitasse platea dictumst. Donec laoreet tellus\n"
+ "at auctor tempus. Praesent nec diam sed urna sollicitudin vehicula eget id est. Vivamus sed laoreet\n"
+ "lectus. Aliquam convallis condimentum risus, vitae porta justo venenatis vitae. Phasellus vitae nunc\n"
+ "varius, volutpat quam nec, mollis urna. Donec tempus, nisi vitae gravida facilisis, sapien sem malesuada\n"
+ "purus, id semper libero ipsum condimentum nulla. Suspendisse vel mi leo. Morbi pellentesque placerat congue.\n"
+ "Nunc sollicitudin nunc diam, nec hendrerit dui commodo sed. Duis dapibus commodo elit, id commodo erat\n"
+ "congue id. Aliquam erat volutpat.\n";
private static final String CHUKED_UPLOAD_PAYLOAD_SHA256 = "2b6da230b03189254b2ceafe689c5298cfdd288869e80b2b9369da8f8f0a3d99";
private static final String PUT_OBJECT_AUTHORIZATION = "AWS4-HMAC-SHA256 "
+ "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ "SignedHeaders=content-encoding;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class, "
+ "Signature=3db48b3d786d599e8e785ba66030e8a9249c678a52f2432bf6fd44c97cb3145f";
private static final String IDENTITY = "AKIAPAEBI3QI4EXAMPLE";
private static final String CREDENTIAL = "oHkkcPcOjJnoAXpjT8GXdNeBjo6Ru7QeFExAmPlE";
private static final String TIMESTAMP = "Thu, 03 Feb 2015 07:11:11 GMT";
private static final String BUCKET_NAME = "test-bucket";
private static final String OBJECT_NAME = "ExampleChunkedObject.txt";
@ConfiguresHttpApi
private static final class TestS3HttpApiModule extends S3HttpApiModule<S3Client> {
@Override
protected String provideTimeStamp(@TimeStamp Supplier<String> cache) {
return TIMESTAMP;
}
@Override
protected Supplier<Date> provideTimeStampCacheDate(
@Named(Constants.PROPERTY_SESSION_INTERVAL) long seconds,
@TimeStamp final Supplier<String> timestamp,
final DateService dateService) {
return Suppliers.ofInstance(dateService.rfc822DateParse(TIMESTAMP));
}
}
public static Injector injector(Credentials creds) {
return ContextBuilder.newBuilder(new S3ApiMetadata())
.credentialsSupplier(Suppliers.<Credentials>ofInstance(creds))
.modules(ImmutableList.<Module>of(new BaseRestApiTest.MockModule(), new NullLoggingModule(),
new TestS3HttpApiModule()))
.buildInjector();
}
public static RequestAuthorizeSignatureV4 filter(Credentials creds) {
return injector(creds).getInstance(RequestAuthorizeSignatureV4.class);
}
Credentials temporaryCredentials = new Credentials.Builder()
.identity(IDENTITY)
.credential(CREDENTIAL)
.build();
@Test
void testPutObjectWithChunkedUpload() {
Invocation invocation = Invocation.create(
method(S3Client.class, "putObject", String.class, S3Object.class, PutObjectOptions[].class),
ImmutableList.<Object>of(BUCKET_NAME));
byte[] content = make65KPayload().getBytes(Charset.forName("UTF-8"));
HttpRequest putObject = GeneratedHttpRequest.builder().invocation(invocation)
.method("PUT")
.endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/" + OBJECT_NAME)
.addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
.addHeader("x-amz-storage-class", "REDUCED_REDUNDANCY")
.build();
Payload payload = Payloads.newInputStreamPayload(new ByteArrayInputStream(content));
payload.getContentMetadata().setContentLength((long) content.length);
payload.getContentMetadata().setContentType("text/plain");
putObject.setPayload(payload);
HttpRequest filtered = filter(temporaryCredentials).filter(putObject);
assertEquals(filtered.getFirstHeaderOrNull("Authorization"), PUT_OBJECT_AUTHORIZATION);
assertEquals(filtered.getPayload().getClass(), ChunkedUploadPayload.class);
InputStream is = null;
try {
is = filtered.getPayload().openStream();
assertEquals(base16().lowerCase().encode(hash(is)), CHUKED_UPLOAD_PAYLOAD_SHA256);
} catch (IOException e) {
fail("open stream error", e);
} finally {
Closeables2.closeQuietly(is);
}
}
/**
* Want sample to upload 3 chunks for our selected chunk size of 64K; one
* full size chunk, one partial chunk and then the 0-byte terminator chunk.
* This routine just takes 1K of seed text and turns it into a 65K-or-so
* string for sample use.
*/
private static String make65KPayload() {
StringBuilder oneKSeed = new StringBuilder();
while (oneKSeed.length() < 1024) {
oneKSeed.append(CONTENT_SEED);
}
// now scale up to meet/exceed our requirement
StringBuilder output = new StringBuilder();
for (int i = 0; i < 66; i++) {
output.append(oneKSeed);
}
return output.toString();
}
/**
* hash input with sha256
*
* @param input
* @return hash result
* @throws HTTPException
*/
private static byte[] hash(InputStream input) {
try {
Hasher hasher = Hashing.sha256().newHasher();
byte[] buffer = new byte[4096];
int r;
while ((r = input.read(buffer)) != -1) {
hasher.putBytes(buffer, 0, r);
}
return hasher.hash().asBytes();
} catch (Exception e) {
throw new RuntimeException("Unable to compute hash while signing request: " + e.getMessage(), e);
}
}
}

193
apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4Test.java Normal file
View File

@ -0,0 +1,193 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.filters;
import static org.jclouds.reflect.Reflection2.method;
import static org.testng.Assert.assertEquals;
import java.util.Date;
import javax.inject.Named;
import org.jclouds.Constants;
import org.jclouds.ContextBuilder;
import org.jclouds.date.DateService;
import org.jclouds.date.TimeStamp;
import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.options.GetOptions;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.reflect.Invocation;
import org.jclouds.rest.ConfiguresHttpApi;
import org.jclouds.rest.internal.BaseRestApiTest;
import org.jclouds.rest.internal.GeneratedHttpRequest;
import org.jclouds.s3.S3ApiMetadata;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.config.S3HttpApiModule;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.s3.options.PutObjectOptions;
import org.testng.annotations.Test;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.net.HttpHeaders;
import com.google.inject.Injector;
import com.google.inject.Module;
/**
* Tests behavior of {@code RequestAuthorizeSignature}
*/
// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
@Test(groups = "unit", testName = "RequestAuthorizeSignatureV4Test")
public class RequestAuthorizeSignatureV4Test {
private static final String IDENTITY = "AKIAPAEBI3QI4EXAMPLE";
private static final String CREDENTIAL = "oHkkcPcOjJnoAXpjT8GXdNeBjo6Ru7QeFExAmPlE";
private static final String TIMESTAMP = "Thu, 03 Feb 2015 07:11:11 GMT";
private static final String GET_BUCKET_LOCATION_SIGNATURE_RESULT = "AWS4-HMAC-SHA256 "
+ "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
+ "Signature=5634847b3ad6a857887ab0ccff2fcaf3d35ef3dc549a3c27ebc0f584a80494c3";
private static final String GET_OBJECT_RESULT = "AWS4-HMAC-SHA256 "
+ "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
+ "Signature=fbd1d0f04a72907fb20ecd771644afd62cb689f91d26e9471b7a234531ec4718";
private static final String GET_OBJECT_ACL_RESULT = "AWS4-HMAC-SHA256 "
+ "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
+ "Signature=52d7f31d249032b59781fe69c8124ff4bf209be3f374b28657a60d906c752381";
private static final String PUT_OBJECT_CONTENT = "text sign";
private static final String PUT_OBJECT_RESULT = "AWS4-HMAC-SHA256 "
+ "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ "SignedHeaders=content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, "
+ "Signature=090f1bb1db984221ae1a20c5d12a82820a0d74b4be85f20daa1431604f41df08";
private static final String BUCKET_NAME = "test-bucket";
private static final String OBJECT_NAME = "ExampleObject.txt";
@ConfiguresHttpApi
private static final class TestS3HttpApiModule extends S3HttpApiModule<S3Client> {
@Override
protected String provideTimeStamp(@TimeStamp Supplier<String> cache) {
return TIMESTAMP;
}
@Override
protected Supplier<Date> provideTimeStampCacheDate(
@Named(Constants.PROPERTY_SESSION_INTERVAL) long seconds,
@TimeStamp final Supplier<String> timestamp,
final DateService dateService) {
return Suppliers.ofInstance(dateService.rfc822DateParse(TIMESTAMP));
}
}
public static Injector injector(Credentials creds) {
return ContextBuilder.newBuilder(new S3ApiMetadata())
.credentialsSupplier(Suppliers.<Credentials>ofInstance(creds))
.modules(ImmutableList.<Module>of(new BaseRestApiTest.MockModule(), new NullLoggingModule(),
new TestS3HttpApiModule()))
.buildInjector();
}
public static RequestAuthorizeSignatureV4 filter(Credentials creds) {
return injector(creds).getInstance(RequestAuthorizeSignatureV4.class);
}
Credentials temporaryCredentials = new Credentials.Builder()
.identity(IDENTITY)
.credential(CREDENTIAL)
.build();
@Test
void testGetBucketLocationSignature() {
Invocation invocation = Invocation.create(method(S3Client.class, "getBucketLocation", String.class),
ImmutableList.<Object>of(BUCKET_NAME));
HttpRequest getBucketLocation = GeneratedHttpRequest.builder().method("GET")
.invocation(invocation)
.endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/")
.addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
.addQueryParam("location", "")
.build();
HttpRequest filtered = filter(temporaryCredentials).filter(getBucketLocation);
assertEquals(filtered.getFirstHeaderOrNull("Authorization"), GET_BUCKET_LOCATION_SIGNATURE_RESULT);
}
@Test
void testGetObjectSignature() {
Invocation invocation = Invocation.create(method(S3Client.class, "getObject", String.class,
String.class, GetOptions[].class),
ImmutableList.<Object>of(BUCKET_NAME, OBJECT_NAME, new GetOptions[0]));
HttpRequest getObject = GeneratedHttpRequest.builder().method("GET")
.invocation(invocation)
.endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/" + OBJECT_NAME)
.addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
.build();
HttpRequest filtered = filter(temporaryCredentials).filter(getObject);
assertEquals(filtered.getFirstHeaderOrNull("Authorization"), GET_OBJECT_RESULT);
}
@Test
void testGetObjectACLSignature() {
Invocation invocation = Invocation.create(method(S3Client.class, "getObjectACL", String.class, String.class),
ImmutableList.<Object>of(BUCKET_NAME));
HttpRequest getObjectACL = GeneratedHttpRequest.builder().method("GET")
.invocation(invocation)
.endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/" + OBJECT_NAME)
.addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
.addQueryParam("acl", "")
.build();
HttpRequest filtered = filter(temporaryCredentials).filter(getObjectACL);
assertEquals(filtered.getFirstHeaderOrNull("Authorization"), GET_OBJECT_ACL_RESULT);
}
@Test
void testPutObjectSignature() {
Invocation invocation = Invocation.create(method(S3Client.class, "putObject", String.class, S3Object.class,
PutObjectOptions[].class),
ImmutableList.<Object>of(BUCKET_NAME));
Payload payload = Payloads.newStringPayload(PUT_OBJECT_CONTENT);
payload.getContentMetadata().setContentType("text/plain");
HttpRequest putObject = GeneratedHttpRequest.builder().method("PUT")
.invocation(invocation)
.endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/" + OBJECT_NAME)
.addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
.addHeader("x-amz-storage-class", "REDUCED_REDUNDANCY")
.payload(payload)
.build();
HttpRequest filtered = filter(temporaryCredentials).filter(putObject);
assertEquals(filtered.getFirstHeaderOrNull("Authorization"), PUT_OBJECT_RESULT);
}
}

3
apis/s3/src/test/java/org/jclouds/s3/internal/BaseS3ClientTest.java
View File

@ -26,6 +26,7 @@ import org.jclouds.s3.S3ApiMetadata;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.filters.RequestAuthorizeSignatureV2;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@ -47,7 +48,7 @@ public abstract class BaseS3ClientTest<T extends S3Client> extends BaseRestAnnot
protected void setupFactory() throws IOException {
super.setupFactory();
blobToS3Object = injector.getInstance(BlobToObject.class);
filter = injector.getInstance(RequestAuthorizeSignature.class);
filter = injector.getInstance(RequestAuthorizeSignatureV2.class);
}
public BaseS3ClientTest() {

6
providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSigner.java
View File

@ -33,7 +33,7 @@ import org.jclouds.reflect.Invocation;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.blobstore.S3BlobRequestSigner;
import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.filters.RequestAuthorizeSignatureV2;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
@ -44,7 +44,7 @@ import com.google.inject.Provider;
public class AWSS3BlobRequestSigner extends S3BlobRequestSigner<AWSS3Client> {
public static final String TEMPORARY_SIGNATURE_PARAM = "Signature";
private final RequestAuthorizeSignature authSigner;
private final RequestAuthorizeSignatureV2 authSigner;
private final String identity;
private final DateService dateService;
private final Provider<String> timeStampProvider;
@ -53,7 +53,7 @@ public class AWSS3BlobRequestSigner extends S3BlobRequestSigner<AWSS3Client> {
public AWSS3BlobRequestSigner(RestAnnotationProcessor processor, BlobToObject blobToObject,
BlobToHttpGetOptions blob2HttpGetOptions, Class<AWSS3Client> interfaceClass,
@org.jclouds.location.Provider Supplier<Credentials> credentials,
RequestAuthorizeSignature authSigner, @TimeStamp Provider<String> timeStampProvider,
RequestAuthorizeSignatureV2 authSigner, @TimeStamp Provider<String> timeStampProvider,
DateService dateService) throws SecurityException, NoSuchMethodException {
super(processor, blobToObject, blob2HttpGetOptions, interfaceClass);
this.authSigner = authSigner;

66
providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSignerV4.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.blobstore;
import static org.jclouds.blobstore.util.BlobStoreUtils.cleanRequest;
import static com.google.common.base.Preconditions.checkNotNull;
import org.jclouds.aws.s3.AWSS3Client;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
import org.jclouds.http.HttpRequest;
import org.jclouds.reflect.Invocation;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.blobstore.S3BlobRequestSigner;
import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.filters.RequestAuthorizeSignatureV4;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
public class AWSS3BlobRequestSignerV4 extends S3BlobRequestSigner<AWSS3Client> {
private final RequestAuthorizeSignatureV4 authSigner;
@Inject
public AWSS3BlobRequestSignerV4(RestAnnotationProcessor processor, BlobToObject blobToObject,
BlobToHttpGetOptions blob2HttpGetOptions, Class<AWSS3Client> interfaceClass,
RequestAuthorizeSignatureV4 authSigner) throws SecurityException, NoSuchMethodException {
super(processor, blobToObject, blob2HttpGetOptions, interfaceClass);
this.authSigner = authSigner;
}
@Override
public HttpRequest signGetBlob(String container, String name, long timeInSeconds) {
checkNotNull(container, "container");
checkNotNull(name, "name");
HttpRequest request = processor.apply(Invocation.create(getMethod, ImmutableList.<Object>of(container, name)));
request = authSigner.signForTemporaryAccess(request, timeInSeconds);
return cleanRequest(request);
}
@Override
public HttpRequest signPutBlob(String container, Blob blob, long timeInSeconds) {
checkNotNull(container, "container");
checkNotNull(blob, "blob");
HttpRequest request = processor.apply(Invocation.create(createMethod,
ImmutableList.<Object>of(container, blobToObject.apply(blob))));
request = authSigner.signForTemporaryAccess(request, timeInSeconds);
return cleanRequest(request);
}
}

4
providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignature.java
View File

@ -34,13 +34,13 @@ import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.jclouds.s3.filters.RequestAuthorizeSignatureV2;
import com.google.common.base.Supplier;
/** Signs the AWS S3 request, supporting temporary signatures. */
@Singleton
public class AWSRequestAuthorizeSignature extends RequestAuthorizeSignature {
public class AWSRequestAuthorizeSignature extends RequestAuthorizeSignatureV2 {
@Inject
public AWSRequestAuthorizeSignature(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,

58
providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignatureV4.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.filters;
import static org.jclouds.http.utils.Queries.queryParser;
import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SIGNATURE_PARAM;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.http.HttpRequest;
import org.jclouds.s3.filters.Aws4SignerForAuthorizationHeader;
import org.jclouds.s3.filters.Aws4SignerForChunkedUpload;
import org.jclouds.s3.filters.Aws4SignerForQueryString;
import org.jclouds.s3.filters.RequestAuthorizeSignatureV4;
/**
* Signs the AWS S3 request, supporting temporary signatures.
*/
@Singleton
public class AWSRequestAuthorizeSignatureV4 extends RequestAuthorizeSignatureV4 {
@Inject
public AWSRequestAuthorizeSignatureV4(Aws4SignerForAuthorizationHeader signerForAuthorizationHeader,
Aws4SignerForChunkedUpload signerForChunkedUpload,
Aws4SignerForQueryString signerForQueryString) {
super(signerForAuthorizationHeader, signerForChunkedUpload, signerForQueryString);
}
@Override
protected HttpRequest signForAuthorizationHeader(HttpRequest request) {
/*
* Only add the Authorization header if the query string doesn't already contain
* the 'X-Amz-Signature' parameter, otherwise S3 will fail the request complaining about
* duplicate authentication methods. The 'Signature' parameter will be added for signed URLs
* with expiration.
*/
if (queryParser().apply(request.getEndpoint().getQuery()).containsKey(AMZ_SIGNATURE_PARAM)) {
return request;
}
return super.signForAuthorizationHeader(request);
}
}

201
providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerV4ExpectTest.java Normal file
View File

@ -0,0 +1,201 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aws.s3.blobstore;
import static org.jclouds.Constants.PROPERTY_CREDENTIAL;
import static org.jclouds.Constants.PROPERTY_IDENTITY;
import static org.testng.Assert.assertEquals;
import java.util.Date;
import java.util.Properties;
import javax.inject.Named;
import org.jclouds.Constants;
import org.jclouds.aws.s3.AWSS3ApiMetadata;
import org.jclouds.aws.s3.AWSS3ProviderMetadata;
import org.jclouds.aws.s3.blobstore.config.AWSS3BlobStoreContextModule;
import org.jclouds.aws.s3.config.AWSS3HttpApiModule;
import org.jclouds.aws.s3.filters.AWSRequestAuthorizeSignatureV4;
import org.jclouds.blobstore.BlobRequestSigner;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.date.DateService;
import org.jclouds.date.TimeStamp;
import org.jclouds.http.HttpRequest;
import org.jclouds.providers.ProviderMetadata;
import org.jclouds.rest.ConfiguresHttpApi;
import org.jclouds.s3.blobstore.S3BlobSignerExpectTest;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
import org.testng.SkipException;
import org.testng.annotations.Test;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.HttpHeaders;
import com.google.inject.Module;
import com.google.inject.Scopes;
@Test(groups = "unit", testName = "AWSS3BlobSignerV4ExpectTest")
public class AWSS3BlobSignerV4ExpectTest extends S3BlobSignerExpectTest {
private static final String IDENTITY = "AKIAPAEBI3QI4EXAMPLE";
private static final String CREDENTIAL = "oHkkcPcOjJnoAXpjT8GXdNeBjo6Ru7QeFExAmPlE";
private static final String TIMESTAMP = "Thu, 03 Feb 2015 07:11:11 GMT";
private static final String BUCKET_NAME = "test-bucket";
private static final String OBJECT_NAME = "ExampleObject.txt";
private static final String HOST = BUCKET_NAME + ".s3.amazonaws.com";
public AWSS3BlobSignerV4ExpectTest() {
provider = null;
}
@Override
protected HttpRequest getBlobWithTime() {
return HttpRequest.builder().method("GET")
.endpoint("https://" + HOST + "/" + OBJECT_NAME
+ "?X-Amz-Algorithm=AWS4-HMAC-SHA256"
+ "&X-Amz-Credential=AKIAPAEBI3QI4EXAMPLE/20150203/us-east-1/s3/aws4_request"
+ "&X-Amz-Date=20150203T071111Z"
+ "&X-Amz-Expires=86400"
+ "&X-Amz-SignedHeaders=host"
+ "&X-Amz-Signature=0bafb6a0d99c8b7c39abe5496e9897e8c442b09278f1a647267acb25e8d1c550")
.addHeader(HttpHeaders.HOST, HOST)
.build();
}
@Test
@Override
public void testSignGetBlobWithTime() {
BlobStore getBlobWithTime = requestsSendResponses(init());
HttpRequest compare = getBlobWithTime();
HttpRequest signedRequest = getBlobWithTime.getContext().getSigner().signGetBlob(BUCKET_NAME, OBJECT_NAME,
86400l /* seconds */);
assertEquals(signedRequest, compare);
}
protected HttpRequest _putBlobWithTime() {
return HttpRequest.builder().method("PUT")
.endpoint("https://" + HOST + "/" + OBJECT_NAME
+ "?X-Amz-Algorithm=AWS4-HMAC-SHA256"
+ "&X-Amz-Credential=AKIAPAEBI3QI4EXAMPLE/20150203/us-east-1/s3/aws4_request"
+ "&X-Amz-Date=20150203T071111Z"
+ "&X-Amz-Expires=86400"
+ "&X-Amz-SignedHeaders=host"
+ "&X-Amz-Signature=41484fb83e0c51b289907979ff96b2c743f6faf8dc70fca1c6fa78d8aeda132f")
.addHeader(HttpHeaders.EXPECT, "100-continue")
.addHeader(HttpHeaders.HOST, HOST)
.build();
}
@Test
@Override
public void testSignPutBlobWithTime() throws Exception {
BlobStore signPutBloblWithTime = requestsSendResponses(init());
Blob blob = signPutBloblWithTime.blobBuilder(OBJECT_NAME).payload(text).contentType("text/plain").build();
HttpRequest compare = _putBlobWithTime();
compare.setPayload(blob.getPayload());
HttpRequest signedRequest = signPutBloblWithTime.getContext().getSigner().signPutBlob(BUCKET_NAME, blob,
86400l /* seconds */);
assertEquals(signedRequest, compare);
}
@Override
protected HttpRequest putBlob() {
throw new SkipException("skip putBlob");
}
@Override
public void testSignPutBlob() {
throw new SkipException("skip testSignPutBlob");
}
@Override
public void testSignGetBlob() {
throw new SkipException("skip testSignGetBlob");
}
@Override
public void testSignGetBlobWithOptions() {
throw new SkipException("skip testSignGetBlobWithOptions");
}
@Override
public void testSignRemoveBlob() {
throw new SkipException("skip testSignRemoveBlob");
}
@Override
protected Module createModule() {
return new TestAWSS3SignerV4HttpApiModule();
}
@Override
protected Properties setupProperties() {
Properties props = super.setupProperties();
props.put(PROPERTY_IDENTITY, IDENTITY);
props.put(PROPERTY_CREDENTIAL, CREDENTIAL);
return props;
}
@Override
protected ProviderMetadata createProviderMetadata() {
AWSS3ApiMetadata.Builder apiBuilder = new AWSS3ApiMetadata().toBuilder();
apiBuilder.defaultModules(ImmutableSet.<Class<? extends Module>>of(TestAWSS3SignerV4HttpApiModule.class,
TestAWSS3BlobStoreContextModule.class));
return new AWSS3ProviderMetadata().toBuilder().apiMetadata(apiBuilder.build()).build();
}
public static final class TestAWSS3BlobStoreContextModule extends AWSS3BlobStoreContextModule {
@Override
protected void bindRequestSigner() {
// replace AWSS3BlobRequestSigner aws s3 with AWSS3BlobRequestSignerV4
bind(BlobRequestSigner.class).to(AWSS3BlobRequestSignerV4.class);
}
}
@ConfiguresHttpApi
public static final class TestAWSS3SignerV4HttpApiModule extends AWSS3HttpApiModule {
@Override
protected void configure() {
super.configure();
}
@Override
protected void bindRequestSigner() {
bind(RequestAuthorizeSignature.class).to(AWSRequestAuthorizeSignatureV4.class).in(Scopes.SINGLETON);
}
@Override
@TimeStamp
protected String provideTimeStamp(@TimeStamp Supplier<String> cache) {
return TIMESTAMP;
}
@Override
@TimeStamp
protected Supplier<Date> provideTimeStampCacheDate(
@Named(Constants.PROPERTY_SESSION_INTERVAL) long seconds,
@TimeStamp final Supplier<String> timestamp,
final DateService dateService) {
return Suppliers.ofInstance(dateService.rfc822DateParse(TIMESTAMP));
}
}
}