Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

issue 430: large blob support: 

- modified all 3 http connectors to add "Expect: 100-continue" header
- refactored the RequestAuthorizeSignature to not conform the specification
- complete-multipart-upload response is returning escaped quote, I extended ETag parser
- added more S3 headers
This commit is contained in:
tibor.kiss 2011-02-26 19:21:51 +01:00
parent bc5b4e8ab3
commit 00d172ce2f
10 changed files with 207 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java
View File

@ -20,19 +20,20 @@
package org.jclouds.s3.filters;
import static com.google.common.base.Preconditions.checkArgument;
import static org.jclouds.Constants.PROPERTY_CREDENTIAL;
import static org.jclouds.Constants.PROPERTY_IDENTITY;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
import static org.jclouds.util.Patterns.NEWLINE_PATTERN;
import java.lang.annotation.Annotation;
import java.util.Arrays;
import java.util.Collection;
import java.util.Locale;
import java.util.Set;
import java.util.TreeSet;
import java.util.Map.Entry;
import javax.annotation.Resource;
import javax.inject.Inject;
@ -43,6 +44,7 @@ import javax.ws.rs.core.HttpHeaders;
import org.jclouds.Constants;
import org.jclouds.s3.Bucket;
import org.jclouds.s3.reference.S3Headers;
import org.jclouds.crypto.Crypto;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.date.TimeStamp;
@ -64,12 +66,15 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.SortedSetMultimap;
import com.google.common.collect.TreeMultimap;
/**
* Signs the S3 request.
*
* @see <a href= "http://docs.amazonwebservices.com/AmazonS3/latest/RESTAuthentication.html" />
* @see <a href= "http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/index.html?RESTAuthentication.html" />
* @author Adrian Cole
*
*/
@ -77,8 +82,14 @@ import com.google.common.collect.Multimaps;
public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSigner {
private final String[] firstHeadersToSign = new String[] { HttpHeaders.DATE };
public static Set<String> SPECIAL_QUERIES = ImmutableSet.of("acl", "torrent", "logging", "location",
"requestPayment", "uploads");
/** Prefix for general Amazon headers: x-amz- */
public static final String AMAZON_PREFIX = "x-amz-";
public static Set<String> SIGNED_PARAMETERS = ImmutableSet.of("acl", "torrent", "logging", "location", "policy", "requestPayment", "versioning",
"versions", "versionId", "notification", "uploadId", "uploads", "partNumber", "website",
"response-content-type", "response-content-language", "response-expires",
"response-cache-control", "response-content-disposition", "response-content-encoding");
private final SignatureWire signatureWire;
private final String accessKey;
private final String secretKey;
@ -137,12 +148,19 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
public String createStringToSign(HttpRequest request) {
utils.logRequest(signatureLog, request, ">>");
SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
StringBuilder buffer = new StringBuilder();
// re-sign the request
appendMethod(request, buffer);
appendPayloadMetadata(request, buffer);
appendHttpHeaders(request, buffer);
appendAmzHeaders(request, buffer);
appendHttpHeaders(request, canonicalizedHeaders);
// Remove default date timestamp if "x-amz-date" is set.
if (canonicalizedHeaders.containsKey(S3Headers.ALTERNATE_DATE)) {
canonicalizedHeaders.put("date", "");
}
appendAmzHeaders(canonicalizedHeaders, buffer);
if (isVhostStyle)
appendBucketName(request, buffer);
appendUriPath(request, buffer);
@ -173,32 +191,43 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
toSign.append(request.getMethod()).append("\n");
}
void appendAmzHeaders(HttpRequest request, StringBuilder toSign) {
Set<String> headers = new TreeSet<String>(request.getHeaders().keySet());
for (String header : headers) {
if (header.startsWith("x-" + headerTag + "-")) {
toSign.append(header.toLowerCase()).append(":");
for (String value : request.getHeaders().get(header)) {
toSign.append(Strings2.replaceAll(value, NEWLINE_PATTERN, "")).append(",");
}
toSign.deleteCharAt(toSign.lastIndexOf(","));
toSign.append("\n");
@VisibleForTesting
void appendAmzHeaders(SortedSetMultimap<String, String> canonicalizedHeaders, StringBuilder toSign) {
for (Entry<String, String> header : canonicalizedHeaders.entries()) {
String key = header.getKey();
if (key.startsWith("x-" + headerTag + "-")) {
toSign.append(String.format("%s: %s\n", key.toLowerCase(), header.getValue()));
}
}
}
void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
// the following request parameters are positional in their nature
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
.getContentMD5())).append("\n");
buffer.append(
utils.valueOrEmpty(request.getPayload() == null ? request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE)
: request.getPayload().getContentMetadata().getContentType())).append("\n");
: request.getPayload().getContentMetadata().getContentType())).append("\n");
for (String header : firstHeadersToSign)
buffer.append(valueOrEmpty(request.getHeaders().get(header))).append("\n");
}
void appendHttpHeaders(HttpRequest request, StringBuilder toSign) {
for (String header : firstHeadersToSign)
toSign.append(valueOrEmpty(request.getHeaders().get(header))).append("\n");
@VisibleForTesting
void appendHttpHeaders(HttpRequest request,
SortedSetMultimap<String, String> canonicalizedHeaders) {
Multimap<String, String> headers = request.getHeaders();
for (Entry<String, String> header : headers.entries()) {
if (header.getKey() == null)
continue;
String key = header.getKey().toString()
.toLowerCase(Locale.getDefault());
// Ignore any headers that are not particularly interesting.
if (key.equalsIgnoreCase(HttpHeaders.CONTENT_TYPE) || key.equalsIgnoreCase("Content-MD5")
|| key.equalsIgnoreCase(HttpHeaders.DATE) || key.startsWith(AMAZON_PREFIX)) {
canonicalizedHeaders.put(key, header.getValue());
}
}
}
@VisibleForTesting
@ -232,19 +261,24 @@ public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSign
// ...however, there are a few exceptions that must be included in the
// signed URI.
if (request.getEndpoint().getQuery() != null) {
StringBuilder paramsToSign = new StringBuilder("?");
SortedSetMultimap<String, String> sortedParams = TreeMultimap.create();
String[] params = request.getEndpoint().getQuery().split("&");
for (String param : params) {
String[] paramNameAndValue = param.split("=");
if (SPECIAL_QUERIES.contains(paramNameAndValue[0])) {
paramsToSign.append(paramNameAndValue[0]);
}
sortedParams.put(paramNameAndValue[0], paramNameAndValue.length == 2 ? paramNameAndValue[1] : null);
}
char separator = '?';
for (Entry<String, String> param: sortedParams.entries()) {
String paramName = param.getKey();
// Skip any parameters that aren't part of the canonical signed string
if (SIGNED_PARAMETERS.contains(paramName) == false) continue;
if (paramsToSign.length() > 1) {
toSign.append(paramsToSign);
toSign.append(separator).append(paramName);
String paramValue = param.getValue();
if (paramValue != null) {
toSign.append("=").append(paramValue);
}
separator = '&';
}
}
}

71
apis/s3/src/main/java/org/jclouds/s3/reference/S3Headers.java
View File

@ -30,12 +30,83 @@ package org.jclouds.s3.reference;
*/
public interface S3Headers {
public static final String CONTENT_MD5 = "Content-MD5";
/** Prefix for general Amazon headers: x-amz- */
public static final String AMAZON_PREFIX = "x-amz-";
/**
* The canned ACL to apply to the object. Options include private, public-read,
* public-read-write, and authenticated-read. For more information, see REST Access Control
* Policy.
*/
public static final String CANNED_ACL = "x-amz-acl";
public static final String AMZ_MD5 = "x-amz-meta-object-eTag";
/** Amazon's alternative date header: x-amz-date */
public static final String ALTERNATE_DATE = "x-amz-date";
/** Prefix for user metadata: x-amz-meta- */
public static final String USER_METADATA_PREFIX = "x-amz-meta-";
/** version ID header */
public static final String VERSION_ID = "x-amz-version-id";
/** Multi-Factor Authentication header */
public static final String MFA = "x-amz-mfa";
/** response header for a request's AWS request ID */
public static final String REQUEST_ID = "x-amz-request-id";
/** response header for a request's extended debugging ID */
public static final String EXTENDED_REQUEST_ID = "x-amz-id-2";
/** request header indicating how to handle metadata when copying an object */
public static final String METADATA_DIRECTIVE = "x-amz-metadata-directive";
/** DevPay token header */
public static final String SECURITY_TOKEN = "x-amz-security-token";
/** Header describing what class of storage a user wants */
public static final String STORAGE_CLASS = "x-amz-storage-class";
/** ETag matching constraint header for the copy object request */
public static final String COPY_SOURCE_IF_MATCH = "x-amz-copy-source-if-match";
/** ETag non-matching constraint header for the copy object request */
public static final String COPY_SOURCE_IF_NO_MATCH = "x-amz-copy-source-if-none-match";
/** Unmodified since constraint header for the copy object request */
public static final String COPY_SOURCE_IF_UNMODIFIED_SINCE = "x-amz-copy-source-if-unmodified-since";
/** Modified since constraint header for the copy object request */
public static final String COPY_SOURCE_IF_MODIFIED_SINCE = "x-amz-copy-source-if-modified-since";
/** Range header for the get object request */
public static final String RANGE = "Range";
/** Modified since constraint header for the get object request */
public static final String GET_OBJECT_IF_MODIFIED_SINCE = "If-Modified-Since";
/** Unmodified since constraint header for the get object request */
public static final String GET_OBJECT_IF_UNMODIFIED_SINCE = "If-Unmodified-Since";
/** ETag matching constraint header for the get object request */
public static final String GET_OBJECT_IF_MATCH = "If-Match";
/** ETag non-matching constraint header for the get object request */
public static final String GET_OBJECT_IF_NONE_MATCH = "If-None-Match";
/** Encrypted symmetric key header that is used in the envelope encryption mechanism */
public static final String CRYPTO_KEY = "x-amz-key";
/** Initialization vector (IV) header that is used in the symmetric and envelope encryption mechanisms */
public static final String CRYPTO_IV = "x-amz-iv";
/** JSON-encoded description of encryption materials used during encryption */
public static final String MATERIALS_DESCRIPTION = "x-amz-matdesc";
/** Instruction file header to be placed in the metadata of instruction files */
public static final String CRYPTO_INSTRUCTION_FILE = "x-amz-crypto-instr-file";
}

8
apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java
View File

@ -39,6 +39,8 @@ import org.jclouds.s3.options.PutObjectOptions;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import com.google.common.collect.SortedSetMultimap;
import com.google.common.collect.TreeMultimap;
import com.google.inject.TypeLiteral;
/**
@ -129,9 +131,11 @@ public class RequestAuthorizeSignatureTest extends BaseS3AsyncClientTest<S3Async
@Test
void testHeadersGoLowercase() throws SecurityException, NoSuchMethodException {
HttpRequest request = putObject();
SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
filter.appendHttpHeaders(request, canonicalizedHeaders);
StringBuilder builder = new StringBuilder();
filter.appendAmzHeaders(request, builder);
assertEquals(builder.toString(), "x-amz-meta-x-amz-adrian:foo\n");
filter.appendAmzHeaders(canonicalizedHeaders, builder);
assertEquals(builder.toString(), "x-amz-meta-x-amz-adrian: foo\n");
}
private HttpRequest putObject() throws NoSuchMethodException {

3
core/src/main/java/org/jclouds/http/internal/JavaUrlHttpCommandExecutorService.java
View File

@ -236,6 +236,9 @@ public class JavaUrlHttpCommandExecutorService extends BaseHttpCommandExecutorSe
checkArgument(length < Integer.MAX_VALUE,
"JDK 1.6 does not support >2GB chunks. Use chunked encoding, if possible.");
connection.setFixedLengthStreamingMode(length.intValue());
if (length.intValue() > 0) {
connection.setRequestProperty("Expect", "100-continue");
}
}
CountingOutputStream out = new CountingOutputStream(connection.getOutputStream());
try {

2
drivers/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCUtils.java
View File

@ -43,6 +43,7 @@ import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.FileEntity;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.params.CoreProtocolPNames;
import org.jclouds.http.HttpRequest;
import org.jclouds.io.Payload;
import org.jclouds.io.payloads.BasePayload;
@ -72,6 +73,7 @@ public class ApacheHCUtils {
apacheRequest = new HttpDelete(request.getEndpoint());
} else if (request.getMethod().equals(HttpMethod.PUT)) {
apacheRequest = new HttpPut(request.getEndpoint());
apacheRequest.getParams().setBooleanParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, true);
} else if (request.getMethod().equals(HttpMethod.POST)) {
apacheRequest = new HttpPost(request.getEndpoint());
} else {

5
drivers/gae/src/main/java/org/jclouds/gae/ConvertToGaeRequest.java
View File

@ -92,6 +92,9 @@ public class ConvertToGaeRequest implements Function<HttpRequest, HTTPRequest> {
HttpUtils.copy(oldPayload.getContentMetadata(), request.getPayload().getContentMetadata());
}
gaeRequest.setPayload(array);
if (array.length > 0) {
gaeRequest.setHeader(new HTTPHeader("Expect", "100-continue"));
}
} catch (IOException e) {
Throwables.propagate(e);
} finally {
@ -101,7 +104,7 @@ public class ConvertToGaeRequest implements Function<HttpRequest, HTTPRequest> {
for (Entry<String, String> header : HttpUtils.getContentHeadersFromMetadata(
request.getPayload().getContentMetadata()).entries()) {
gaeRequest.setHeader(new HTTPHeader(header.getKey(), header.getValue()));
}
}
} else {
gaeRequest.setHeader(new HTTPHeader(HttpHeaders.CONTENT_LENGTH, "0"));
}

14
providers/aws-s3/src/main/java/org/jclouds/aws/s3/functions/ETagFromHttpResponseViaRegex.java
View File

@ -36,7 +36,8 @@ import com.google.common.base.Function;
*/
@Singleton
public class ETagFromHttpResponseViaRegex implements Function<HttpResponse, String> {
Pattern pattern = Pattern.compile("<ETag>([\\S&&[^<]]+)</ETag>");
private static Pattern pattern = Pattern.compile("<ETag>([\\S&&[^<]]+)</ETag>");
private static Pattern quotPattern = Pattern.compile("(&quot;)");
private final ReturnStringIf2xx returnStringIf200;
@Inject
@ -52,6 +53,17 @@ public class ETagFromHttpResponseViaRegex implements Function<HttpResponse, Stri
Matcher matcher = pattern.matcher(content);
if (matcher.find()) {
value = matcher.group(1);
Matcher quotMatcher = quotPattern.matcher(value);
StringBuffer quotBuffer = new StringBuffer();
boolean foundUnescapedQuote = false;
while (quotMatcher.find()) {
quotMatcher.appendReplacement(quotBuffer, "\"");
foundUnescapedQuote = true;
}
if (foundUnescapedQuote) {
quotMatcher.appendTail(quotBuffer);
value = quotBuffer.toString();
}
}
}
return value;

12
providers/aws-s3/src/test/java/org/jclouds/aws/s3/AWSS3ClientLiveTest.java
View File

@ -19,7 +19,8 @@
package org.jclouds.aws.s3;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -57,6 +58,12 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
public AWSS3Client getApi() {
return (AWSS3Client) context.getProviderSpecificContext().getApi();
}
@Override
protected Module createHttpModule() {
// in order to be able to debug the wire protocol I use ApacheHC...
return new ApacheHCHttpCommandExecutorServiceModule();
}
@BeforeClass(groups = { "integration", "live" })
@Override
@ -92,12 +99,11 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
part1.getContentMetadata().setContentLength((long) buffer.length);
part1.getContentMetadata().setContentMD5(oneHundredOneConstitutionsMD5);
// failure here looks very similar to http://java.net/jira/browse/GLASSFISH-15773
String eTagOf1 = getApi().uploadPart(containerName, key, 1, uploadId, part1);
String eTag = getApi().completeMultipartUpload(containerName, key, uploadId, ImmutableMap.of(1, eTagOf1));
assertEquals(eTagOf1, eTag);
assertTrue(true);
} finally {
returnContainer(containerName);

2
providers/aws-s3/src/test/resources/complete-multipart-upload.xml
View File

@ -3,5 +3,5 @@
<Location>http://Example-Bucket.s3.amazonaws.com/Example-Object</Location>
<Bucket>Example-Bucket</Bucket>
<Key>Example-Object</Key>
<ETag>"3858f62230ac3c915f300c664312c11f-9"</ETag>
<ETag>&quot;3858f62230ac3c915f300c664312c11f-9&quot;</ETag>
</CompleteMultipartUploadResult>

35
providers/aws-s3/src/test/resources/log4j.xml
View File

@ -27,6 +27,28 @@
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
debug="false">
<!-- A time/date based rolling appender -->
<appender name="HTTPWIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="target/test-data/http-wire.log" />
<param name="Append" value="true" />
<!-- Rollover at midnight each day -->
<param name="DatePattern" value="'.'yyyy-MM-dd" />
<param name="Threshold" value="TRACE" />
<layout class="org.apache.log4j.PatternLayout">
<!-- The default pattern: Date Priority [Category] Message\n -->
<param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
<!--
The full pattern: Date MS Priority [Category]
(Thread:NDC) Message\n <param name="ConversionPattern"
value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
-->
</layout>
</appender>
<!-- A time/date based rolling appender -->
<appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="target/test-data/jclouds-wire.log" />
@ -85,6 +107,10 @@
<appender-ref ref="FILE" />
</appender>
<appender name="ASYNCHTTPWIRE" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="HTTPWIREFILE" />
</appender>
<appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
<appender-ref ref="WIREFILE" />
</appender>
@ -101,6 +127,15 @@
<appender-ref ref="ASYNC" />
</category>
<category name="org.apache.http">
<priority value="DEBUG" />
<appender-ref ref="ASYNCHTTPWIRE" />
</category>
<category name="org.apache.http.wire">
<priority value="ERROR" />
<appender-ref ref="ASYNCHTTPWIRE" />
</category>
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />