Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactored out s3nio module and centralized http handling code 

git-svn-id: http://jclouds.googlecode.com/svn/trunk@485 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-05-09 20:31:55 +00:00
parent 9ef49c6e5e
commit db14a0232d
67 changed files with 978 additions and 1271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPool.java
View File

@ -89,10 +89,10 @@ public abstract class FutureCommandConnectionPool<C, O extends FutureCommand<?,
.debug(
"%1s - attempting to acquire connection; %d currently available",
this, available.size());
C conn = available.poll(1, TimeUnit.SECONDS);
C conn = available.poll(5, TimeUnit.SECONDS);
if (conn == null)
throw new TimeoutException(
"could not obtain a pooled connection within 1 seconds");
"could not obtain a pooled connection within 5 seconds");
logger.trace("%1s - %2d - aquired", conn, conn.hashCode());
if (connectionValid(conn)) {

87
core/src/main/java/org/jclouds/http/BaseHttpFutureCommandClient.java Normal file
View File

@ -0,0 +1,87 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.http;
import java.net.URL;
import java.util.Collections;
import java.util.List;
import javax.annotation.Resource;
import org.jclouds.http.annotation.ClientErrorHandler;
import org.jclouds.http.annotation.RedirectHandler;
import org.jclouds.http.annotation.ServerErrorHandler;
import org.jclouds.logging.Logger;
import com.google.inject.Inject;
public abstract class BaseHttpFutureCommandClient implements HttpFutureCommandClient {
protected final URL target;
@Resource
protected Logger logger = Logger.NULL;
@Inject(optional = true)
protected List<HttpRequestFilter> requestFilters = Collections.emptyList();
@RedirectHandler
@Inject(optional = true)
protected HttpResponseHandler redirectHandler = new CloseContentAndSetExceptionHandler();
@ClientErrorHandler
@Inject(optional = true)
protected HttpResponseHandler clientErrorHandler = new CloseContentAndSetExceptionHandler();
@ServerErrorHandler
@Inject(optional = true)
protected HttpResponseHandler serverErrorHandler = new CloseContentAndSetExceptionHandler();
@Inject
public BaseHttpFutureCommandClient(URL target) {
this.target = target;
}
protected boolean isRetryable(HttpFutureCommand<?> command,
HttpResponse response) {
int code = response.getStatusCode();
if (command.getRequest().isReplayable() && code >= 500) {
logger.info("resubmitting command: %1s", command);
return true;
}
return false;
}
protected void handleResponse(HttpFutureCommand<?> command, HttpResponse response) {
int code = response.getStatusCode();
if (code >= 500) {
this.serverErrorHandler.handle(command, response);
} else if (code >= 400 && code < 500) {
this.clientErrorHandler.handle(command, response);
} else if (code >= 300 && code < 400) {
this.redirectHandler.handle(command, response);
} else {
command.getResponseFuture().setResponse(response);
command.getResponseFuture().run();
}
}
}

42
core/src/main/java/org/jclouds/http/CloseContentAndSetExceptionHandler.java Normal file
View File

@ -0,0 +1,42 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.http;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
/**
*
* @author Adrian Cole
*/
public class CloseContentAndSetExceptionHandler implements HttpResponseHandler {
public void handle(HttpFutureCommand<?> command, HttpResponse response) {
String message = String.format("Command: %2s failed; response: %1s",
response, command);
command.setException(new IOException(message));
IOUtils.closeQuietly(response.getContent());
}
}

9
core/src/main/java/org/jclouds/http/HttpFutureCommand.java
View File

@ -27,6 +27,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
import javax.annotation.Resource;
import org.apache.commons.io.IOUtils;
import org.jclouds.command.FutureCommand;
import org.jclouds.logging.Logger;
@ -64,6 +65,14 @@ public class HttpFutureCommand<T> extends
@Resource
protected Logger logger = Logger.NULL;
public void checkCode() {
int code = getResponse().getStatusCode();
if (code >= 300){
IOUtils.closeQuietly(getResponse().getContent());
throw new IllegalStateException("incorrect code for this operation: "+getResponse());
}
}
private HttpResponse response;
public HttpResponse getResponse() {

13
core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java
View File

@ -23,22 +23,13 @@
*/
package org.jclouds.http;
import java.util.List;
import org.jclouds.command.FutureCommandClient;
import com.google.inject.Inject;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public interface HttpFutureCommandClient
extends FutureCommandClient<HttpFutureCommand<?>> {
List<HttpRequestFilter> getRequestFilters();
@Inject
void setRequestFilters(List<HttpRequestFilter> requestFilters);
public interface HttpFutureCommandClient extends
FutureCommandClient<HttpFutureCommand<?>> {
}

1
core/src/main/java/org/jclouds/http/HttpRequestFilter.java
View File

@ -30,5 +30,4 @@ package org.jclouds.http;
*/
public interface HttpRequestFilter {
void filter(HttpRequest request) throws HttpException;
}

19
s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteCallable.java → core/src/main/java/org/jclouds/http/HttpResponseHandler.java
View File

@ -21,23 +21,18 @@
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.commands.callables;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
package org.jclouds.http;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public class DeleteCallable extends HttpFutureCommand.ResponseCallable<Boolean> {
public interface HttpResponseHandler {
public static final HttpResponseHandler NOOP = new HttpResponseHandler() {
public void handle(HttpFutureCommand<?> command, HttpResponse response) {
}
};
public Boolean call() throws HttpException {
if (getResponse().getStatusCode() == 204) {
return true;
} else {
throw new HttpException("Error deleting bucket " + getResponse());
}
}
void handle(HttpFutureCommand<?> command, HttpResponse response);
}

38
core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java
View File

@ -32,51 +32,33 @@ import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.List;
import javax.annotation.Resource;
import org.apache.commons.io.IOUtils;
import org.jclouds.logging.Logger;
import com.google.inject.Inject;
/**
* // TODO: Adrian: Document this!
* Basic implementation of a {@link HttpFutureCommandClient}.
*
* @author Adrian Cole
*/
public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
private URL target;
private List<HttpRequestFilter> requestFilters = Collections.emptyList();
@Resource
protected Logger logger = Logger.NULL;
public List<HttpRequestFilter> getRequestFilters() {
return requestFilters;
}
@Inject(optional = true)
public void setRequestFilters(List<HttpRequestFilter> requestFilters) {
this.requestFilters = requestFilters;
}
public class JavaUrlHttpFutureCommandClient extends BaseHttpFutureCommandClient {
@Inject
public JavaUrlHttpFutureCommandClient(URL target)
throws MalformedURLException {
this.target = target;
super(target);
}
public void submit(HttpFutureCommand<?> command) {
HttpRequest request = command.getRequest();
HttpURLConnection connection = null;
try {
for (HttpRequestFilter filter : getRequestFilters()) {
filter.filter(request);
}
HttpResponse response = null;
for (;;) {
for (HttpRequestFilter filter : requestFilters) {
filter.filter(request);
}
logger.trace("%1s - converting request %2s", target, request);
connection = openJavaConnection(request);
logger
@ -84,15 +66,11 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
connection);
response = getResponse(connection);
logger.trace("%1s - received response %2s", target, response);
if (command.getRequest().isReplayable()
&& response.getStatusCode() >= 500) {
logger.info("resubmitting command: %1s", command);
if (isRetryable(command, response))
continue;
}
break;
}
command.getResponseFuture().setResponse(response);
command.getResponseFuture().run();
handleResponse(command, response);
} catch (Exception e) {
command.setException(e);
} finally {

44
core/src/main/java/org/jclouds/http/annotation/ClientErrorHandler.java Normal file
View File

@ -0,0 +1,44 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.http.annotation;
import com.google.inject.BindingAnnotation;
import java.lang.annotation.Target;
import java.lang.annotation.Retention;
import static java.lang.annotation.RetentionPolicy.RUNTIME;
import static java.lang.annotation.ElementType.PARAMETER;
import static java.lang.annotation.ElementType.FIELD;
import static java.lang.annotation.ElementType.METHOD;
/**
* Implies that the object can address {@link HttpResponse}s that contain status
* code 4xx.
*
* @author Adrian Cole
*/
@BindingAnnotation
@Target( { FIELD, PARAMETER, METHOD })
@Retention(RUNTIME)
public @interface ClientErrorHandler {
}

44
core/src/main/java/org/jclouds/http/annotation/RedirectHandler.java Normal file
View File

@ -0,0 +1,44 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.http.annotation;
import com.google.inject.BindingAnnotation;
import java.lang.annotation.Target;
import java.lang.annotation.Retention;
import static java.lang.annotation.RetentionPolicy.RUNTIME;
import static java.lang.annotation.ElementType.PARAMETER;
import static java.lang.annotation.ElementType.FIELD;
import static java.lang.annotation.ElementType.METHOD;
/**
* Implies that the object can address {@link HttpResponse}s that contain status
* code 3xx.
*
* @author Adrian Cole
*/
@BindingAnnotation
@Target( { FIELD, PARAMETER, METHOD })
@Retention(RUNTIME)
public @interface RedirectHandler {
}

44
core/src/main/java/org/jclouds/http/annotation/ServerErrorHandler.java Normal file
View File

@ -0,0 +1,44 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.http.annotation;
import com.google.inject.BindingAnnotation;
import java.lang.annotation.Target;
import java.lang.annotation.Retention;
import static java.lang.annotation.RetentionPolicy.RUNTIME;
import static java.lang.annotation.ElementType.PARAMETER;
import static java.lang.annotation.ElementType.FIELD;
import static java.lang.annotation.ElementType.METHOD;
/**
* Implies that the object can address {@link HttpResponse}s that contain status
* code 5xx.
*
* @author Adrian Cole
*/
@BindingAnnotation
@Target( { FIELD, PARAMETER, METHOD })
@Retention(RUNTIME)
public @interface ServerErrorHandler {
}

4
core/src/main/java/org/jclouds/http/commands/Head.java
View File

@ -26,7 +26,7 @@ package org.jclouds.http.commands;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.commands.callables.ReturnTrueIf200;
import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
/**
* // TODO: Adrian: Document this!
@ -36,7 +36,7 @@ import org.jclouds.http.commands.callables.ReturnTrueIf200;
public class Head extends HttpFutureCommand<Boolean> {
@Inject
public Head(ReturnTrueIf200 callable, @Assisted String uri) {
public Head(ReturnTrueIf2xx callable, @Assisted String uri) {
super("HEAD", uri, callable);
}
}

7
core/src/main/java/org/jclouds/http/commands/callables/ReturnStringIf200.java
View File

@ -46,11 +46,8 @@ public class ReturnStringIf200 extends
}
public String call() throws HttpException {
int code = getResponse().getStatusCode();
if (code >= 400 && code < 500) {
throw new HttpException(String.format("Content not found - %1s",
getResponse()));
} else if (code == 200) {
checkCode();
if (getResponse().getStatusCode() == 200) {
InputStream entity = getResponse().getContent();
if (entity == null)
throw new HttpException("no content");

15
core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf200.java → core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf2xx.java
View File

@ -23,31 +23,28 @@
*/
package org.jclouds.http.commands.callables;
import org.apache.commons.io.IOUtils;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
import com.google.inject.Inject;
/**
* // TODO: Adrian: Document this!
* Simply returns true when the http response code is in the range 200-299.
*
* @author Adrian Cole
*/
public class ReturnTrueIf200 extends
public class ReturnTrueIf2xx extends
HttpFutureCommand.ResponseCallable<Boolean> {
@Inject
public ReturnTrueIf200() {
public ReturnTrueIf2xx() {
super();
}
public Boolean call() throws HttpException {
if (getResponse().getStatusCode() == 200) {
checkCode();
IOUtils.closeQuietly(getResponse().getContent());
return true;
} else if (getResponse().getStatusCode() == 404) {
return false;
} else {
throw new HttpException("Error checking bucket " + getResponse());
}
}
}

24
core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java
View File

@ -68,7 +68,7 @@ import com.google.inject.name.Names;
*
* @author Adrian Cole
*/
@Test(threadPoolSize = 100)
@Test(threadPoolSize = 10)
public abstract class BaseHttpFutureCommandClientTest {
protected static final String XML = "<foo><bar>whoppers</bar></foo>";
protected Server server = null;
@ -158,9 +158,9 @@ public abstract class BaseHttpFutureCommandClientTest {
server.stop();
}
@Test(invocationCount = 500, timeOut = 1500)
public void testRequestFilter() throws MalformedURLException, ExecutionException,
InterruptedException, TimeoutException {
@Test(invocationCount = 50, timeOut = 1500)
public void testRequestFilter() throws MalformedURLException,
ExecutionException, InterruptedException, TimeoutException {
GetString get = factory.createGetString("/");
get.getRequest().getHeaders().put("filterme", "filterme");
client.submit(get);
@ -169,7 +169,7 @@ public abstract class BaseHttpFutureCommandClientTest {
TimeUnit.SECONDS));
}
@Test(invocationCount = 500, timeOut = 1500)
@Test(invocationCount = 50, timeOut = 1500)
public void testGetStringWithHeader() throws MalformedURLException,
ExecutionException, InterruptedException, TimeoutException {
GetString get = factory.createGetString("/");
@ -180,9 +180,9 @@ public abstract class BaseHttpFutureCommandClientTest {
TimeUnit.SECONDS));
}
@Test(invocationCount = 500, timeOut = 1500)
public void testGetString() throws MalformedURLException, ExecutionException,
InterruptedException, TimeoutException {
@Test(invocationCount = 50, timeOut = 1500)
public void testGetString() throws MalformedURLException,
ExecutionException, InterruptedException, TimeoutException {
GetString get = factory.createGetString("/");
assert get != null;
client.submit(get);
@ -191,7 +191,7 @@ public abstract class BaseHttpFutureCommandClientTest {
TimeUnit.SECONDS));
}
@Test(invocationCount = 500, timeOut = 1500)
@Test(invocationCount = 50, timeOut = 1500)
public void testHead() throws MalformedURLException, ExecutionException,
InterruptedException, TimeoutException {
Head head = factory.createHead("/");
@ -200,9 +200,9 @@ public abstract class BaseHttpFutureCommandClientTest {
assert head.get(10, TimeUnit.SECONDS);
}
@Test(invocationCount = 500, timeOut = 1500)
public void testGetAndParseSax() throws MalformedURLException, ExecutionException,
InterruptedException, TimeoutException {
@Test(invocationCount = 50, timeOut = 1500)
public void testGetAndParseSax() throws MalformedURLException,
ExecutionException, InterruptedException, TimeoutException {
GetAndParseSax<?> getAndParseSax = factory.createGetAndParseSax("/",
new ParseSax.HandlerWithResult<String>() {
@Override

6
core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java
View File

@ -58,7 +58,7 @@ public class ReturnStringIf200Test {
public void testExceptionWhenNoContentOn200() throws ExecutionException,
InterruptedException, TimeoutException, IOException {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
expect(response.getContent()).andReturn(null);
replay(response);
callable.setResponse(response);
@ -74,7 +74,7 @@ public class ReturnStringIf200Test {
public void testExceptionWhenIOExceptionOn200() throws ExecutionException,
InterruptedException, TimeoutException, IOException {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
RuntimeException exception = new RuntimeException("bad");
expect(response.getContent()).andThrow(exception);
replay(response);
@ -90,7 +90,7 @@ public class ReturnStringIf200Test {
@Test
public void testResponseOk() throws Exception {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
expect(response.getContent()).andReturn(IOUtils.toInputStream("hello"));
replay(response);
callable.setResponse(response);

26
extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java
View File

@ -45,20 +45,26 @@ public class HttpNioUtils {
HttpRequest object) {
BasicHttpEntityEnclosingRequest apacheRequest = new BasicHttpEntityEnclosingRequest(
object.getMethod(), object.getUri(), HttpVersion.HTTP_1_1);
Object content = object.getPayload();
// Since we may remove headers, ensure they are added to the apache
// request after this block
if (content != null) {
long contentLength = Long.parseLong(object
.getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH));
object.getHeaders().removeAll(HttpHeaders.CONTENT_LENGTH);
String contentType = object
.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE);
object.getHeaders().removeAll(HttpHeaders.CONTENT_TYPE);
addEntityForContent(apacheRequest, content, contentType,
contentLength);
}
for (String header : object.getHeaders().keySet()) {
for (String value : object.getHeaders().get(header))
apacheRequest.addHeader(header, value);
}
Object content = object.getPayload();
if (content != null) {
addEntityForContent(
apacheRequest,
content,
object.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE),
Long
.parseLong(object
.getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH)));
}
return apacheRequest;
}

65
extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandExecutionHandler.java
View File

@ -35,8 +35,13 @@ import org.apache.http.HttpResponse;
import org.apache.http.nio.entity.ConsumingNHttpEntity;
import org.apache.http.nio.protocol.NHttpRequestExecutionHandler;
import org.apache.http.protocol.HttpContext;
import org.jclouds.http.CloseContentAndSetExceptionHandler;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpResponseHandler;
import org.jclouds.http.annotation.ClientErrorHandler;
import org.jclouds.http.annotation.RedirectHandler;
import org.jclouds.http.annotation.ServerErrorHandler;
import org.jclouds.http.httpnio.HttpNioUtils;
import org.jclouds.logging.Logger;
@ -55,6 +60,18 @@ public class HttpNioFutureCommandExecutionHandler implements
private final ConsumingNHttpEntityFactory entityFactory;
private final BlockingQueue<HttpFutureCommand<?>> commandQueue;
@RedirectHandler
@Inject(optional = true)
private HttpResponseHandler redirectHandler = new CloseContentAndSetExceptionHandler();
@ClientErrorHandler
@Inject(optional = true)
private HttpResponseHandler clientErrorHandler = new CloseContentAndSetExceptionHandler();
@ServerErrorHandler
@Inject(optional = true)
private HttpResponseHandler serverErrorHandler = new CloseContentAndSetExceptionHandler();
public interface ConsumingNHttpEntityFactory {
public ConsumingNHttpEntity create(HttpEntity httpEntity);
}
@ -88,27 +105,29 @@ public class HttpNioFutureCommandExecutionHandler implements
return entityFactory.create(response.getEntity());
}
public void handleResponse(HttpResponse response, HttpContext context)
public void handleResponse(HttpResponse apacheResponse, HttpContext context)
throws IOException {
HttpNioFutureCommandConnectionHandle handle = (HttpNioFutureCommandConnectionHandle) context
.removeAttribute("command-handle");
if (handle != null) {
try {
HttpFutureCommand<?> command = handle.getCommand();
int code = response.getStatusLine().getStatusCode();
// normal codes for rest commands
if ((code >= 200 && code < 300) || code == 404) {
processResponse(response, command);
org.jclouds.http.HttpResponse response = HttpNioUtils
.convertToJavaCloudsResponse(apacheResponse);
int code = response.getStatusCode();
if (code >= 500) {
if (isRetryable(command)) {
commandQueue.add(command);
} else {
if (isRetryable(response)) {
attemptReplay(command);
} else {
String message = String
.format(
"response is not retryable: %1s; Command: %2s failed",
response.getStatusLine(), command);
command.setException(new IOException(message));
this.serverErrorHandler.handle(command, response);
}
} else if (code >= 400 && code < 500) {
this.clientErrorHandler.handle(command, response);
} else if (code >= 300 && code < 400) {
this.redirectHandler.handle(command, response);
} else {
processResponse(response, command);
}
} finally {
releaseConnectionToPool(handle);
@ -119,18 +138,12 @@ public class HttpNioFutureCommandExecutionHandler implements
}
}
protected void attemptReplay(HttpFutureCommand<?> command) {
if (command.getRequest().isReplayable())
commandQueue.add(command);
else
command.setException(new IOException(String.format(
"%1s: command failed and request is not replayable: %2s",
command, command.getRequest())));
protected boolean isRetryable(HttpFutureCommand<?> command) {
if (command.getRequest().isReplayable()) {
logger.info("resubmitting command: %1s", command);
return true;
}
protected boolean isRetryable(HttpResponse response) throws IOException {
int code = response.getStatusLine().getStatusCode();
return code == 500 || code == 503;
return false;
}
protected void releaseConnectionToPool(
@ -142,10 +155,8 @@ public class HttpNioFutureCommandExecutionHandler implements
}
}
protected void processResponse(HttpResponse apacheResponse,
protected void processResponse(org.jclouds.http.HttpResponse response,
HttpFutureCommand<?> command) throws IOException {
org.jclouds.http.HttpResponse response = HttpNioUtils
.convertToJavaCloudsResponse(apacheResponse);
command.getResponseFuture().setResponse(response);
logger.trace("submitting response task %1s", command
.getResponseFuture());

93
extensions/s3nio/pom.xml
View File

@ -1,93 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
$HeadURL$
$Revision$
$Date$
Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0.html
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<groupId>org.jclouds</groupId>
<artifactId>jclouds-project</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../../project/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.jclouds</groupId>
<artifactId>jclouds-s3nio</artifactId>
<name>Connection pooling NIO connection for S3</name>
<packaging>jar</packaging>
<description>Connection pooling NIO connection for S3</description>
<scm>
<connection>scm:svn:http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</connection>
<developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk/extensions/s3nio</developerConnection>
<url>http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</url>
</scm>
<properties>
<jclouds.aws.accesskeyid></jclouds.aws.accesskeyid>
<jclouds.aws.secretaccesskey></jclouds.aws.secretaccesskey>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-httpnio</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemProperties>
<property>
<name>jclouds.aws.accesskeyid</name>
<value>${jclouds.aws.accesskeyid}</value>
</property>
<property>
<name>jclouds.aws.secretaccesskey</name>
<value>${jclouds.aws.secretaccesskey}</value>
</property>
</systemProperties>
</configuration>
</plugin>
</plugins>
</build>
</project>

91
extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandler.java
View File

@ -1,91 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.nio;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.nio.entity.NStringEntity;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandExecutionHandler;
import com.google.inject.Inject;
import com.google.inject.Singleton;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Singleton
public class S3HttpNioFutureCommandExecutionHandler extends
HttpNioFutureCommandExecutionHandler {
@Inject
public S3HttpNioFutureCommandExecutionHandler(
ConsumingNHttpEntityFactory entityFactory,
ExecutorService executor,
BlockingQueue<HttpFutureCommand<?>> commandQueue) {
super(entityFactory, executor, commandQueue);
}
@Override
protected boolean isRetryable(HttpResponse response) throws IOException {
if (super.isRetryable(response))
return true;
int code = response.getStatusLine().getStatusCode();
if (code == 409) {
return true;
} else if (code == 400) {
if (response.getEntity() != null) {
InputStream input = response.getEntity().getContent();
if (input != null) {
String reason = null;
try {
reason = IOUtils.toString(input);
} finally {
IOUtils.closeQuietly(input);
}
if (reason != null) {
try {
if (reason.indexOf("RequestTime") >= 0)
return true;
} finally {
IOUtils.closeQuietly(input);
response.setEntity(new NStringEntity(reason));
}
}
}
}
}
return false;
}
}

51
extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModule.java
View File

@ -1,51 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.nio.config;
import org.apache.http.nio.protocol.NHttpRequestExecutionHandler;
import org.jclouds.aws.s3.nio.S3HttpNioFutureCommandExecutionHandler;
import org.jclouds.http.config.HttpFutureCommandClientModule;
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
import com.google.inject.AbstractModule;
import com.google.inject.util.Modules;
/**
* This installs a {@link HttpNioConnectionPoolClientModule}, but overrides it
* binding {@link S3HttpNioFutureCommandExecutionHandler}.
*
* @author Adrian Cole
*/
@HttpFutureCommandClientModule
public class S3HttpNioConnectionPoolClientModule extends AbstractModule {
protected void configure() {
install(Modules.override(new HttpNioConnectionPoolClientModule()).with(
new AbstractModule() {
protected void configure() {
bind(NHttpRequestExecutionHandler.class).to(
S3HttpNioFutureCommandExecutionHandler.class);
}
}));
}
}

61
extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3InputStreamMapTest.java
View File

@ -1,61 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.nio;
import java.util.Properties;
import org.jclouds.aws.s3.S3InputStreamMapTest;
import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test;
import com.google.inject.Module;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test(groups = "unit", sequential = true, testName = "s3.NioS3InputStreamMapTest")
public class NioS3InputStreamMapTest extends S3InputStreamMapTest {
@Override
protected Properties buildS3Properties(String AWSAccessKeyId,
String AWSSecretAccessKey) {
Properties properties = super.buildS3Properties(AWSAccessKeyId,
AWSSecretAccessKey);
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
properties.setProperty("jclouds.http.pool.max_session_failures", "2");
properties
.setProperty("jclouds.http.pool.request_invoker_threads", "1");
properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
properties.setProperty("jclouds.pool.max_connections", "12");
return properties;
}
@Override
protected Module createHttpModule() {
return new S3HttpNioConnectionPoolClientModule();
}
}

61
extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java
View File

@ -1,61 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.nio;
import java.util.Properties;
import org.jclouds.aws.s3.S3ObjectMapTest;
import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test;
import com.google.inject.Module;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test(groups = "unit", sequential = true, testName = "s3.NioS3ObjectMapTest")
public class NioS3ObjectMapTest extends S3ObjectMapTest {
@Override
protected Properties buildS3Properties(String AWSAccessKeyId,
String AWSSecretAccessKey) {
Properties properties = super.buildS3Properties(AWSAccessKeyId,
AWSSecretAccessKey);
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
properties.setProperty("jclouds.http.pool.max_session_failures", "2");
properties
.setProperty("jclouds.http.pool.request_invoker_threads", "1");
properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
properties.setProperty("jclouds.pool.max_connections", "12");
return properties;
}
@Override
protected Module createHttpModule() {
return new S3HttpNioConnectionPoolClientModule();
}
}

148
extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandlerTest.java
View File

@ -1,148 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.nio;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.isA;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import static org.easymock.classextension.EasyMock.verify;
import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.nio.entity.NStringEntity;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandExecutionHandler;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test
public class S3HttpNioFutureCommandExecutionHandlerTest {
S3HttpNioFutureCommandExecutionHandler handler = null;
HttpResponse response = null;
StatusLine statusline = null;
@BeforeMethod
public void createHandler() {
handler = new S3HttpNioFutureCommandExecutionHandler(
createMock(HttpNioFutureCommandExecutionHandler.ConsumingNHttpEntityFactory.class),
createMock(ExecutorService.class),
new ArrayBlockingQueue<HttpFutureCommand<?>>(1));
response = createMock(HttpResponse.class);
statusline = createMock(StatusLine.class);
expect(response.getStatusLine()).andReturn(statusline).atLeastOnce();
}
@Test
void test500isRetryable() throws IOException {
isRetryable(500);
}
@Test
void test503isRetryable() throws IOException {
isRetryable(503);
}
@Test
void test409isRetryable() throws IOException {
isRetryable(409);
}
@Test
void test404NotRetryable() throws IOException {
expect(statusline.getStatusCode()).andReturn(404).atLeastOnce();
replay(statusline);
replay(response);
assert !handler.isRetryable(response);
verify(statusline);
verify(response);
}
@Test
void test400WithNoEnitityNotRetryable() throws IOException {
expect(statusline.getStatusCode()).andReturn(400).atLeastOnce();
expect(response.getEntity()).andReturn(null);
replay(statusline);
replay(response);
assert !handler.isRetryable(response);
verify(statusline);
verify(response);
}
@Test
void test400WithIrrelevantEnitityNotRetryable() throws IOException {
expect(statusline.getStatusCode()).andReturn(400).atLeastOnce();
HttpEntity entity = createMock(HttpEntity.class);
expect(response.getEntity()).andReturn(entity).atLeastOnce();
expect(entity.getContent()).andReturn(IOUtils.toInputStream("hello"));
response.setEntity(isA(NStringEntity.class));
replay(entity);
replay(statusline);
replay(response);
assert !handler.isRetryable(response);
verify(statusline);
verify(response);
verify(entity);
}
@Test
void test400WithRequestTimeTooSkewedTimeEnitityRetryable()
throws IOException {
expect(statusline.getStatusCode()).andReturn(400).atLeastOnce();
HttpEntity entity = createMock(HttpEntity.class);
expect(response.getEntity()).andReturn(entity).atLeastOnce();
expect(entity.getContent()).andReturn(
IOUtils.toInputStream("RequestTimeTooSkewed"));
response.setEntity(isA(NStringEntity.class));
replay(entity);
replay(statusline);
replay(response);
assert handler.isRetryable(response);
verify(statusline);
verify(response);
verify(entity);
}
private void isRetryable(int code) throws IOException {
expect(statusline.getStatusCode()).andReturn(code).atLeastOnce();
replay(statusline);
replay(response);
assert handler.isRetryable(response);
verify(statusline);
verify(response);
}
}

64
extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModuleTest.java
View File

@ -1,64 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.nio.config;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.name.Names;
import org.apache.http.nio.protocol.NHttpRequestExecutionHandler;
import org.jclouds.aws.s3.nio.S3HttpNioFutureCommandExecutionHandler;
import org.testng.annotations.Test;
import java.util.Properties;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test
public class S3HttpNioConnectionPoolClientModuleTest {
public void testConfigureBindsS3Handler() {
final Properties properties = new Properties();
properties.put("jclouds.http.address", "localhost");
properties.put("jclouds.http.port", "8088");
properties.put("jclouds.http.secure", "false");
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
properties.setProperty("jclouds.http.pool.max_session_failures", "2");
properties.setProperty("jclouds.http.pool.request_invoker_threads", "1");
properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
properties.setProperty("jclouds.pool.max_connections", "12");
Injector i = Guice.createInjector(new S3HttpNioConnectionPoolClientModule() {
@Override
protected void configure() {
Names.bindProperties(binder(), properties);
super.configure();
}
});
NHttpRequestExecutionHandler handler = i.getInstance(NHttpRequestExecutionHandler.class);
assert handler instanceof S3HttpNioFutureCommandExecutionHandler;
}
}

61
gae/src/main/java/org/jclouds/gae/URLFetchServiceClient.java
View File

@ -32,18 +32,15 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.List;
import javax.annotation.Resource;
import org.apache.commons.io.IOUtils;
import org.jclouds.http.BaseHttpFutureCommandClient;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpResponse;
import org.jclouds.logging.Logger;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
@ -58,67 +55,61 @@ import com.google.inject.Inject;
*
* @author Adrian Cole
*/
public class URLFetchServiceClient implements HttpFutureCommandClient {
private final URL target;
private List<HttpRequestFilter> requestFilters = Collections.emptyList();
@Resource
private Logger logger = Logger.NULL;
public class URLFetchServiceClient extends BaseHttpFutureCommandClient {
private final URLFetchService urlFetchService;
public List<HttpRequestFilter> getRequestFilters() {
return requestFilters;
}
@Inject(optional = true)
public void setRequestFilters(List<HttpRequestFilter> requestFilters) {
this.requestFilters = requestFilters;
}
@Inject
public URLFetchServiceClient(URL target, URLFetchService urlFetchService)
throws MalformedURLException {
super(target);
this.urlFetchService = urlFetchService;
this.target = target;
this.logger.info("configured to connect to target: %1s", target);
}
public void submit(HttpFutureCommand<?> operation) {
HttpRequest request = operation.getRequest();
public void submit(HttpFutureCommand<?> command) {
HttpRequest request = command.getRequest();
HTTPResponse gaeResponse = null;
try {
for (HttpRequestFilter filter : getRequestFilters()) {
for (HttpRequestFilter filter : requestFilters) {
filter.filter(request);
}
HttpResponse response = null;
for (;;) {
logger.trace("%1s - converting request %2s", target, request);
HTTPRequest gaeRequest = convert(request);
logger
.trace("%1s - submitting request %2s", target,
gaeRequest);
if (logger.isTraceEnabled())
logger.trace("%1s - submitting request %2s, headers: %3s",
target, gaeRequest.getURL(),
headersAsString(gaeRequest.getHeaders()));
gaeResponse = this.urlFetchService.fetch(gaeRequest);
logger
.trace("%1s - received response %2s", target,
gaeResponse);
if (logger.isTraceEnabled())
logger.trace(
"%1s - received response code %2s, headers: %3s",
target, gaeResponse.getResponseCode(),
headersAsString(gaeResponse.getHeaders()));
response = convert(gaeResponse);
if (response.getStatusCode() >= 500) {
if (isRetryable(command, response))
continue;
}
break;
}
operation.getResponseFuture().setResponse(response);
operation.getResponseFuture().run();
handleResponse(command, response);
} catch (Exception e) {
if (gaeResponse != null && gaeResponse.getContent() != null) {
logger.error(e,
"error encountered during the execution: %1s%n%2s",
gaeResponse, new String(gaeResponse.getContent()));
}
operation.setException(e);
command.setException(e);
}
}
String headersAsString(List<HTTPHeader> headers) {
StringBuilder builder = new StringBuilder("");
for (HTTPHeader header : headers)
builder.append("[").append(header.getName()).append("=").append(
header.getValue()).append("],");
return builder.toString();
}
/**
* byte [] content is replayable and the only content type supportable by
* GAE. As such, we convert the original request content to a byte array.

10
gae/src/test/java/org/jclouds/gae/URLFetchServiceClientTest.java
View File

@ -41,7 +41,6 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.jclouds.http.HttpHeaders;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpResponse;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Parameters;
@ -163,15 +162,6 @@ public class URLFetchServiceClientTest {
}
@Test
void testRequestFilters() {
List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>();
filters.add(createNiceMock(HttpRequestFilter.class));
assertEquals(client.getRequestFilters().size(), 0);
client.setRequestFilters(filters);
assertEquals(client.getRequestFilters(), filters);
}
@Test
@Parameters("basedir")
void testConvertRequestFileContent(String basedir) throws IOException {

1
pom.xml
View File

@ -41,7 +41,6 @@
<module>project</module>
<module>core</module>
<module>extensions/httpnio</module>
<module>extensions/s3nio</module>
<module>extensions/log4j</module>
<module>gae</module>
<module>s3</module>

2
s3/perftest/pom.xml
View File

@ -74,7 +74,7 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3nio</artifactId>
<artifactId>jclouds-httpnio</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>

10
s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java → s3/perftest/src/test/java/com/amazon/s3/AmazonPerformanceTest.java
View File

@ -42,8 +42,8 @@ import org.testng.annotations.Test;
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformance", groups = "performance")
public class AmazonPerformance extends BasePerformance {
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformance")
public class AmazonPerformanceTest extends BasePerformance {
private AWSAuthConnection amzClient;
@Override
@ -58,33 +58,39 @@ public class AmazonPerformance extends BasePerformance {
}
@Override
@Test(enabled = false)
public void testPutFileSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();

1
s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java
View File

@ -51,6 +51,7 @@ import com.google.inject.Provider;
*
* @author Adrian Cole
*/
@Test
public abstract class BasePerformance extends S3IntegrationTest {
protected boolean debugEnabled() {
return false;

2
s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java
View File

@ -39,7 +39,7 @@ import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest", groups = "performance")
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest")
public class DateServiceTest extends PerformanceTest {
Injector i = Guice.createInjector();

15
s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformance.java → s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformanceTest.java
View File

@ -24,14 +24,15 @@
package com.amazon.s3;
import com.google.inject.Module;
import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test;
import java.util.Properties;
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsNioPerformance", groups = "performance")
public class JCloudsNioPerformance extends BaseJCloudsPerformance {
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test;
import com.google.inject.Module;
@Test(sequential=true, testName = "s3.JCloudsNioPerformance")
public class JCloudsNioPerformanceTest extends BaseJCloudsPerformance {
@Override
protected Properties buildS3Properties(String AWSAccessKeyId,
@ -47,6 +48,6 @@ public class JCloudsNioPerformance extends BaseJCloudsPerformance {
@Override
protected Module createHttpModule() {
return new S3HttpNioConnectionPoolClientModule();
return new HttpNioConnectionPoolClientModule();
}
}

4
s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformance.java → s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformanceTest.java
View File

@ -26,7 +26,7 @@ package com.amazon.s3;
import org.testng.annotations.Test;
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsPerformance", groups = "performance")
public class JCloudsPerformance extends BaseJCloudsPerformance {
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsPerformance")
public class JCloudsPerformanceTest extends BaseJCloudsPerformance {
}

22
s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformance.java → s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformanceTest.java
View File

@ -37,8 +37,8 @@ import org.testng.annotations.Test;
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformance", groups = "performance")
public class Jets3tPerformance extends BasePerformance {
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformance")
public class Jets3tPerformanceTest extends BasePerformance {
private S3Service jetClient;
@Override
@ -50,28 +50,46 @@ public class Jets3tPerformance extends BasePerformance {
}
@Override
@Test(enabled=false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled=false)
public void testPutStringParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled=false)
public void testPutBytesSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled=false)
public void testPutBytesParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled=false)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled=false)
public void testPutFileParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled=false)
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
throw new UnsupportedOperationException();

2
s3/perftest/src/test/java/com/amazon/s3/S3ParserTest.java
View File

@ -42,7 +42,7 @@ import java.util.concurrent.ExecutorCompletionService;
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3ParserTest", groups = "performance")
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3ParserTest")
public class S3ParserTest extends org.jclouds.aws.s3.commands.S3ParserTest {
class MockHttpURLConnection extends HttpURLConnection {

2
s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java
View File

@ -33,7 +33,7 @@ import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest", groups = "performance")
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest")
public class S3UtilsTest extends org.jclouds.aws.s3.S3UtilsTest {
@Test(dataProvider = "hmacsha1")

2
s3/src/main/java/org/jclouds/aws/s3/S3Constants.java
View File

@ -35,6 +35,6 @@ public interface S3Constants extends HttpConstants, PoolConstants, S3Headers {
public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey";
public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid";
public static final String PROPERTY_AWS_MAP_TIMEOUT = "jclouds.aws.map.timeout";
public static final String PROPERTY_S3_MAP_TIMEOUT = "jclouds.s3.map.timeout";
}

6
s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java
View File

@ -39,8 +39,8 @@ import java.util.List;
import java.util.Properties;
import org.jclouds.aws.s3.config.S3ContextModule;
import org.jclouds.aws.s3.config.S3JavaUrlHttpFutureCommandClientModule;
import org.jclouds.http.config.HttpFutureCommandClientModule;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import org.jclouds.logging.config.LoggingModule;
import org.jclouds.logging.jdk.config.JDKLoggingModule;
@ -165,7 +165,7 @@ public class S3ContextFactory {
/**
* Bind the given properties and install the list of modules. If no modules
* are specified, install the default {@link JDKLoggingModule}
* {@link S3JavaUrlHttpFutureCommandClientModule}
* {@link JavaUrlHttpFutureCommandClientModule}
*
* @param properties
* - contains constants used by jclouds
@ -201,7 +201,7 @@ public class S3ContextFactory {
}
}))
modules.add(new S3JavaUrlHttpFutureCommandClientModule());
modules.add(new JavaUrlHttpFutureCommandClientModule());
}
@VisibleForTesting

33
s3/src/main/java/org/jclouds/aws/s3/S3ResponseException.java
View File

@ -24,19 +24,33 @@
package org.jclouds.aws.s3;
import org.jclouds.aws.s3.domain.S3Error;
import org.jclouds.http.HttpMessage;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse;
public class S3ResponseException extends RuntimeException {
private static final long serialVersionUID = 1L;
private final HttpFutureCommand<?> command;
private final HttpResponse response;
private S3Error error;
private HttpMessage response;
public S3ResponseException(S3Error error, HttpMessage response) {
super(error.toString());
public S3ResponseException(HttpFutureCommand<?> command,
HttpResponse response, S3Error error) {
super(String.format(
"command: %1s failed with response: %2s; error from s3: %3s",
command, response, error));
this.command = command;
this.response = response;
this.setError(error);
this.setResponse(response);
}
public S3ResponseException(HttpFutureCommand<?> command,
HttpResponse response) {
super(String.format("command: %1s failed with response: %2s", command,
response));
this.command = command;
this.response = response;
}
public void setError(S3Error error) {
@ -47,11 +61,12 @@ public class S3ResponseException extends RuntimeException {
return error;
}
public void setResponse(HttpMessage response) {
this.response = response;
public HttpFutureCommand<?> getCommand() {
return command;
}
public HttpMessage getResponse() {
public HttpResponse getResponse() {
return response;
}
}

86
s3/src/main/java/org/jclouds/aws/s3/S3Utils.java
View File

@ -23,8 +23,10 @@
*/
package org.jclouds.aws.s3;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
@ -32,6 +34,8 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.bouncycastle.crypto.digests.MD5Digest;
import org.bouncycastle.crypto.digests.SHA1Digest;
import org.bouncycastle.crypto.macs.HMac;
@ -60,6 +64,39 @@ public class S3Utils extends Utils {
return new String(hex, "ASCII");
}
public static long calculateSize(Object data) {
long size = -1;
if (data instanceof byte[]) {
size = ((byte[]) data).length;
} else if (data instanceof String) {
size = ((String) data).length();
} else if (data instanceof File) {
size = ((File) data).length();
}
return size;
}
/**
*
* @throws IOException
*/
public static byte[] md5(Object data) throws IOException {
checkNotNull(data, "data must be set before calling generateMd5()");
byte[] md5 = null;
if (data == null || data instanceof byte[]) {
md5 = S3Utils.md5((byte[]) data);
} else if (data instanceof String) {
md5 = S3Utils.md5(((String) data).getBytes());
} else if (data instanceof File) {
md5 = S3Utils.md5(((File) data));
} else {
throw new UnsupportedOperationException("Content not supported "
+ data.getClass());
}
return md5;
}
public static byte[] fromHexString(String hex) {
byte[] bytes = new byte[hex.length() / 2];
for (int i = 0; i < bytes.length; i++) {
@ -108,21 +145,64 @@ public class S3Utils extends Utils {
return resBuf;
}
public static byte[] md5(InputStream toEncode) throws IOException {
public static byte[] md5(File toEncode) throws IOException {
MD5Digest md5 = new MD5Digest();
byte[] resBuf = new byte[md5.getDigestSize()];
byte[] buffer = new byte[1024];
int numRead = -1;
InputStream i = new FileInputStream(toEncode);
try {
do {
numRead = toEncode.read(buffer);
numRead = i.read(buffer);
if (numRead > 0) {
md5.update(buffer, 0, numRead);
}
} while (numRead != -1);
} finally {
IOUtils.closeQuietly(i);
}
md5.doFinal(resBuf, 0);
return resBuf;
}
public static Md5InputStreamResult generateMd5Result(InputStream toEncode)
throws IOException {
MD5Digest md5 = new MD5Digest();
byte[] resBuf = new byte[md5.getDigestSize()];
byte[] buffer = new byte[1024];
ByteArrayOutputStream out = new ByteArrayOutputStream();
long length = 0;
int numRead = -1;
try {
do {
numRead = toEncode.read(buffer);
if (numRead > 0) {
length += numRead;
md5.update(buffer, 0, numRead);
out.write(buffer, 0, numRead);
}
} while (numRead != -1);
} finally {
IOUtils.closeQuietly(toEncode);
}
md5.doFinal(resBuf, 0);
return new Md5InputStreamResult(out.toByteArray(), resBuf, length);
}
public static class Md5InputStreamResult {
public final byte[] data;
public final byte[] md5;
public final long length;
Md5InputStreamResult(byte[] data, byte[] md5, long length) {
this.data = checkNotNull(data, "data");
this.md5 = checkNotNull(md5, "md5");
checkArgument(length >= 0, "length cannot me negative");
this.length = length;
}
}
public static String getContentAsStringAndClose(S3Object object)
throws IOException {
checkNotNull(object, "s3Object");

46
s3/src/main/java/org/jclouds/aws/s3/commands/BucketExists.java
View File

@ -24,8 +24,15 @@
package org.jclouds.aws.s3.commands;
import static org.jclouds.aws.s3.commands.options.ListBucketOptions.Builder.maxResults;
import org.jclouds.http.commands.callables.ReturnTrueIf200;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3ResponseException;
import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.name.Named;
@ -34,8 +41,39 @@ public class BucketExists extends S3FutureCommand<Boolean> {
@Inject
public BucketExists(@Named("jclouds.http.address") String amazonHost,
ReturnTrueIf200 callable, @Assisted String s3Bucket) {
super("HEAD", "/" + maxResults(0).buildQueryString(), callable, amazonHost,
s3Bucket);
ReturnTrueIf2xx callable, @Assisted String s3Bucket) {
super("HEAD", "/" + maxResults(0).buildQueryString(), callable,
amazonHost, s3Bucket);
}
@Override
public Boolean get() throws InterruptedException, ExecutionException {
try {
return super.get();
} catch (ExecutionException e) {
return attemptNotFound(e);
}
}
@VisibleForTesting
Boolean attemptNotFound(ExecutionException e) throws ExecutionException {
if (e.getCause() != null && e.getCause() instanceof S3ResponseException) {
S3ResponseException responseException = (S3ResponseException) e
.getCause();
if (responseException.getResponse().getStatusCode() == 404) {
return false;
}
}
throw e;
}
@Override
public Boolean get(long l, TimeUnit timeUnit) throws InterruptedException,
ExecutionException, TimeoutException {
try {
return super.get(l, timeUnit);
} catch (ExecutionException e) {
return attemptNotFound(e);
}
}
}

44
s3/src/main/java/org/jclouds/aws/s3/commands/DeleteBucket.java
View File

@ -23,8 +23,14 @@
*/
package org.jclouds.aws.s3.commands;
import org.jclouds.aws.s3.commands.callables.DeleteBucketCallable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3ResponseException;
import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.name.Named;
@ -33,7 +39,41 @@ public class DeleteBucket extends S3FutureCommand<Boolean> {
@Inject
public DeleteBucket(@Named("jclouds.http.address") String amazonHost,
DeleteBucketCallable callable, @Assisted String s3Bucket) {
ReturnTrueIf2xx callable, @Assisted String s3Bucket) {
super("DELETE", "/", callable, amazonHost, s3Bucket);
}
@Override
public Boolean get() throws InterruptedException, ExecutionException {
try {
return super.get();
} catch (ExecutionException e) {
return attemptNotFound(e);
}
}
@VisibleForTesting
Boolean attemptNotFound(ExecutionException e) throws ExecutionException {
if (e.getCause() != null && e.getCause() instanceof S3ResponseException) {
S3ResponseException responseException = (S3ResponseException) e
.getCause();
if (responseException.getResponse().getStatusCode() == 404) {
return true;
} else if ("BucketNotEmpty".equals(responseException.getError()
.getCode())) {
return false;
}
}
throw e;
}
@Override
public Boolean get(long l, TimeUnit timeUnit) throws InterruptedException,
ExecutionException, TimeoutException {
try {
return super.get(l, timeUnit);
} catch (ExecutionException e) {
return attemptNotFound(e);
}
}
}

4
s3/src/main/java/org/jclouds/aws/s3/commands/DeleteObject.java
View File

@ -25,7 +25,7 @@ package org.jclouds.aws.s3.commands;
import static com.google.common.base.Preconditions.checkNotNull;
import org.jclouds.aws.s3.commands.callables.DeleteCallable;
import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@ -35,7 +35,7 @@ public class DeleteObject extends S3FutureCommand<Boolean> {
@Inject
public DeleteObject(@Named("jclouds.http.address") String amazonHost,
DeleteCallable callable, @Assisted("bucketName") String bucket,
ReturnTrueIf2xx callable, @Assisted("bucketName") String bucket,
@Assisted("key") String key) {
super("DELETE", "/" + checkNotNull(key), callable, amazonHost, bucket);
}

4
s3/src/main/java/org/jclouds/aws/s3/commands/GetObject.java
View File

@ -30,7 +30,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3ResponseException;
import org.jclouds.aws.s3.commands.callables.GetObjectCallable;
import org.jclouds.aws.s3.commands.callables.ParseObjectFromHeadersAndHttpContent;
import org.jclouds.aws.s3.commands.options.GetObjectOptions;
import org.jclouds.aws.s3.domain.S3Object;
@ -50,7 +50,7 @@ public class GetObject extends S3FutureCommand<S3Object> {
@Inject
public GetObject(@Named("jclouds.http.address") String amazonHost,
GetObjectCallable callable,
ParseObjectFromHeadersAndHttpContent callable,
@Assisted("bucketName") String s3Bucket,
@Assisted("key") String key, @Assisted GetObjectOptions options) {
super("GET", "/" + checkNotNull(key), callable, amazonHost, s3Bucket);

43
s3/src/main/java/org/jclouds/aws/s3/commands/HeadMetaData.java
View File

@ -25,9 +25,15 @@ package org.jclouds.aws.s3.commands;
import static com.google.common.base.Preconditions.checkNotNull;
import org.jclouds.aws.s3.commands.callables.HeadMetaDataCallable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3ResponseException;
import org.jclouds.aws.s3.commands.callables.ParseMetadataFromHeaders;
import org.jclouds.aws.s3.domain.S3Object;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.name.Named;
@ -43,9 +49,42 @@ public class HeadMetaData extends S3FutureCommand<S3Object.Metadata> {
@Inject
public HeadMetaData(@Named("jclouds.http.address") String amazonHost,
HeadMetaDataCallable callable,
ParseMetadataFromHeaders callable,
@Assisted("bucketName") String bucket, @Assisted("key") String key) {
super("HEAD", "/" + checkNotNull(key), callable, amazonHost, bucket);
callable.setKey(key);
}
@Override
public S3Object.Metadata get() throws InterruptedException,
ExecutionException {
try {
return super.get();
} catch (ExecutionException e) {
return attemptNotFound(e);
}
}
@VisibleForTesting
S3Object.Metadata attemptNotFound(ExecutionException e)
throws ExecutionException {
if (e.getCause() != null && e.getCause() instanceof S3ResponseException) {
S3ResponseException responseException = (S3ResponseException) e
.getCause();
if (responseException.getResponse().getStatusCode() == 404) {
return S3Object.Metadata.NOT_FOUND;
}
}
throw e;
}
@Override
public S3Object.Metadata get(long l, TimeUnit timeUnit)
throws InterruptedException, ExecutionException, TimeoutException {
try {
return super.get(l, timeUnit);
} catch (ExecutionException e) {
return attemptNotFound(e);
}
}
}

4
s3/src/main/java/org/jclouds/aws/s3/commands/PutBucket.java
View File

@ -23,9 +23,9 @@
*/
package org.jclouds.aws.s3.commands;
import org.jclouds.aws.s3.commands.callables.PutBucketCallable;
import org.jclouds.aws.s3.commands.options.PutBucketOptions;
import org.jclouds.http.HttpHeaders;
import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@ -41,7 +41,7 @@ public class PutBucket extends S3FutureCommand<Boolean> {
@Inject
public PutBucket(@Named("jclouds.http.address") String amazonHost,
PutBucketCallable callable, @Assisted String s3Bucket,
ReturnTrueIf2xx callable, @Assisted String s3Bucket,
@Assisted PutBucketOptions options) {
super("PUT", "/", callable, amazonHost, s3Bucket);
getRequest().getHeaders().putAll(options.buildRequestHeaders());

10
s3/src/main/java/org/jclouds/aws/s3/commands/PutObject.java
View File

@ -23,10 +23,10 @@
*/
package org.jclouds.aws.s3.commands;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.*;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.aws.s3.commands.callables.PutObjectCallable;
import org.jclouds.aws.s3.commands.callables.ParseMd5FromETagHeader;
import org.jclouds.aws.s3.commands.options.PutObjectOptions;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.http.HttpHeaders;
@ -39,10 +39,12 @@ public class PutObject extends S3FutureCommand<byte[]> {
@Inject
public PutObject(@Named("jclouds.http.address") String amazonHost,
PutObjectCallable callable, @Assisted String s3Bucket,
ParseMd5FromETagHeader callable, @Assisted String s3Bucket,
@Assisted S3Object object, @Assisted PutObjectOptions options) {
super("PUT", "/" + checkNotNull(object.getKey()), callable, amazonHost,
s3Bucket);
checkArgument(object.getMetaData().getSize() >=0,"size must be set");
getRequest().setPayload(
checkNotNull(object.getData(), "object.getContent()"));
@ -50,6 +52,7 @@ public class PutObject extends S3FutureCommand<byte[]> {
HttpHeaders.CONTENT_TYPE,
checkNotNull(object.getMetaData().getContentType(),
"object.metaData.contentType()"));
getRequest().getHeaders().put(HttpHeaders.CONTENT_LENGTH,
object.getMetaData().getSize() + "");
@ -75,5 +78,4 @@ public class PutObject extends S3FutureCommand<byte[]> {
getRequest().getHeaders().putAll(options.buildRequestHeaders());
}
}

60
s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallable.java
View File

@ -1,60 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.commands.callables;
import java.io.IOException;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.http.HttpException;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public class DeleteBucketCallable extends DeleteCallable {
public Boolean call() throws HttpException {
if (getResponse().getStatusCode() == 404) {
return true;
} else if (getResponse().getStatusCode() == 409) {
try {
if (getResponse().getContent() == null) {
throw new HttpException(
"cannot determine error as there is no content");
}
String reason = S3Utils.toStringAndClose(getResponse()
.getContent());
if (reason.indexOf("BucketNotEmpty") >= 0)
return false;
else
throw new HttpException("Error deleting bucket.\n" + reason);
} catch (IOException e) {
throw new HttpException("Error deleting bucket", e);
}
} else {
return super.call();
}
}
}

24
s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutBucketCallable.java → s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMd5FromETagHeader.java
View File

@ -23,24 +23,28 @@
*/
package org.jclouds.aws.s3.commands.callables;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3Headers;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
/**
* Returns true, if the bucket was created.
* Parses an MD5 checksum from the header {@link S3Headers#ETAG}.
*
* @author Adrian Cole
*/
public class PutBucketCallable extends
HttpFutureCommand.ResponseCallable<Boolean> {
public class ParseMd5FromETagHeader extends
HttpFutureCommand.ResponseCallable<byte[]> {
public Boolean call() throws HttpException {
if (getResponse().getStatusCode() == 200) {
return true;
} else if (getResponse().getStatusCode() == 404) {
return false;
} else {
throw new HttpException("Error checking bucket " + getResponse());
public byte[] call() throws HttpException {
checkCode();
IOUtils.closeQuietly(getResponse().getContent());
String eTag = getResponse().getFirstHeaderOrNull(S3Headers.ETAG);
if (eTag != null) {
return S3Utils.fromHexString(eTag.replaceAll("\"", ""));
}
throw new HttpException("did not receive ETag");
}
}

57
s3/src/main/java/org/jclouds/aws/s3/commands/callables/HeadMetaDataCallable.java → s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMetadataFromHeaders.java
View File

@ -23,10 +23,8 @@
*/
package org.jclouds.aws.s3.commands.callables;
import java.io.IOException;
import java.util.Map.Entry;
import org.jclouds.Utils;
import org.jclouds.aws.s3.DateService;
import org.jclouds.aws.s3.S3Headers;
import org.jclouds.aws.s3.S3Utils;
@ -38,39 +36,35 @@ import org.jclouds.http.HttpHeaders;
import com.google.inject.Inject;
/**
* This parses @{link {@link S3Object.Metadata} from http headers or returns
* {@link S3Object.Metadata#NOT_FOUND} on 404.
* This parses @{link {@link S3Object.Metadata} from http headers.
*
* @author Adrian Cole
*/
public class HeadMetaDataCallable extends
public class ParseMetadataFromHeaders extends
HttpFutureCommand.ResponseCallable<S3Object.Metadata> {
private final DateService dateParser;
private String key;
@Inject
public HeadMetaDataCallable(DateService dateParser) {
public ParseMetadataFromHeaders(DateService dateParser) {
this.dateParser = dateParser;
}
/**
* @return S3Content.NOT_FOUND, if not found.
* @throws org.jclouds.http.HttpException
* parses the http response headers to create a new
* {@link S3Object.MetaData} object.
*/
public S3Object.Metadata call() throws HttpException {
if (getResponse().getStatusCode() == 200) {
checkCode();
S3Object.Metadata metaData = new S3Object.Metadata(key);
extractUserMetadata(metaData);
addMd5(metaData);
metaData.setLastModified(dateParser
.dateTimeFromHeaderFormat(getResponse()
.getFirstHeaderOrNull(HttpHeaders.LAST_MODIFIED)));
metaData.setContentType(getResponse().getFirstHeaderOrNull(
HttpHeaders.CONTENT_TYPE));
metaData.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull(
HttpHeaders.CONTENT_LENGTH)));
parseLastModifiedOrThrowException(metaData);
setContentTypeOrThrowException(metaData);
metaData.setCacheControl(getResponse().getFirstHeaderOrNull(
HttpHeaders.CACHE_CONTROL));
metaData.setContentDisposition(getResponse().getFirstHeaderOrNull(
@ -78,18 +72,29 @@ public class HeadMetaDataCallable extends
metaData.setContentEncoding(getResponse().getFirstHeaderOrNull(
HttpHeaders.CONTENT_ENCODING));
return metaData;
} else if (getResponse().getStatusCode() == 404) {
return S3Object.Metadata.NOT_FOUND;
} else {
String reason = null;
try {
reason = Utils.toStringAndClose(getResponse().getContent());
} catch (IOException e) {
logger.error(e, "error parsing reason");
}
throw new HttpException("Error parsing object " + getResponse()
+ " reason: " + reason);
private void setContentTypeOrThrowException(S3Object.Metadata metaData)
throws HttpException {
String contentType = getResponse().getFirstHeaderOrNull(
HttpHeaders.CONTENT_TYPE);
if (contentType == null)
throw new HttpException(HttpHeaders.CONTENT_TYPE
+ " not found in headers");
else
metaData.setContentType(contentType);
}
private void parseLastModifiedOrThrowException(S3Object.Metadata metaData)
throws HttpException {
String lastModified = getResponse().getFirstHeaderOrNull(
HttpHeaders.LAST_MODIFIED);
metaData.setLastModified(dateParser
.dateTimeFromHeaderFormat(lastModified));
if (metaData.getLastModified() == null)
throw new HttpException("could not parse: "
+ HttpHeaders.LAST_MODIFIED + ": " + lastModified);
}
private void addMd5(S3Object.Metadata metaData) {

46
s3/src/main/java/org/jclouds/aws/s3/commands/callables/GetObjectCallable.java → s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseObjectFromHeadersAndHttpContent.java
View File

@ -23,51 +23,55 @@
*/
package org.jclouds.aws.s3.commands.callables;
import java.io.IOException;
import org.jclouds.Utils;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Object.Metadata;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpHeaders;
import com.google.inject.Inject;
/**
* // TODO: Adrian: Document this!
* Parses response headers and creates a new S3Object from them and the http
* content.
*
* @author Adrian Cole
*/
public class GetObjectCallable extends
public class ParseObjectFromHeadersAndHttpContent extends
HttpFutureCommand.ResponseCallable<S3Object> {
private final HeadMetaDataCallable metaDataParser;
private String key;
private final ParseMetadataFromHeaders metaDataParser;
@Inject
public GetObjectCallable(HeadMetaDataCallable metaDataParser) {
public ParseObjectFromHeadersAndHttpContent(ParseMetadataFromHeaders metaDataParser) {
this.metaDataParser = metaDataParser;
}
/**
* @return S3Content.NOT_FOUND, if not found.
* First, calls {@link ParseMetadataFromHeaders}.
*
* Then, sets the object size based on the Content-Length header and adds
* the content to the {@link S3Object} result.
*
* @throws org.jclouds.http.HttpException
*/
public S3Object call() throws HttpException {
checkCode();
metaDataParser.setResponse(getResponse());
S3Object.Metadata metaData = metaDataParser.call();
if (metaData == S3Object.Metadata.NOT_FOUND)
return S3Object.NOT_FOUND;
if (getResponse().getContent() != null) {
parseContentLengthOrThrowException(metaData);
return new S3Object(metaData, getResponse().getContent());
} else {
String reason = null;
try {
reason = Utils.toStringAndClose(getResponse().getContent());
} catch (IOException e) {
logger.error(e, "error parsing reason");
}
throw new HttpException("No content retrieving object " + key + ":"
+ getResponse() + " reason: " + reason);
}
private void parseContentLengthOrThrowException(Metadata metaData)
throws HttpException {
String contentLength = getResponse().getFirstHeaderOrNull(
HttpHeaders.CONTENT_LENGTH);
if (contentLength == null)
throw new HttpException(HttpHeaders.CONTENT_LENGTH
+ " header not present in headers: "
+ getResponse().getHeaders());
metaData.setSize(Long.parseLong(contentLength));
}
public void setKey(String key) {

65
s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutObjectCallable.java
View File

@ -1,65 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.commands.callables;
import java.io.IOException;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public class PutObjectCallable extends
HttpFutureCommand.ResponseCallable<byte []> {
public byte [] call() throws HttpException {
if (getResponse().getStatusCode() == 200) {
try {
getResponse().getContent().close();
} catch (IOException e) {
logger.error(e, "error consuming content");
}
String eTag = getResponse().getFirstHeaderOrNull("ETag");
if (eTag != null) {
return S3Utils
.fromHexString(eTag.replaceAll("\"", ""));
}
throw new HttpException("did not receive ETag");
} else {
try {
String reason = S3Utils.toStringAndClose(getResponse()
.getContent());
throw new HttpException(getResponse().getStatusCode()
+ ": Problem uploading content.\n" + reason);
} catch (IOException e) {
throw new HttpException(getResponse().getStatusCode()
+ ": Problem uploading content", e);
}
}
}
}

33
s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java
View File

@ -26,22 +26,33 @@ package org.jclouds.aws.s3.config;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.Resource;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Context;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.filters.ParseS3ErrorFromXmlContent;
import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader;
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
import org.jclouds.aws.s3.internal.GuiceS3Context;
import org.jclouds.aws.s3.internal.LiveS3Connection;
import org.jclouds.aws.s3.internal.LiveS3InputStreamMap;
import org.jclouds.aws.s3.internal.LiveS3ObjectMap;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.HttpRequestFilter;
import org.jclouds.http.HttpResponseHandler;
import org.jclouds.http.annotation.ClientErrorHandler;
import org.jclouds.http.annotation.RedirectHandler;
import org.jclouds.http.annotation.ServerErrorHandler;
import org.jclouds.logging.Logger;
import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Provides;
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryProvider;
import com.google.inject.name.Named;
/**
* // TODO: Adrian: Document this!
@ -49,6 +60,18 @@ import com.google.inject.assistedinject.FactoryProvider;
* @author Adrian Cole
*/
public class S3ContextModule extends AbstractModule {
@Resource
protected Logger logger = Logger.NULL;
@Inject
@Named(HttpConstants.PROPERTY_HTTP_ADDRESS)
String address;
@Inject
@Named(HttpConstants.PROPERTY_HTTP_PORT)
int port;
@Inject
@Named(HttpConstants.PROPERTY_HTTP_SECURE)
boolean isSecure;
@Override
protected void configure() {
@ -64,6 +87,15 @@ public class S3ContextModule extends AbstractModule {
GuiceS3Context.S3InputStreamMapFactory.class,
LiveS3InputStreamMap.class));
bind(S3Context.class).to(GuiceS3Context.class);
bind(HttpResponseHandler.class).annotatedWith(RedirectHandler.class)
.to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON);
bind(HttpResponseHandler.class).annotatedWith(ClientErrorHandler.class)
.to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON);
bind(HttpResponseHandler.class).annotatedWith(ServerErrorHandler.class)
.to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON);
requestInjection(this);
logger.info("S3 Context = %1s://%2s:%3s",
(isSecure ? "https" : "http"), address, port);
}
@Provides
@ -76,4 +108,5 @@ public class S3ContextModule extends AbstractModule {
filters.add(requestAuthorizeSignature);
return filters;
}
}

46
s3/src/main/java/org/jclouds/aws/s3/config/S3JavaUrlHttpFutureCommandClientModule.java
View File

@ -1,46 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.config;
import org.jclouds.aws.s3.internal.S3JavaUrlHttpFutureCommandClient;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.config.HttpFutureCommandClientModule;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
/**
* Configures {@link S3JavaUrlHttpFutureCommandClient}.
*
* @author Adrian Cole
*/
@HttpFutureCommandClientModule
public class S3JavaUrlHttpFutureCommandClientModule extends
JavaUrlHttpFutureCommandClientModule {
@Override
protected void bindClient() {
// note this is not threadsafe, so it cannot be singleton
bind(HttpFutureCommandClient.class).to(
S3JavaUrlHttpFutureCommandClient.class);
}
}

54
s3/src/main/java/org/jclouds/aws/s3/domain/S3Object.java
View File

@ -27,14 +27,12 @@ import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.aws.s3.S3Utils.Md5InputStreamResult;
import org.jclouds.http.ContentTypes;
import org.joda.time.DateTime;
@ -62,12 +60,12 @@ public class S3Object {
public S3Object(Metadata metaData, Object data) {
this(metaData);
this.data = data;
setData(data);
}
public S3Object(String key, Object data) {
this(key);
this.data = data;
setData(data);
}
public static class Metadata {
@ -76,7 +74,7 @@ public class S3Object {
// parsed during list, head, or get
private final String key;
private byte[] md5;
private long size = -1;
private volatile long size = -1;
// only parsed during head or get
private Multimap<String, String> userMetadata = HashMultimap.create();
@ -238,44 +236,22 @@ public class S3Object {
}
public void setData(Object data) {
this.data = data;
this.data = checkNotNull(data, "data");
if (getMetaData().getSize() == -1)
this.getMetaData().setSize(S3Utils.calculateSize(data));
}
/**
* converts String or InputStreams to byte [] type before generating an md5;
*
* @throws IOException
*/
public void generateMd5() throws IOException {
checkState(data != null,
"data must be set before calling generateMd5()");
byte[] md5;
if (data == null || data instanceof byte[]) {
md5 = S3Utils.md5((byte[]) data);
} else if (data instanceof String) {
this.data = ((String) data).getBytes();
md5 = S3Utils.md5((byte[]) data);
} else if (data instanceof File) {
InputStream i = new FileInputStream((File) data);
try {
md5 = S3Utils.md5(i);
} finally {
IOUtils.closeQuietly(i);
}
} else if (data instanceof InputStream) {
InputStream i = (InputStream) data;
try {
this.data = (IOUtils.toByteArray(i));
md5 = S3Utils.md5(i);
} finally {
IOUtils.closeQuietly(i);
}
checkState(data != null, "data");
if (data instanceof InputStream) {
Md5InputStreamResult result = S3Utils
.generateMd5Result((InputStream) data);
getMetaData().setSize(result.length);
getMetaData().setMd5(result.md5);
setData(result.data);
} else {
throw new UnsupportedOperationException("Content not supported "
+ data.getClass());
getMetaData().setMd5(S3Utils.md5(data));
}
getMetaData().setMd5(md5);
}
public Object getData() {

46
s3/src/main/java/org/jclouds/aws/s3/internal/S3JavaUrlHttpFutureCommandClient.java → s3/src/main/java/org/jclouds/aws/s3/filters/ParseS3ErrorFromXmlContent.java
View File

@ -21,40 +21,42 @@
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.internal;
package org.jclouds.aws.s3.filters;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import javax.annotation.Resource;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3ResponseException;
import org.jclouds.aws.s3.domain.S3Error;
import org.jclouds.aws.s3.xml.S3ParserFactory;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.JavaUrlHttpFutureCommandClient;
import org.jclouds.http.HttpResponseHandler;
import org.jclouds.logging.Logger;
import com.google.inject.Inject;
public class S3JavaUrlHttpFutureCommandClient extends
JavaUrlHttpFutureCommandClient {
/**
* This will parse and set an appropriate exception on the command object.
*
* @author Adrian Cole
*
*/
public class ParseS3ErrorFromXmlContent implements HttpResponseHandler {
@Resource
protected Logger logger = Logger.NULL;
private S3ParserFactory parserFactory;
private final S3ParserFactory parserFactory;
@Inject
public S3JavaUrlHttpFutureCommandClient(S3ParserFactory parserFactory,
URL target) throws MalformedURLException {
super(target);
public ParseS3ErrorFromXmlContent(S3ParserFactory parserFactory) {
this.parserFactory = parserFactory;
}
@Override
protected HttpResponse getResponse(HttpURLConnection connection)
throws IOException {
HttpResponse response = super.getResponse(connection);
public void handle(HttpFutureCommand<?> command, HttpResponse response) {
int code = response.getStatusCode();
if (code >= 300) {
InputStream errorStream = response.getContent();
@ -62,16 +64,20 @@ public class S3JavaUrlHttpFutureCommandClient extends
try {
S3Error error = parserFactory.createErrorParser().parse(
errorStream);
logger.error("received the following error from s3: %1s",
logger.trace("received the following error from s3: %1s",
error);
throw new S3ResponseException(error, response);
command.setException(new S3ResponseException(command,
response, error));
} catch (HttpException he) {
logger.error(he, "error parsing response");
logger.error(he, "error parsing response %1s", response);
} finally {
IOUtils.closeQuietly(errorStream);
}
} else {
command
.setException(new S3ResponseException(command, response));
}
}
return response;
}
}

28
s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java
View File

@ -24,8 +24,6 @@
package org.jclouds.aws.s3.filters;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@ -36,20 +34,17 @@ import org.jclouds.http.HttpException;
import org.jclouds.http.HttpHeaders;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
import org.joda.time.DateTime;
import com.google.inject.Inject;
import com.google.inject.name.Named;
public class RequestAuthorizeSignature implements HttpRequestFilter {
private static final String[] firstHeadersToSign = new String[] {
HttpHeaders.CONTENT_MD5, HttpHeaders.CONTENT_TYPE,
HttpHeaders.DATE };
HttpHeaders.CONTENT_MD5, HttpHeaders.CONTENT_TYPE, HttpHeaders.DATE };
private final String accessKey;
private final String secretKey;
private final DateService dateService;
private Map<String, DateTime> dateCache = new ConcurrentHashMap<String, DateTime>();
public static final long BILLION = 1000000000;
private final AtomicReference<String> timeStamp;
@ -80,24 +75,6 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
return timeStamp.get();
}
public DateTime dateTimeFromXMLFormat(String toParse) {
DateTime time = dateCache.get(toParse);
if (time == null) {
time = dateService.dateTimeFromXMLFormat(toParse);
dateCache.put(toParse, time);
}
return time;
}
public DateTime dateTimeFromHeaderFormat(String toParse) {
DateTime time = dateCache.get(toParse);
if (time == null) {
time = dateService.dateTimeFromHeaderFormat(toParse);
dateCache.put(toParse, time);
}
return time;
}
@Inject
public RequestAuthorizeSignature(
@Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey,
@ -149,8 +126,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
}
private void addDateHeader(HttpRequest request) {
request.getHeaders().put(HttpHeaders.DATE,
dateService.timestampAsHeaderString());
request.getHeaders().put(HttpHeaders.DATE, timestampAsHeaderString());
}
private void appendAmzHeaders(HttpRequest request, StringBuilder toSign) {

49
s3/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java
View File

@ -25,11 +25,8 @@ package org.jclouds.aws.s3.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@ -45,7 +42,6 @@ import org.jclouds.Utils;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Constants;
import org.jclouds.aws.s3.S3Map;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
@ -62,7 +58,7 @@ public abstract class BaseS3Map<T> implements Map<String, T>, S3Map {
* maximum duration of an S3 Request
*/
@Inject(optional = true)
@Named(S3Constants.PROPERTY_AWS_MAP_TIMEOUT)
@Named(S3Constants.PROPERTY_S3_MAP_TIMEOUT)
protected long requestTimeoutMilliseconds = 10000;
@Inject
@ -71,6 +67,13 @@ public abstract class BaseS3Map<T> implements Map<String, T>, S3Map {
this.bucket = checkNotNull(bucket, "bucket");
}
/**
* {@inheritDoc}
* <p/>
* This returns the number of keys in the {@link S3Bucket}
*
* @see S3Bucket#getContents()
*/
public int size() {
try {
S3Bucket bucket = refreshBucket();
@ -95,28 +98,15 @@ public abstract class BaseS3Map<T> implements Map<String, T>, S3Map {
protected byte[] getMd5(Object value) throws IOException,
FileNotFoundException, InterruptedException, ExecutionException,
TimeoutException {
byte[] md5;
if (value instanceof InputStream) {
md5 = S3Utils.md5((InputStream) value);
} else if (value instanceof byte[]) {
md5 = S3Utils.md5((byte[]) value);
} else if (value instanceof String) {
md5 = S3Utils.md5(((String) value).getBytes());
} else if (value instanceof File) {
md5 = S3Utils.md5(new FileInputStream((File) value));
} else if (value instanceof S3Object) {
S3Object object = (S3Object) value;
object = connection.getObject(bucket, object.getKey()).get(
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
if (S3Object.NOT_FOUND.equals(object))
throw new FileNotFoundException("not found: " + object.getKey());
md5 = object.getMetaData().getMd5();
S3Object object = null;
if (value instanceof S3Object) {
object = (S3Object) value;
} else {
throw new IllegalArgumentException("unsupported value type: "
+ value.getClass());
object = new S3Object("dummy", value);
}
return md5;
if (object.getMetaData().getMd5() == null)
object.generateMd5();
return object.getMetaData().getMd5();
}
protected Set<S3Object> getAllObjects() {
@ -141,13 +131,14 @@ public abstract class BaseS3Map<T> implements Map<String, T>, S3Map {
return objects;
}
/*
* (non-Javadoc)
/**
* {@inheritDoc}
*
* @see org.jclouds.aws.s3.S3ObjectMapi#containsValue(java.lang.Object)
* Note that if value is an instance of InputStream, it will be read and
* closed following this method. To reuse data from InputStreams, pass
* {@link InputStream}s inside {@link S3Object}s
*/
public boolean containsValue(Object value) {
try {
byte[] md5 = getMd5(value);
return containsMd5(md5);

16
s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3InputStreamMap.java
View File

@ -23,9 +23,7 @@
*/
package org.jclouds.aws.s3.internal;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
@ -37,7 +35,6 @@ import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.jclouds.Utils;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3InputStreamMap;
@ -151,7 +148,7 @@ public class LiveS3InputStreamMap extends BaseS3Map<InputStream> implements
try {
InputStream returnVal = containsKey(s) ? get(s) : null;
object.setData(o);
setSizeIfContentIsInputStream(object);
object.generateMd5();
connection.putObject(bucket, object).get(
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
return returnVal;
@ -189,7 +186,7 @@ public class LiveS3InputStreamMap extends BaseS3Map<InputStream> implements
for (String key : map.keySet()) {
S3Object object = new S3Object(key);
object.setData(map.get(key));
setSizeIfContentIsInputStream(object);
object.generateMd5();
puts.add(connection.putObject(bucket, object));
}
for (Future<byte[]> put : puts)
@ -202,15 +199,6 @@ public class LiveS3InputStreamMap extends BaseS3Map<InputStream> implements
}
}
private void setSizeIfContentIsInputStream(S3Object object)
throws IOException {
if (object.getData() instanceof InputStream) {
byte[] buffer = IOUtils.toByteArray((InputStream) object.getData());
object.getMetaData().setSize(buffer.length);
object.setData(new ByteArrayInputStream(buffer));
}
}
/*
* (non-Javadoc)
*

7
s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java
View File

@ -44,10 +44,10 @@ import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import org.jclouds.aws.s3.config.S3JavaUrlHttpFutureCommandClientModule;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
@ -90,6 +90,7 @@ public class S3IntegrationTest {
.getContents().size(), 1);
S3Object newObject = client.getObject(sourceBucket, key).get(10,
TimeUnit.SECONDS);
assert newObject != S3Object.NOT_FOUND;
assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING);
return newObject;
}
@ -147,7 +148,7 @@ public class S3IntegrationTest {
}
protected boolean debugEnabled() {
return true;
return false;
}
protected S3Context createS3Context(String AWSAccessKeyId,
@ -171,7 +172,7 @@ public class S3IntegrationTest {
}
protected Module createHttpModule() {
return new S3JavaUrlHttpFutureCommandClientModule();
return new JavaUrlHttpFutureCommandClientModule();
}
protected void deleteEverything() throws Exception {

4
s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java
View File

@ -126,11 +126,11 @@ public class S3ObjectMapTest extends BaseS3MapTest<S3Object> {
public void testPut() throws IOException {
S3Object object = new S3Object("one");
object.setData(IOUtils.toInputStream("apple"));
object.getMetaData().setSize("apple".getBytes().length);
object.generateMd5();
S3Object old = map.put(object.getKey(), object);
getOneReturnsAppleAndOldValueIsNull(old);
object.setData(IOUtils.toInputStream("bear"));
object.getMetaData().setSize("bear".getBytes().length);
object.generateMd5();
S3Object apple = map.put(object.getKey(), object);
getOneReturnsBearAndOldValueIsApple(apple);
}

22
s3/src/test/java/org/jclouds/aws/s3/commands/PutObjectIntegrationTest.java
View File

@ -30,6 +30,7 @@ import static org.testng.Assert.assertNotNull;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.concurrent.TimeUnit;
@ -77,6 +78,9 @@ public class PutObjectIntegrationTest extends S3IntegrationTest {
S3Object object = new S3Object(key);
object.getMetaData().setContentType(type);
object.setData(content);
if (content instanceof InputStream) {
object.generateMd5();
}
assertNotNull(client.putObject(bucketName, object).get(10,
TimeUnit.SECONDS));
object = client.getObject(bucketName, object.getKey()).get(10,
@ -105,19 +109,19 @@ public class PutObjectIntegrationTest extends S3IntegrationTest {
object.getMetaData().setMd5(S3Utils.md5(TEST_STRING.getBytes()));
addObjectToBucket(bucketName, object);
object = validateContent(bucketName, key);
S3Object newObject = validateContent(bucketName, key);
// TODO.. why does this come back as binary/octetstring
assertEquals(object.getMetaData().getContentType(),
assertEquals(newObject.getMetaData().getContentType(),
"binary/octet-stream");
assertEquals(object.getMetaData().getContentEncoding(), "x-compress");
assertEquals(object.getMetaData().getContentDisposition(),
assertEquals(newObject.getMetaData().getContentEncoding(), "x-compress");
assertEquals(newObject.getMetaData().getContentDisposition(),
"attachment; filename=hello.txt");
assertEquals(object.getMetaData().getCacheControl(), "no-cache");
assertEquals(object.getMetaData().getSize(), TEST_STRING.length());
assertEquals(object.getMetaData().getUserMetadata().values().iterator()
.next(), "powderpuff");
assertEquals(object.getMetaData().getMd5(), S3Utils.md5(TEST_STRING
assertEquals(newObject.getMetaData().getCacheControl(), "no-cache");
assertEquals(newObject.getMetaData().getSize(), TEST_STRING.length());
assertEquals(newObject.getMetaData().getUserMetadata().values()
.iterator().next(), "powderpuff");
assertEquals(newObject.getMetaData().getMd5(), S3Utils.md5(TEST_STRING
.getBytes()));
}

16
s3/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java
View File

@ -118,19 +118,21 @@ public class S3CommandFactoryTest {
void testCreatePutObject() {
S3Object.Metadata metaData = createMock(S3Object.Metadata.class);
S3Object object = new S3Object(metaData);
object.setData("<a></a>");
expect(metaData.getSize()).andReturn(4L).atLeastOnce();
expect(metaData.getKey()).andReturn("rawr");
expect(metaData.getContentType()).andReturn("text/xml").atLeastOnce();
expect(metaData.getSize()).andReturn(4L);
expect(metaData.getCacheControl()).andReturn("no-cache").atLeastOnce();
expect(metaData.getContentDisposition()).andReturn("disposition").atLeastOnce();
expect(metaData.getContentEncoding()).andReturn("encoding").atLeastOnce();
expect(metaData.getMd5()).andReturn("encoding".getBytes()).atLeastOnce();
Multimap<String,String> userMdata = HashMultimap.create();
expect(metaData.getContentDisposition()).andReturn("disposition")
.atLeastOnce();
expect(metaData.getContentEncoding()).andReturn("encoding")
.atLeastOnce();
expect(metaData.getMd5()).andReturn("encoding".getBytes())
.atLeastOnce();
Multimap<String, String> userMdata = HashMultimap.create();
expect(metaData.getUserMetadata()).andReturn(userMdata).atLeastOnce();
replay(metaData);
object.setData("<a></a>");
assert commandFactory.createPutObject("test", object,
PutObjectOptions.NONE) != null;

137
s3/src/test/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallableTest.java
View File

@ -23,85 +23,70 @@
*/
package org.jclouds.aws.s3.commands.callables;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import static org.easymock.classextension.EasyMock.verify;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
public class DeleteBucketCallableTest {
private HttpFutureCommand.ResponseCallable<Boolean> callable = null;
@BeforeMethod
void setUp() {
callable = new DeleteBucketCallable();
}
@AfterMethod
void tearDown() {
callable = null;
}
@Test(expectedExceptions = HttpException.class)
public void testExceptionWhenNoContentOn409() throws Exception {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(409).atLeastOnce();
expect(response.getContent()).andReturn(null);
replay(response);
callable.setResponse(response);
callable.call();
}
@Test
public void testExceptionWhenIOExceptionOn409() throws ExecutionException,
InterruptedException, TimeoutException, IOException {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(409).atLeastOnce();
RuntimeException exception = new RuntimeException("bad");
expect(response.getContent()).andThrow(exception);
replay(response);
callable.setResponse(response);
try {
callable.call();
} catch (Exception e) {
assert e.equals(exception);
}
verify(response);
}
@Test
public void testFalseWhenBucketNotEmptyOn409() throws Exception {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(409).atLeastOnce();
expect(response.getContent()).andReturn(
IOUtils.toInputStream("BucketNotEmpty")).atLeastOnce();
replay(response);
callable.setResponse(response);
assert !callable.call().booleanValue();
verify(response);
}
@Test
public void testResponseOk() throws Exception {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(204).atLeastOnce();
replay(response);
callable.setResponse(response);
assertEquals(callable.call(), new Boolean(true));
verify(response);
}
// private HttpFutureCommand.ResponseCallable<Boolean> callable = null;
//
// @BeforeMethod
// void setUp() {
// callable = new DeleteBucketCallable();
// }
//
// @AfterMethod
// void tearDown() {
// callable = null;
// }
//
// @Test(expectedExceptions = HttpException.class)
// public void testExceptionWhenNoContentOn409() throws Exception {
// HttpResponse response = createMock(HttpResponse.class);
// expect(response.getStatusCode()).andReturn(409).atLeastOnce();
// expect(response.getContent()).andReturn(null);
// replay(response);
// callable.setResponse(response);
// callable.call();
// }
//
// @Test
// public void testExceptionWhenIOExceptionOn409() throws
// ExecutionException,
// InterruptedException, TimeoutException, IOException {
// HttpResponse response = createMock(HttpResponse.class);
// expect(response.getStatusCode()).andReturn(409).atLeastOnce();
// RuntimeException exception = new RuntimeException("bad");
// expect(response.getContent()).andThrow(exception);
// replay(response);
// callable.setResponse(response);
// try {
// callable.call();
// } catch (Exception e) {
// assert e.equals(exception);
// }
// verify(response);
// }
//
// @Test
// public void testFalseWhenBucketNotEmptyOn409() throws Exception {
// HttpResponse response = createMock(HttpResponse.class);
// expect(response.getStatusCode()).andReturn(409).atLeastOnce();
// expect(response.getContent()).andReturn(
// IOUtils.toInputStream("BucketNotEmpty")).atLeastOnce();
// replay(response);
// callable.setResponse(response);
// assert !callable.call().booleanValue();
// verify(response);
// }
//
// @Test
// public void testResponseOk() throws Exception {
// HttpResponse response = createMock(HttpResponse.class);
// expect(response.getStatusCode()).andReturn(204).atLeastOnce();
// replay(response);
// callable.setResponse(response);
// assertEquals(callable.call(), new Boolean(true));
// verify(response);
// }
}

13
samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java
View File

@ -80,8 +80,17 @@ public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
assert string.indexOf("Hello World!") >= 0 : string;
}
@Test(invocationCount = 50, enabled = true, threadPoolSize = 10)
public void testGuiceJCloudsServed() throws InterruptedException,
@Test(invocationCount = 5, enabled = true)
public void testGuiceJCloudsSerial() throws InterruptedException,
IOException {
URL gurl = new URL(url, "/guice/listbuckets.s3");
InputStream i = gurl.openStream();
String string = IOUtils.toString(i);
assert string.indexOf("List") >= 0 : string;
}
@Test(invocationCount = 50, enabled = false, threadPoolSize = 10)
public void testGuiceJCloudsParallel() throws InterruptedException,
IOException {
URL gurl = new URL(url, "/guice/listbuckets.s3");
InputStream i = gurl.openStream();

1
samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java
View File

@ -75,7 +75,6 @@ public class GuiceServletConfig extends GuiceServletContextListener {
} finally {
IOUtils.closeQuietly(input);
}
props.setProperty("jclouds.http.sax.debug", "true");
return props;
}