refactored out s3nio module and centralized http handling code

git-svn-id: http://jclouds.googlecode.com/svn/trunk@485 3d8758e0-26b5-11de-8745-db77d3ebf521

core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPool.java

@@ -89,10 +89,10 @@ public abstract class FutureCommandConnectionPool<C, O extends FutureCommand<?,
 		.debug(
 			"%1s - attempting to acquire connection; %d currently available",
 			this, available.size());
-	C conn = available.poll(1, TimeUnit.SECONDS);
+	C conn = available.poll(5, TimeUnit.SECONDS);
 	if (conn == null)
 	    throw new TimeoutException(
-		    "could not obtain a pooled connection within 1 seconds");
+		    "could not obtain a pooled connection within 5 seconds");
 
 	logger.trace("%1s - %2d - aquired", conn, conn.hashCode());
 	if (connectionValid(conn)) {

core/src/main/java/org/jclouds/http/BaseHttpFutureCommandClient.java

@@ -0,0 +1,87 @@
+/**
+ *
+ * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+package org.jclouds.http;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.List;
+
+import javax.annotation.Resource;
+
+import org.jclouds.http.annotation.ClientErrorHandler;
+import org.jclouds.http.annotation.RedirectHandler;
+import org.jclouds.http.annotation.ServerErrorHandler;
+import org.jclouds.logging.Logger;
+
+import com.google.inject.Inject;
+
+public abstract class BaseHttpFutureCommandClient  implements HttpFutureCommandClient {
+
+    protected final URL target;
+    
+    @Resource
+    protected Logger logger = Logger.NULL;
+    
+    @Inject(optional = true)
+    protected List<HttpRequestFilter> requestFilters = Collections.emptyList();
+    @RedirectHandler
+    @Inject(optional = true)
+    protected HttpResponseHandler redirectHandler = new CloseContentAndSetExceptionHandler();
+    @ClientErrorHandler
+    @Inject(optional = true)
+    protected HttpResponseHandler clientErrorHandler = new CloseContentAndSetExceptionHandler();
+    @ServerErrorHandler
+    @Inject(optional = true)
+    protected HttpResponseHandler serverErrorHandler = new CloseContentAndSetExceptionHandler();
+
+    @Inject
+    public BaseHttpFutureCommandClient(URL target) {
+	this.target = target;
+    }
+    
+    protected boolean isRetryable(HttpFutureCommand<?> command,
+	    HttpResponse response) {
+	int code = response.getStatusCode();
+	if (command.getRequest().isReplayable() && code >= 500) {
+	    logger.info("resubmitting command: %1s", command);
+	    return true;
+	}
+	return false;
+    }
+    
+    protected void handleResponse(HttpFutureCommand<?> command, HttpResponse response) {
+        int code = response.getStatusCode();
+        if (code >= 500) {
+        this.serverErrorHandler.handle(command, response);
+        } else if (code >= 400 && code < 500) {
+        this.clientErrorHandler.handle(command, response);
+        } else if (code >= 300 && code < 400) {
+        this.redirectHandler.handle(command, response);
+        } else {
+        command.getResponseFuture().setResponse(response);
+        command.getResponseFuture().run();
+        }
+    }
+
+}

core/src/main/java/org/jclouds/http/CloseContentAndSetExceptionHandler.java

@@ -0,0 +1,42 @@
+/**
+ *
+ * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+package org.jclouds.http;
+
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+public class CloseContentAndSetExceptionHandler implements HttpResponseHandler {
+
+    public void handle(HttpFutureCommand<?> command, HttpResponse response) {
+	String message = String.format("Command: %2s failed; response: %1s",
+		response, command);
+	command.setException(new IOException(message));
+	IOUtils.closeQuietly(response.getContent());
+    }
+}

core/src/main/java/org/jclouds/http/HttpFutureCommand.java

@@ -27,6 +27,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 
 import javax.annotation.Resource;
 
+import org.apache.commons.io.IOUtils;
 import org.jclouds.command.FutureCommand;
 import org.jclouds.logging.Logger;
 
@@ -64,6 +65,14 @@ public class HttpFutureCommand<T> extends
 	@Resource
 	protected Logger logger = Logger.NULL;
 
+	public void checkCode() {
+	    int code = getResponse().getStatusCode();
+	    if (code >= 300){
+		IOUtils.closeQuietly(getResponse().getContent());
+		throw new IllegalStateException("incorrect code for this operation: "+getResponse());
+	    }
+	}
+
 	private HttpResponse response;
 
 	public HttpResponse getResponse() {

core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java

@@ -23,22 +23,13 @@
  */
 package org.jclouds.http;
 
-import java.util.List;
-
 import org.jclouds.command.FutureCommandClient;
 
-import com.google.inject.Inject;
-
 /**
  * // TODO: Adrian: Document this!
  * 
  * @author Adrian Cole
  */
-public interface HttpFutureCommandClient
+public interface HttpFutureCommandClient extends
-	extends FutureCommandClient<HttpFutureCommand<?>> {
+	FutureCommandClient<HttpFutureCommand<?>> {
-    List<HttpRequestFilter> getRequestFilters();
-
-    @Inject
-    void setRequestFilters(List<HttpRequestFilter> requestFilters);
-
 }

core/src/main/java/org/jclouds/http/HttpRequestFilter.java

@@ -30,5 +30,4 @@ package org.jclouds.http;
  */
 public interface HttpRequestFilter {
     void filter(HttpRequest request) throws HttpException;
-
 }

core/src/main/java/org/jclouds/http/HttpResponseHandler.java

@@ -21,23 +21,18 @@
  * under the License.
  * ====================================================================
  */
-package org.jclouds.aws.s3.commands.callables;
+package org.jclouds.http;
-
-import org.jclouds.http.HttpException;
-import org.jclouds.http.HttpFutureCommand;
 
 /**
  * // TODO: Adrian: Document this!
  * 
  * @author Adrian Cole
  */
-public class DeleteCallable extends HttpFutureCommand.ResponseCallable<Boolean> {
+public interface HttpResponseHandler {
-
+    public static final HttpResponseHandler NOOP = new HttpResponseHandler() {
-    public Boolean call() throws HttpException {
+	public void handle(HttpFutureCommand<?> command, HttpResponse response) {
-	if (getResponse().getStatusCode() == 204) {
-	    return true;
-	} else {
-	    throw new HttpException("Error deleting bucket " + getResponse());
 	}
-    }
+    };
-}
+
+    void handle(HttpFutureCommand<?> command, HttpResponse response);
+}

core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java

@@ -32,51 +32,33 @@ import java.io.OutputStreamWriter;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.Collections;
-import java.util.List;
-
-import javax.annotation.Resource;
 
 import org.apache.commons.io.IOUtils;
-import org.jclouds.logging.Logger;
 
 import com.google.inject.Inject;
 
 /**
- * // TODO: Adrian: Document this!
+ * Basic implementation of a {@link HttpFutureCommandClient}.
  * 
  * @author Adrian Cole
  */
-public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
+public class JavaUrlHttpFutureCommandClient extends BaseHttpFutureCommandClient {
-    private URL target;
-    private List<HttpRequestFilter> requestFilters = Collections.emptyList();
-    @Resource
-    protected Logger logger = Logger.NULL;
-
-    public List<HttpRequestFilter> getRequestFilters() {
-	return requestFilters;
-    }
-
-    @Inject(optional = true)
-    public void setRequestFilters(List<HttpRequestFilter> requestFilters) {
-	this.requestFilters = requestFilters;
-    }
 
     @Inject
     public JavaUrlHttpFutureCommandClient(URL target)
 	    throws MalformedURLException {
-	this.target = target;
+	super(target);
     }
 
     public void submit(HttpFutureCommand<?> command) {
 	HttpRequest request = command.getRequest();
 	HttpURLConnection connection = null;
 	try {
-	    for (HttpRequestFilter filter : getRequestFilters()) {
-		filter.filter(request);
-	    }
 	    HttpResponse response = null;
 	    for (;;) {
+		for (HttpRequestFilter filter : requestFilters) {
+		    filter.filter(request);
+		}
 		logger.trace("%1s - converting request %2s", target, request);
 		connection = openJavaConnection(request);
 		logger
@@ -84,15 +66,11 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
 				connection);
 		response = getResponse(connection);
 		logger.trace("%1s - received response %2s", target, response);
-		if (command.getRequest().isReplayable()
+		if (isRetryable(command, response))
-			&& response.getStatusCode() >= 500) {
-		    logger.info("resubmitting command: %1s", command);
 		    continue;
-		}
 		break;
 	    }
-	    command.getResponseFuture().setResponse(response);
+	    handleResponse(command, response);
-	    command.getResponseFuture().run();
 	} catch (Exception e) {
 	    command.setException(e);
 	} finally {

core/src/main/java/org/jclouds/http/annotation/ClientErrorHandler.java

@@ -0,0 +1,44 @@
+/**
+ *
+ * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+package org.jclouds.http.annotation;
+
+import com.google.inject.BindingAnnotation;
+import java.lang.annotation.Target;
+import java.lang.annotation.Retention;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+
+/**
+ * Implies that the object can address {@link HttpResponse}s that contain status
+ * code 4xx.
+ * 
+ * @author Adrian Cole
+ */
+@BindingAnnotation
+@Target( { FIELD, PARAMETER, METHOD })
+@Retention(RUNTIME)
+public @interface ClientErrorHandler {
+}

core/src/main/java/org/jclouds/http/annotation/RedirectHandler.java

@@ -0,0 +1,44 @@
+/**
+ *
+ * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+package org.jclouds.http.annotation;
+
+import com.google.inject.BindingAnnotation;
+import java.lang.annotation.Target;
+import java.lang.annotation.Retention;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+
+/**
+ * Implies that the object can address {@link HttpResponse}s that contain status
+ * code 3xx.
+ * 
+ * @author Adrian Cole
+ */
+@BindingAnnotation
+@Target( { FIELD, PARAMETER, METHOD })
+@Retention(RUNTIME)
+public @interface RedirectHandler {
+}

core/src/main/java/org/jclouds/http/annotation/ServerErrorHandler.java

@@ -0,0 +1,44 @@
+/**
+ *
+ * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+package org.jclouds.http.annotation;
+
+import com.google.inject.BindingAnnotation;
+import java.lang.annotation.Target;
+import java.lang.annotation.Retention;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+
+/**
+ * Implies that the object can address {@link HttpResponse}s that contain status
+ * code 5xx.
+ * 
+ * @author Adrian Cole
+ */
+@BindingAnnotation
+@Target( { FIELD, PARAMETER, METHOD })
+@Retention(RUNTIME)
+public @interface ServerErrorHandler {
+}

core/src/main/java/org/jclouds/http/commands/Head.java

@@ -26,7 +26,7 @@ package org.jclouds.http.commands;
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
 import org.jclouds.http.HttpFutureCommand;
-import org.jclouds.http.commands.callables.ReturnTrueIf200;
+import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
 
 /**
  * // TODO: Adrian: Document this!
@@ -36,7 +36,7 @@ import org.jclouds.http.commands.callables.ReturnTrueIf200;
 public class Head extends HttpFutureCommand<Boolean> {
 
     @Inject
-    public Head(ReturnTrueIf200 callable, @Assisted String uri) {
+    public Head(ReturnTrueIf2xx callable, @Assisted String uri) {
         super("HEAD", uri, callable);
     }
 }

core/src/main/java/org/jclouds/http/commands/callables/ReturnStringIf200.java

@@ -46,11 +46,8 @@ public class ReturnStringIf200 extends
     }
 
     public String call() throws HttpException {
-	int code = getResponse().getStatusCode();
+	checkCode();
-	if (code >= 400 && code < 500) {
+	if (getResponse().getStatusCode() == 200) {
-	    throw new HttpException(String.format("Content not found - %1s",
-		    getResponse()));
-	} else if (code == 200) {
 	    InputStream entity = getResponse().getContent();
 	    if (entity == null)
 		throw new HttpException("no content");

core/src/main/java/org/jclouds/http/commands/callables/ReturnTrueIf2xx.java

@@ -23,31 +23,28 @@
  */
 package org.jclouds.http.commands.callables;
 
+import org.apache.commons.io.IOUtils;
 import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpFutureCommand;
 
 import com.google.inject.Inject;
 
 /**
- * // TODO: Adrian: Document this!
+ * Simply returns true when the http response code is in the range 200-299.
  * 
  * @author Adrian Cole
  */
-public class ReturnTrueIf200 extends
+public class ReturnTrueIf2xx extends
 	HttpFutureCommand.ResponseCallable<Boolean> {
 
     @Inject
-    public ReturnTrueIf200() {
+    public ReturnTrueIf2xx() {
 	super();
     }
 
     public Boolean call() throws HttpException {
-	if (getResponse().getStatusCode() == 200) {
+	checkCode();
-	    return true;
+	IOUtils.closeQuietly(getResponse().getContent());
-	} else if (getResponse().getStatusCode() == 404) {
+	return true;
-	    return false;
-	} else {
-	    throw new HttpException("Error checking bucket " + getResponse());
-	}
     }
 }

core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java

@@ -68,7 +68,7 @@ import com.google.inject.name.Names;
  * 
  * @author Adrian Cole
  */
-@Test(threadPoolSize = 100)
+@Test(threadPoolSize = 10)
 public abstract class BaseHttpFutureCommandClientTest {
     protected static final String XML = "<foo><bar>whoppers</bar></foo>";
     protected Server server = null;
@@ -158,9 +158,9 @@ public abstract class BaseHttpFutureCommandClientTest {
 	server.stop();
     }
 
-    @Test(invocationCount = 500, timeOut = 1500)
+    @Test(invocationCount = 50, timeOut = 1500)
-    public void testRequestFilter() throws MalformedURLException, ExecutionException,
+    public void testRequestFilter() throws MalformedURLException,
-	    InterruptedException, TimeoutException {
+	    ExecutionException, InterruptedException, TimeoutException {
 	GetString get = factory.createGetString("/");
 	get.getRequest().getHeaders().put("filterme", "filterme");
 	client.submit(get);
@@ -169,7 +169,7 @@ public abstract class BaseHttpFutureCommandClientTest {
 			TimeUnit.SECONDS));
     }
 
-    @Test(invocationCount = 500, timeOut = 1500)
+    @Test(invocationCount = 50, timeOut = 1500)
     public void testGetStringWithHeader() throws MalformedURLException,
 	    ExecutionException, InterruptedException, TimeoutException {
 	GetString get = factory.createGetString("/");
@@ -180,9 +180,9 @@ public abstract class BaseHttpFutureCommandClientTest {
 			TimeUnit.SECONDS));
     }
 
-    @Test(invocationCount = 500, timeOut = 1500)
+    @Test(invocationCount = 50, timeOut = 1500)
-    public void testGetString() throws MalformedURLException, ExecutionException,
+    public void testGetString() throws MalformedURLException,
-	    InterruptedException, TimeoutException {
+	    ExecutionException, InterruptedException, TimeoutException {
 	GetString get = factory.createGetString("/");
 	assert get != null;
 	client.submit(get);
@@ -191,7 +191,7 @@ public abstract class BaseHttpFutureCommandClientTest {
 			TimeUnit.SECONDS));
     }
 
-    @Test(invocationCount = 500, timeOut = 1500)
+    @Test(invocationCount = 50, timeOut = 1500)
     public void testHead() throws MalformedURLException, ExecutionException,
 	    InterruptedException, TimeoutException {
 	Head head = factory.createHead("/");
@@ -200,9 +200,9 @@ public abstract class BaseHttpFutureCommandClientTest {
 	assert head.get(10, TimeUnit.SECONDS);
     }
 
-    @Test(invocationCount = 500, timeOut = 1500)
+    @Test(invocationCount = 50, timeOut = 1500)
-    public void testGetAndParseSax() throws MalformedURLException, ExecutionException,
+    public void testGetAndParseSax() throws MalformedURLException,
-	    InterruptedException, TimeoutException {
+	    ExecutionException, InterruptedException, TimeoutException {
 	GetAndParseSax<?> getAndParseSax = factory.createGetAndParseSax("/",
 		new ParseSax.HandlerWithResult<String>() {
 		    @Override

core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java

@@ -58,7 +58,7 @@ public class ReturnStringIf200Test {
     public void testExceptionWhenNoContentOn200() throws ExecutionException,
 	    InterruptedException, TimeoutException, IOException {
 	HttpResponse response = createMock(HttpResponse.class);
-	expect(response.getStatusCode()).andReturn(200);
+	expect(response.getStatusCode()).andReturn(200).atLeastOnce();
 	expect(response.getContent()).andReturn(null);
 	replay(response);
 	callable.setResponse(response);
@@ -74,7 +74,7 @@ public class ReturnStringIf200Test {
     public void testExceptionWhenIOExceptionOn200() throws ExecutionException,
 	    InterruptedException, TimeoutException, IOException {
 	HttpResponse response = createMock(HttpResponse.class);
-	expect(response.getStatusCode()).andReturn(200);
+	expect(response.getStatusCode()).andReturn(200).atLeastOnce();
 	RuntimeException exception = new RuntimeException("bad");
 	expect(response.getContent()).andThrow(exception);
 	replay(response);
@@ -90,7 +90,7 @@ public class ReturnStringIf200Test {
     @Test
     public void testResponseOk() throws Exception {
 	HttpResponse response = createMock(HttpResponse.class);
-	expect(response.getStatusCode()).andReturn(200);
+	expect(response.getStatusCode()).andReturn(200).atLeastOnce();
 	expect(response.getContent()).andReturn(IOUtils.toInputStream("hello"));
 	replay(response);
 	callable.setResponse(response);

extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java

@@ -45,20 +45,26 @@ public class HttpNioUtils {
 	    HttpRequest object) {
 	BasicHttpEntityEnclosingRequest apacheRequest = new BasicHttpEntityEnclosingRequest(
 		object.getMethod(), object.getUri(), HttpVersion.HTTP_1_1);
+
+	Object content = object.getPayload();
+
+	// Since we may remove headers, ensure they are added to the apache
+	// request after this block
+	if (content != null) {
+	    long contentLength = Long.parseLong(object
+		    .getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH));
+	    object.getHeaders().removeAll(HttpHeaders.CONTENT_LENGTH);
+	    String contentType = object
+		    .getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE);
+	    object.getHeaders().removeAll(HttpHeaders.CONTENT_TYPE);
+	    addEntityForContent(apacheRequest, content, contentType,
+		    contentLength);
+	}
+
 	for (String header : object.getHeaders().keySet()) {
 	    for (String value : object.getHeaders().get(header))
 		apacheRequest.addHeader(header, value);
 	}
-	Object content = object.getPayload();
-	if (content != null) {
-	    addEntityForContent(
-		    apacheRequest,
-		    content,
-		    object.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE),
-		    Long
-			    .parseLong(object
-				    .getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH)));
-	}
 	return apacheRequest;
     }
 

extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandExecutionHandler.java

@@ -35,8 +35,13 @@ import org.apache.http.HttpResponse;
 import org.apache.http.nio.entity.ConsumingNHttpEntity;
 import org.apache.http.nio.protocol.NHttpRequestExecutionHandler;
 import org.apache.http.protocol.HttpContext;
+import org.jclouds.http.CloseContentAndSetExceptionHandler;
 import org.jclouds.http.HttpFutureCommand;
 import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponseHandler;
+import org.jclouds.http.annotation.ClientErrorHandler;
+import org.jclouds.http.annotation.RedirectHandler;
+import org.jclouds.http.annotation.ServerErrorHandler;
 import org.jclouds.http.httpnio.HttpNioUtils;
 import org.jclouds.logging.Logger;
 
@@ -55,6 +60,18 @@ public class HttpNioFutureCommandExecutionHandler implements
     private final ConsumingNHttpEntityFactory entityFactory;
     private final BlockingQueue<HttpFutureCommand<?>> commandQueue;
 
+    @RedirectHandler
+    @Inject(optional = true)
+    private HttpResponseHandler redirectHandler = new CloseContentAndSetExceptionHandler();
+
+    @ClientErrorHandler
+    @Inject(optional = true)
+    private HttpResponseHandler clientErrorHandler = new CloseContentAndSetExceptionHandler();
+
+    @ServerErrorHandler
+    @Inject(optional = true)
+    private HttpResponseHandler serverErrorHandler = new CloseContentAndSetExceptionHandler();
+
     public interface ConsumingNHttpEntityFactory {
 	public ConsumingNHttpEntity create(HttpEntity httpEntity);
     }
@@ -88,27 +105,29 @@ public class HttpNioFutureCommandExecutionHandler implements
 	return entityFactory.create(response.getEntity());
     }
 
-    public void handleResponse(HttpResponse response, HttpContext context)
+    public void handleResponse(HttpResponse apacheResponse, HttpContext context)
 	    throws IOException {
 	HttpNioFutureCommandConnectionHandle handle = (HttpNioFutureCommandConnectionHandle) context
 		.removeAttribute("command-handle");
 	if (handle != null) {
 	    try {
 		HttpFutureCommand<?> command = handle.getCommand();
-		int code = response.getStatusLine().getStatusCode();
+		org.jclouds.http.HttpResponse response = HttpNioUtils
-		// normal codes for rest commands
+			.convertToJavaCloudsResponse(apacheResponse);
-		if ((code >= 200 && code < 300) || code == 404) {
+
-		    processResponse(response, command);
+		int code = response.getStatusCode();
-		} else {
+		if (code >= 500) {
-		    if (isRetryable(response)) {
+		    if (isRetryable(command)) {
-			attemptReplay(command);
+			commandQueue.add(command);
 		    } else {
-			String message = String
+			this.serverErrorHandler.handle(command, response);
-				.format(
-					"response is not retryable: %1s;  Command: %2s failed",
-					response.getStatusLine(), command);
-			command.setException(new IOException(message));
 		    }
+		} else if (code >= 400 && code < 500) {
+		    this.clientErrorHandler.handle(command, response);
+		} else if (code >= 300 && code < 400) {
+		    this.redirectHandler.handle(command, response);
+		} else {
+		    processResponse(response, command);
 		}
 	    } finally {
 		releaseConnectionToPool(handle);
@@ -119,18 +138,12 @@ public class HttpNioFutureCommandExecutionHandler implements
 	}
     }
 
-    protected void attemptReplay(HttpFutureCommand<?> command) {
+    protected boolean isRetryable(HttpFutureCommand<?> command) {
-	if (command.getRequest().isReplayable())
+	if (command.getRequest().isReplayable()) {
-	    commandQueue.add(command);
+	    logger.info("resubmitting command: %1s", command);
-	else
+	    return true;
-	    command.setException(new IOException(String.format(
+	}
-		    "%1s: command failed and request is not replayable: %2s",
+	return false;
-		    command, command.getRequest())));
-    }
-
-    protected boolean isRetryable(HttpResponse response) throws IOException {
-	int code = response.getStatusLine().getStatusCode();
-	return code == 500 || code == 503;
     }
 
     protected void releaseConnectionToPool(
@@ -142,10 +155,8 @@ public class HttpNioFutureCommandExecutionHandler implements
 	}
     }
 
-    protected void processResponse(HttpResponse apacheResponse,
+    protected void processResponse(org.jclouds.http.HttpResponse response,
 	    HttpFutureCommand<?> command) throws IOException {
-	org.jclouds.http.HttpResponse response = HttpNioUtils
-		.convertToJavaCloudsResponse(apacheResponse);
 	command.getResponseFuture().setResponse(response);
 	logger.trace("submitting response task %1s", command
 		.getResponseFuture());

extensions/s3nio/pom.xml

@@ -1,93 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-   $HeadURL$
-   $Revision$
-   $Date$
-
-   Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
-
-   ====================================================================
-   Licensed to the Apache Software Foundation (ASF) under one
-   or more contributor license agreements.  See the NOTICE file
-   distributed with this work for additional information
-   regarding copyright ownership.  The ASF licenses this file
-   to you under the Apache License, Version 2.0 (the
-   "License"); you may not use this file except in compliance
-   with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0.html 
-
-   Unless required by applicable law or agreed to in writing,
-   software distributed under the License is distributed on an
-   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-   KIND, either express or implied.  See the License for the
-   specific language governing permissions and limitations
-   under the License.
-   ====================================================================
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-    <parent>
-        <groupId>org.jclouds</groupId>
-        <artifactId>jclouds-project</artifactId>
-        <version>1.0-SNAPSHOT</version>
-        <relativePath>../../project/pom.xml</relativePath>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>org.jclouds</groupId>
-    <artifactId>jclouds-s3nio</artifactId>
-    <name>Connection pooling NIO connection for S3</name>
-    <packaging>jar</packaging>
-    <description>Connection pooling NIO connection for S3</description>
-
-    <scm>
-        <connection>scm:svn:http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</connection>
-        <developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk/extensions/s3nio</developerConnection>
-        <url>http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</url>
-    </scm>
-
-    <properties>
-        <jclouds.aws.accesskeyid></jclouds.aws.accesskeyid>
-        <jclouds.aws.secretaccesskey></jclouds.aws.secretaccesskey>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>${project.groupId}</groupId>
-            <artifactId>jclouds-httpnio</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>${project.groupId}</groupId>
-            <artifactId>jclouds-s3</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>${project.groupId}</groupId>
-            <artifactId>jclouds-s3</artifactId>
-            <version>${project.version}</version>
-            <type>test-jar</type>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <configuration>
-                    <systemProperties>
-                        <property>
-                            <name>jclouds.aws.accesskeyid</name>
-                            <value>${jclouds.aws.accesskeyid}</value>
-                        </property>
-                        <property>
-                            <name>jclouds.aws.secretaccesskey</name>
-                            <value>${jclouds.aws.secretaccesskey}</value>
-                        </property>
-                    </systemProperties>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-</project>

extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandler.java

@@ -1,91 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.nio;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.nio.entity.NStringEntity;
-import org.jclouds.http.HttpFutureCommand;
-import org.jclouds.http.httpnio.pool.HttpNioFutureCommandExecutionHandler;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-/**
- * // TODO: Adrian: Document this!
- * 
- * @author Adrian Cole
- */
-@Singleton
-public class S3HttpNioFutureCommandExecutionHandler extends
-	HttpNioFutureCommandExecutionHandler {
-
-    @Inject
-    public S3HttpNioFutureCommandExecutionHandler(
-	    ConsumingNHttpEntityFactory entityFactory,
-	    ExecutorService executor,
-	    BlockingQueue<HttpFutureCommand<?>> commandQueue) {
-	super(entityFactory, executor, commandQueue);
-    }
-
-    @Override
-    protected boolean isRetryable(HttpResponse response) throws IOException {
-	if (super.isRetryable(response))
-	    return true;
-	int code = response.getStatusLine().getStatusCode();
-	if (code == 409) {
-	    return true;
-	} else if (code == 400) {
-	    if (response.getEntity() != null) {
-		InputStream input = response.getEntity().getContent();
-		if (input != null) {
-		    String reason = null;
-		    try {
-			reason = IOUtils.toString(input);
-		    } finally {
-			IOUtils.closeQuietly(input);
-		    }
-		    if (reason != null) {
-			try {
-			    if (reason.indexOf("RequestTime") >= 0)
-				return true;
-			} finally {
-			    IOUtils.closeQuietly(input);
-			    response.setEntity(new NStringEntity(reason));
-			}
-		    }
-
-		}
-
-	    }
-	}
-	return false;
-    }
-
-}

extensions/s3nio/src/main/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModule.java

@@ -1,51 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.nio.config;
-
-import org.apache.http.nio.protocol.NHttpRequestExecutionHandler;
-import org.jclouds.aws.s3.nio.S3HttpNioFutureCommandExecutionHandler;
-import org.jclouds.http.config.HttpFutureCommandClientModule;
-import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.util.Modules;
-
-/**
- * This installs a {@link HttpNioConnectionPoolClientModule}, but overrides it
- * binding {@link S3HttpNioFutureCommandExecutionHandler}.
- * 
- * @author Adrian Cole
- */
-@HttpFutureCommandClientModule
-public class S3HttpNioConnectionPoolClientModule extends AbstractModule {
-    protected void configure() {
-	install(Modules.override(new HttpNioConnectionPoolClientModule()).with(
-		new AbstractModule() {
-		    protected void configure() {
-			bind(NHttpRequestExecutionHandler.class).to(
-				S3HttpNioFutureCommandExecutionHandler.class);
-		    }
-		}));
-    }
-}

extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3InputStreamMapTest.java

@@ -1,61 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.nio;
-
-import java.util.Properties;
-
-import org.jclouds.aws.s3.S3InputStreamMapTest;
-import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
-import org.testng.annotations.Test;
-
-import com.google.inject.Module;
-
-/**
- * // TODO: Adrian: Document this!
- * 
- * @author Adrian Cole
- */
-@Test(groups = "unit", sequential = true, testName = "s3.NioS3InputStreamMapTest")
-public class NioS3InputStreamMapTest extends S3InputStreamMapTest {
-    
-    @Override
-    protected Properties buildS3Properties(String AWSAccessKeyId,
-	    String AWSSecretAccessKey) {
-	Properties properties = super.buildS3Properties(AWSAccessKeyId,
-		AWSSecretAccessKey);
-	properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
-	properties.setProperty("jclouds.http.pool.max_session_failures", "2");
-	properties
-		.setProperty("jclouds.http.pool.request_invoker_threads", "1");
-	properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
-	properties.setProperty("jclouds.pool.max_connections", "12");
-	return properties;
-    }
-
-    @Override
-    protected Module createHttpModule() {
-	return new S3HttpNioConnectionPoolClientModule();
-    }
-
-}

extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java

@@ -1,61 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.nio;
-
-import java.util.Properties;
-
-import org.jclouds.aws.s3.S3ObjectMapTest;
-import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
-import org.testng.annotations.Test;
-
-import com.google.inject.Module;
-
-/**
- * // TODO: Adrian: Document this!
- * 
- * @author Adrian Cole
- */
-@Test(groups = "unit", sequential = true, testName = "s3.NioS3ObjectMapTest")
-public class NioS3ObjectMapTest extends S3ObjectMapTest {
-
-    @Override
-    protected Properties buildS3Properties(String AWSAccessKeyId,
-	    String AWSSecretAccessKey) {
-	Properties properties = super.buildS3Properties(AWSAccessKeyId,
-		AWSSecretAccessKey);
-	properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
-	properties.setProperty("jclouds.http.pool.max_session_failures", "2");
-	properties
-		.setProperty("jclouds.http.pool.request_invoker_threads", "1");
-	properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
-	properties.setProperty("jclouds.pool.max_connections", "12");
-	return properties;
-    }
-
-    @Override
-    protected Module createHttpModule() {
-	return new S3HttpNioConnectionPoolClientModule();
-    }
-
-}

extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/S3HttpNioFutureCommandExecutionHandlerTest.java

@@ -1,148 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.nio;
-
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.isA;
-import static org.easymock.classextension.EasyMock.createMock;
-import static org.easymock.classextension.EasyMock.replay;
-import static org.easymock.classextension.EasyMock.verify;
-
-import java.io.IOException;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.StatusLine;
-import org.apache.http.nio.entity.NStringEntity;
-import org.jclouds.http.HttpFutureCommand;
-import org.jclouds.http.httpnio.pool.HttpNioFutureCommandExecutionHandler;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-/**
- * // TODO: Adrian: Document this!
- * 
- * @author Adrian Cole
- */
-@Test
-public class S3HttpNioFutureCommandExecutionHandlerTest {
-    S3HttpNioFutureCommandExecutionHandler handler = null;
-    HttpResponse response = null;
-    StatusLine statusline = null;
-
-    @BeforeMethod
-    public void createHandler() {
-	handler = new S3HttpNioFutureCommandExecutionHandler(
-		createMock(HttpNioFutureCommandExecutionHandler.ConsumingNHttpEntityFactory.class),
-		createMock(ExecutorService.class),
-		new ArrayBlockingQueue<HttpFutureCommand<?>>(1));
-	response = createMock(HttpResponse.class);
-	statusline = createMock(StatusLine.class);
-	expect(response.getStatusLine()).andReturn(statusline).atLeastOnce();
-    }
-
-    @Test
-    void test500isRetryable() throws IOException {
-	isRetryable(500);
-    }
-
-    @Test
-    void test503isRetryable() throws IOException {
-	isRetryable(503);
-    }
-
-    @Test
-    void test409isRetryable() throws IOException {
-	isRetryable(409);
-    }
-
-    @Test
-    void test404NotRetryable() throws IOException {
-	expect(statusline.getStatusCode()).andReturn(404).atLeastOnce();
-
-	replay(statusline);
-	replay(response);
-	assert !handler.isRetryable(response);
-	verify(statusline);
-	verify(response);
-    }
-
-    @Test
-    void test400WithNoEnitityNotRetryable() throws IOException {
-	expect(statusline.getStatusCode()).andReturn(400).atLeastOnce();
-	expect(response.getEntity()).andReturn(null);
-	replay(statusline);
-	replay(response);
-	assert !handler.isRetryable(response);
-	verify(statusline);
-	verify(response);
-    }
-
-    @Test
-    void test400WithIrrelevantEnitityNotRetryable() throws IOException {
-	expect(statusline.getStatusCode()).andReturn(400).atLeastOnce();
-	HttpEntity entity = createMock(HttpEntity.class);
-	expect(response.getEntity()).andReturn(entity).atLeastOnce();
-	expect(entity.getContent()).andReturn(IOUtils.toInputStream("hello"));
-	response.setEntity(isA(NStringEntity.class));
-	replay(entity);
-	replay(statusline);
-	replay(response);
-	assert !handler.isRetryable(response);
-	verify(statusline);
-	verify(response);
-	verify(entity);
-    }
-
-    @Test
-    void test400WithRequestTimeTooSkewedTimeEnitityRetryable()
-	    throws IOException {
-	expect(statusline.getStatusCode()).andReturn(400).atLeastOnce();
-	HttpEntity entity = createMock(HttpEntity.class);
-	expect(response.getEntity()).andReturn(entity).atLeastOnce();
-	expect(entity.getContent()).andReturn(
-		IOUtils.toInputStream("RequestTimeTooSkewed"));
-	response.setEntity(isA(NStringEntity.class));
-	replay(entity);
-	replay(statusline);
-	replay(response);
-	assert handler.isRetryable(response);
-	verify(statusline);
-	verify(response);
-	verify(entity);
-    }
-
-    private void isRetryable(int code) throws IOException {
-	expect(statusline.getStatusCode()).andReturn(code).atLeastOnce();
-	replay(statusline);
-	replay(response);
-	assert handler.isRetryable(response);
-	verify(statusline);
-	verify(response);
-    }
-
-}

extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/config/S3HttpNioConnectionPoolClientModuleTest.java

@@ -1,64 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.nio.config;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.name.Names;
-import org.apache.http.nio.protocol.NHttpRequestExecutionHandler;
-import org.jclouds.aws.s3.nio.S3HttpNioFutureCommandExecutionHandler;
-import org.testng.annotations.Test;
-
-import java.util.Properties;
-
-/**
- * // TODO: Adrian: Document this!
- *
- * @author Adrian Cole
- */
-@Test
-public class S3HttpNioConnectionPoolClientModuleTest {
-
-    public void testConfigureBindsS3Handler() {
-        final Properties properties = new Properties();
-        properties.put("jclouds.http.address", "localhost");
-        properties.put("jclouds.http.port", "8088");
-        properties.put("jclouds.http.secure", "false");
-        properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
-        properties.setProperty("jclouds.http.pool.max_session_failures", "2");
-        properties.setProperty("jclouds.http.pool.request_invoker_threads", "1");
-        properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
-        properties.setProperty("jclouds.pool.max_connections", "12");
-
-        Injector i = Guice.createInjector(new S3HttpNioConnectionPoolClientModule() {
-            @Override
-            protected void configure() {
-                Names.bindProperties(binder(), properties);
-                super.configure();
-            }
-        });
-        NHttpRequestExecutionHandler handler = i.getInstance(NHttpRequestExecutionHandler.class);
-        assert handler instanceof S3HttpNioFutureCommandExecutionHandler;
-    }
-}

gae/src/main/java/org/jclouds/gae/URLFetchServiceClient.java

@@ -32,18 +32,15 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.Collections;
 import java.util.List;
 
-import javax.annotation.Resource;
-
 import org.apache.commons.io.IOUtils;
+import org.jclouds.http.BaseHttpFutureCommandClient;
 import org.jclouds.http.HttpFutureCommand;
 import org.jclouds.http.HttpFutureCommandClient;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.http.HttpRequestFilter;
 import org.jclouds.http.HttpResponse;
-import org.jclouds.logging.Logger;
 
 import com.google.appengine.api.urlfetch.HTTPHeader;
 import com.google.appengine.api.urlfetch.HTTPMethod;
@@ -58,67 +55,61 @@ import com.google.inject.Inject;
  * 
  * @author Adrian Cole
  */
-public class URLFetchServiceClient implements HttpFutureCommandClient {
+public class URLFetchServiceClient extends BaseHttpFutureCommandClient {
-    private final URL target;
-    private List<HttpRequestFilter> requestFilters = Collections.emptyList();
-    @Resource
-    private Logger logger = Logger.NULL;
     private final URLFetchService urlFetchService;
 
-    public List<HttpRequestFilter> getRequestFilters() {
-	return requestFilters;
-    }
-
-    @Inject(optional = true)
-    public void setRequestFilters(List<HttpRequestFilter> requestFilters) {
-	this.requestFilters = requestFilters;
-    }
-
     @Inject
     public URLFetchServiceClient(URL target, URLFetchService urlFetchService)
 	    throws MalformedURLException {
+	super(target);
 	this.urlFetchService = urlFetchService;
-	this.target = target;
-	this.logger.info("configured to connect to target: %1s", target);
     }
 
-    public void submit(HttpFutureCommand<?> operation) {
+    public void submit(HttpFutureCommand<?> command) {
-	HttpRequest request = operation.getRequest();
+	HttpRequest request = command.getRequest();
 	HTTPResponse gaeResponse = null;
 	try {
-	    for (HttpRequestFilter filter : getRequestFilters()) {
+	    for (HttpRequestFilter filter : requestFilters) {
 		filter.filter(request);
 	    }
 	    HttpResponse response = null;
 	    for (;;) {
 		logger.trace("%1s - converting request %2s", target, request);
 		HTTPRequest gaeRequest = convert(request);
-		logger
+		if (logger.isTraceEnabled())
-			.trace("%1s - submitting request %2s", target,
+		    logger.trace("%1s - submitting request %2s, headers: %3s",
-				gaeRequest);
+			    target, gaeRequest.getURL(),
+			    headersAsString(gaeRequest.getHeaders()));
 		gaeResponse = this.urlFetchService.fetch(gaeRequest);
-		logger
+		if (logger.isTraceEnabled())
-			.trace("%1s - received response %2s", target,
+		    logger.trace(
-				gaeResponse);
+			    "%1s - received response code %2s, headers: %3s",
+			    target, gaeResponse.getResponseCode(),
+			    headersAsString(gaeResponse.getHeaders()));
 		response = convert(gaeResponse);
-		if (response.getStatusCode() >= 500) {
+		if (isRetryable(command, response))
 		    continue;
-		}
 		break;
-
 	    }
-	    operation.getResponseFuture().setResponse(response);
+	    handleResponse(command, response);
-	    operation.getResponseFuture().run();
 	} catch (Exception e) {
 	    if (gaeResponse != null && gaeResponse.getContent() != null) {
 		logger.error(e,
 			"error encountered during the execution: %1s%n%2s",
 			gaeResponse, new String(gaeResponse.getContent()));
 	    }
-	    operation.setException(e);
+	    command.setException(e);
 	}
     }
 
+    String headersAsString(List<HTTPHeader> headers) {
+	StringBuilder builder = new StringBuilder("");
+	for (HTTPHeader header : headers)
+	    builder.append("[").append(header.getName()).append("=").append(
+		    header.getValue()).append("],");
+	return builder.toString();
+    }
+
     /**
      * byte [] content is replayable and the only content type supportable by
      * GAE. As such, we convert the original request content to a byte array.

gae/src/test/java/org/jclouds/gae/URLFetchServiceClientTest.java

@@ -41,7 +41,6 @@ import java.util.List;
 import org.apache.commons.io.IOUtils;
 import org.jclouds.http.HttpHeaders;
 import org.jclouds.http.HttpRequest;
-import org.jclouds.http.HttpRequestFilter;
 import org.jclouds.http.HttpResponse;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Parameters;
@@ -163,15 +162,6 @@ public class URLFetchServiceClientTest {
 
     }
 
-    @Test
-    void testRequestFilters() {
-	List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>();
-	filters.add(createNiceMock(HttpRequestFilter.class));
-	assertEquals(client.getRequestFilters().size(), 0);
-	client.setRequestFilters(filters);
-	assertEquals(client.getRequestFilters(), filters);
-    }
-
     @Test
     @Parameters("basedir")
     void testConvertRequestFileContent(String basedir) throws IOException {

pom.xml

@@ -41,7 +41,6 @@
         <module>project</module>        
         <module>core</module>
         <module>extensions/httpnio</module>
-        <module>extensions/s3nio</module>
         <module>extensions/log4j</module>
         <module>gae</module>
         <module>s3</module>

s3/perftest/pom.xml

@@ -74,7 +74,7 @@
         </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
-            <artifactId>jclouds-s3nio</artifactId>
+            <artifactId>jclouds-httpnio</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

s3/perftest/src/test/java/com/amazon/s3/AmazonPerformanceTest.java

@@ -42,8 +42,8 @@ import org.testng.annotations.Test;
  * 
  * @author Adrian Cole
  */
-@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformance", groups = "performance")
+@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformance")
-public class AmazonPerformance extends BasePerformance {
+public class AmazonPerformanceTest extends BasePerformance {
     private AWSAuthConnection amzClient;
 
     @Override
@@ -58,33 +58,39 @@ public class AmazonPerformance extends BasePerformance {
     }
 
     @Override
+    @Test(enabled = false)
     public void testPutFileSerial() throws Exception {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled = false)
     public void testPutFileParallel() throws InterruptedException,
 	    ExecutionException {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled = false)
     public void testPutInputStreamSerial() throws Exception {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled = false)
     public void testPutInputStreamParallel() throws InterruptedException,
 	    ExecutionException {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled = false)
     public void testPutStringSerial() throws Exception {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled = false)
     public void testPutStringParallel() throws InterruptedException,
 	    ExecutionException {
 	throw new UnsupportedOperationException();

s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java

@@ -51,6 +51,7 @@ import com.google.inject.Provider;
  * 
  * @author Adrian Cole
  */
+@Test
 public abstract class BasePerformance extends S3IntegrationTest {
     protected boolean debugEnabled() {
 	return false;

s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java

@@ -39,7 +39,7 @@ import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 
-@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest", groups = "performance")
+@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest")
 public class DateServiceTest extends PerformanceTest {
     Injector i = Guice.createInjector();
 

s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformanceTest.java

@@ -24,14 +24,15 @@
 package com.amazon.s3;
 
 
-import com.google.inject.Module;
-import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
-import org.testng.annotations.Test;
-
 import java.util.Properties;
 
-@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsNioPerformance", groups = "performance")
+import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
-public class JCloudsNioPerformance extends BaseJCloudsPerformance {
+import org.testng.annotations.Test;
+
+import com.google.inject.Module;
+
+@Test(sequential=true, testName = "s3.JCloudsNioPerformance")
+public class JCloudsNioPerformanceTest extends BaseJCloudsPerformance {
 
     @Override
     protected Properties buildS3Properties(String AWSAccessKeyId,
@@ -47,6 +48,6 @@ public class JCloudsNioPerformance extends BaseJCloudsPerformance {
 
     @Override
     protected Module createHttpModule() {
-        return new S3HttpNioConnectionPoolClientModule();
+        return new HttpNioConnectionPoolClientModule();
     }
 }

s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformanceTest.java

@@ -26,7 +26,7 @@ package com.amazon.s3;
 
 import org.testng.annotations.Test;
 
-@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsPerformance", groups = "performance")
+@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.JCloudsPerformance")
-public class JCloudsPerformance extends BaseJCloudsPerformance {
+public class JCloudsPerformanceTest extends BaseJCloudsPerformance {
 
 }

s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformanceTest.java

@@ -37,8 +37,8 @@ import org.testng.annotations.Test;
  * 
  * @author Adrian Cole
  */
-@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformance", groups = "performance")
+@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformance")
-public class Jets3tPerformance extends BasePerformance {
+public class Jets3tPerformanceTest extends BasePerformance {
     private S3Service jetClient;
 
     @Override
@@ -50,28 +50,46 @@ public class Jets3tPerformance extends BasePerformance {
     }
 
     @Override
+    @Test(enabled=false)
     public void testPutStringSerial() throws Exception {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled=false)
     public void testPutStringParallel() throws InterruptedException,
 	    ExecutionException {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled=false)
     public void testPutBytesSerial() throws Exception {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled=false)
     public void testPutBytesParallel() throws InterruptedException,
 	    ExecutionException {
 	throw new UnsupportedOperationException();
     }
 
     @Override
+    @Test(enabled=false)
+    public void testPutInputStreamParallel() throws InterruptedException,
+	    ExecutionException {
+	throw new UnsupportedOperationException();
+    }
+    @Override
+    @Test(enabled=false)
+    public void testPutFileParallel() throws InterruptedException,
+	    ExecutionException {
+	throw new UnsupportedOperationException();
+    }
+    
+    @Override
+    @Test(enabled=false)
     protected boolean putByteArray(String bucket, String key, byte[] data,
 	    String contentType) throws Exception {
 	throw new UnsupportedOperationException();

s3/perftest/src/test/java/com/amazon/s3/S3ParserTest.java

@@ -42,7 +42,7 @@ import java.util.concurrent.ExecutorCompletionService;
  * 
  * @author Adrian Cole
  */
-@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3ParserTest", groups = "performance")
+@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3ParserTest")
 public class S3ParserTest extends org.jclouds.aws.s3.commands.S3ParserTest {
 
     class MockHttpURLConnection extends HttpURLConnection {

s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java

@@ -33,7 +33,7 @@ import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 
-@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest", groups = "performance")
+@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest")
 public class S3UtilsTest extends org.jclouds.aws.s3.S3UtilsTest {
 
     @Test(dataProvider = "hmacsha1")

s3/src/main/java/org/jclouds/aws/s3/S3Constants.java

@@ -35,6 +35,6 @@ public interface S3Constants extends HttpConstants, PoolConstants, S3Headers {
 
     public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey";
     public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid";
-    public static final String PROPERTY_AWS_MAP_TIMEOUT = "jclouds.aws.map.timeout";
+    public static final String PROPERTY_S3_MAP_TIMEOUT = "jclouds.s3.map.timeout";
 
 }

s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java

@@ -39,8 +39,8 @@ import java.util.List;
 import java.util.Properties;
 
 import org.jclouds.aws.s3.config.S3ContextModule;
-import org.jclouds.aws.s3.config.S3JavaUrlHttpFutureCommandClientModule;
 import org.jclouds.http.config.HttpFutureCommandClientModule;
+import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
 import org.jclouds.logging.config.LoggingModule;
 import org.jclouds.logging.jdk.config.JDKLoggingModule;
 
@@ -165,7 +165,7 @@ public class S3ContextFactory {
     /**
      * Bind the given properties and install the list of modules. If no modules
      * are specified, install the default {@link JDKLoggingModule}
-     * {@link S3JavaUrlHttpFutureCommandClientModule}
+     * {@link JavaUrlHttpFutureCommandClientModule}
      * 
      * @param properties
      *            - contains constants used by jclouds
@@ -201,7 +201,7 @@ public class S3ContextFactory {
 	    }
 
 	}))
-	    modules.add(new S3JavaUrlHttpFutureCommandClientModule());
+	    modules.add(new JavaUrlHttpFutureCommandClientModule());
     }
 
     @VisibleForTesting

s3/src/main/java/org/jclouds/aws/s3/S3ResponseException.java

@@ -24,19 +24,33 @@
 package org.jclouds.aws.s3;
 
 import org.jclouds.aws.s3.domain.S3Error;
-import org.jclouds.http.HttpMessage;
+import org.jclouds.http.HttpFutureCommand;
+import org.jclouds.http.HttpResponse;
 
 public class S3ResponseException extends RuntimeException {
     private static final long serialVersionUID = 1L;
 
+    private final HttpFutureCommand<?> command;
+    private final HttpResponse response;
     private S3Error error;
 
-    private HttpMessage response;
+    public S3ResponseException(HttpFutureCommand<?> command,
-
+	    HttpResponse response, S3Error error) {
-    public S3ResponseException(S3Error error, HttpMessage response) {
+	super(String.format(
-	super(error.toString());
+		"command: %1s failed with response: %2s; error from s3: %3s",
+		command, response, error));
+	this.command = command;
+	this.response = response;
 	this.setError(error);
-	this.setResponse(response);
+
+    }
+
+    public S3ResponseException(HttpFutureCommand<?> command,
+	    HttpResponse response) {
+	super(String.format("command: %1s failed with response: %2s", command,
+		response));
+	this.command = command;
+	this.response = response;
     }
 
     public void setError(S3Error error) {
@@ -47,11 +61,12 @@ public class S3ResponseException extends RuntimeException {
 	return error;
     }
 
-    public void setResponse(HttpMessage response) {
+    public HttpFutureCommand<?> getCommand() {
-	this.response = response;
+	return command;
     }
 
-    public HttpMessage getResponse() {
+    public HttpResponse getResponse() {
 	return response;
     }
+
 }

s3/src/main/java/org/jclouds/aws/s3/S3Utils.java

@@ -23,8 +23,10 @@
  */
 package org.jclouds.aws.s3;
 
-import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.*;
 
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
@@ -32,6 +34,8 @@ import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;
 
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.bouncycastle.crypto.digests.MD5Digest;
 import org.bouncycastle.crypto.digests.SHA1Digest;
 import org.bouncycastle.crypto.macs.HMac;
@@ -60,6 +64,39 @@ public class S3Utils extends Utils {
 	return new String(hex, "ASCII");
     }
 
+    public static long calculateSize(Object data) {
+	long size = -1;
+	if (data instanceof byte[]) {
+	    size = ((byte[]) data).length;
+	} else if (data instanceof String) {
+	    size = ((String) data).length();
+	} else if (data instanceof File) {
+	    size = ((File) data).length();
+	}
+	return size;
+    }
+
+    /**
+     * 
+     * @throws IOException
+     */
+    public static byte[] md5(Object data) throws IOException {
+	checkNotNull(data, "data must be set before calling generateMd5()");
+	byte[] md5 = null;
+	if (data == null || data instanceof byte[]) {
+	    md5 = S3Utils.md5((byte[]) data);
+	} else if (data instanceof String) {
+	    md5 = S3Utils.md5(((String) data).getBytes());
+	} else if (data instanceof File) {
+	    md5 = S3Utils.md5(((File) data));
+	} else {
+	    throw new UnsupportedOperationException("Content not supported "
+		    + data.getClass());
+	}
+	return md5;
+
+    }
+
     public static byte[] fromHexString(String hex) {
 	byte[] bytes = new byte[hex.length() / 2];
 	for (int i = 0; i < bytes.length; i++) {
@@ -107,22 +144,65 @@ public class S3Utils extends Utils {
 	md5.doFinal(resBuf, 0);
 	return resBuf;
     }
-    
+
-    public static byte[] md5(InputStream toEncode) throws IOException {
+    public static byte[] md5(File toEncode) throws IOException {
 	MD5Digest md5 = new MD5Digest();
 	byte[] resBuf = new byte[md5.getDigestSize()];
 	byte[] buffer = new byte[1024];
 	int numRead = -1;
-	do {
+	InputStream i = new FileInputStream(toEncode);
-	    numRead = toEncode.read(buffer);
+	try {
-	    if (numRead > 0) {
+	    do {
-		md5.update(buffer, 0, numRead);
+		numRead = i.read(buffer);
-	    }
+		if (numRead > 0) {
-	} while (numRead != -1);
+		    md5.update(buffer, 0, numRead);
+		}
+	    } while (numRead != -1);
+	} finally {
+	    IOUtils.closeQuietly(i);
+	}
 	md5.doFinal(resBuf, 0);
 	return resBuf;
     }
 
+    public static Md5InputStreamResult generateMd5Result(InputStream toEncode)
+	    throws IOException {
+	MD5Digest md5 = new MD5Digest();
+	byte[] resBuf = new byte[md5.getDigestSize()];
+	byte[] buffer = new byte[1024];
+	ByteArrayOutputStream out = new ByteArrayOutputStream();
+	long length = 0;
+	int numRead = -1;
+	try {
+	    do {
+		numRead = toEncode.read(buffer);
+		if (numRead > 0) {
+		    length += numRead;
+		    md5.update(buffer, 0, numRead);
+		    out.write(buffer, 0, numRead);
+		}
+	    } while (numRead != -1);
+	} finally {
+	    IOUtils.closeQuietly(toEncode);
+	}
+	md5.doFinal(resBuf, 0);
+	return new Md5InputStreamResult(out.toByteArray(), resBuf, length);
+    }
+
+    public static class Md5InputStreamResult {
+	public final byte[] data;
+	public final byte[] md5;
+	public final long length;
+
+	Md5InputStreamResult(byte[] data, byte[] md5, long length) {
+	    this.data = checkNotNull(data, "data");
+	    this.md5 = checkNotNull(md5, "md5");
+	    checkArgument(length >= 0, "length cannot me negative");
+	    this.length = length;
+	}
+
+    }
+
     public static String getContentAsStringAndClose(S3Object object)
 	    throws IOException {
 	checkNotNull(object, "s3Object");

s3/src/main/java/org/jclouds/aws/s3/commands/BucketExists.java

@@ -24,8 +24,15 @@
 package org.jclouds.aws.s3.commands;
 
 import static org.jclouds.aws.s3.commands.options.ListBucketOptions.Builder.maxResults;
-import org.jclouds.http.commands.callables.ReturnTrueIf200;
 
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.jclouds.aws.s3.S3ResponseException;
+import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
 import com.google.inject.name.Named;
@@ -34,8 +41,39 @@ public class BucketExists extends S3FutureCommand<Boolean> {
 
     @Inject
     public BucketExists(@Named("jclouds.http.address") String amazonHost,
-	    ReturnTrueIf200 callable, @Assisted String s3Bucket) {
+	    ReturnTrueIf2xx callable, @Assisted String s3Bucket) {
-	super("HEAD", "/" + maxResults(0).buildQueryString(), callable, amazonHost,
+	super("HEAD", "/" + maxResults(0).buildQueryString(), callable,
-		s3Bucket);
+		amazonHost, s3Bucket);
+    }
+
+    @Override
+    public Boolean get() throws InterruptedException, ExecutionException {
+	try {
+	    return super.get();
+	} catch (ExecutionException e) {
+	    return attemptNotFound(e);
+	}
+    }
+
+    @VisibleForTesting
+    Boolean attemptNotFound(ExecutionException e) throws ExecutionException {
+	if (e.getCause() != null && e.getCause() instanceof S3ResponseException) {
+	    S3ResponseException responseException = (S3ResponseException) e
+		    .getCause();
+	    if (responseException.getResponse().getStatusCode() == 404) {
+		return false;
+	    }
+	}
+	throw e;
+    }
+
+    @Override
+    public Boolean get(long l, TimeUnit timeUnit) throws InterruptedException,
+	    ExecutionException, TimeoutException {
+	try {
+	    return super.get(l, timeUnit);
+	} catch (ExecutionException e) {
+	    return attemptNotFound(e);
+	}
     }
 }

s3/src/main/java/org/jclouds/aws/s3/commands/DeleteBucket.java

@@ -23,8 +23,14 @@
  */
 package org.jclouds.aws.s3.commands;
 
-import org.jclouds.aws.s3.commands.callables.DeleteBucketCallable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
+import org.jclouds.aws.s3.S3ResponseException;
+import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
 import com.google.inject.name.Named;
@@ -33,7 +39,41 @@ public class DeleteBucket extends S3FutureCommand<Boolean> {
 
     @Inject
     public DeleteBucket(@Named("jclouds.http.address") String amazonHost,
-	    DeleteBucketCallable callable, @Assisted String s3Bucket) {
+	    ReturnTrueIf2xx callable, @Assisted String s3Bucket) {
 	super("DELETE", "/", callable, amazonHost, s3Bucket);
     }
+
+    @Override
+    public Boolean get() throws InterruptedException, ExecutionException {
+	try {
+	    return super.get();
+	} catch (ExecutionException e) {
+	    return attemptNotFound(e);
+	}
+    }
+
+    @VisibleForTesting
+    Boolean attemptNotFound(ExecutionException e) throws ExecutionException {
+	if (e.getCause() != null && e.getCause() instanceof S3ResponseException) {
+	    S3ResponseException responseException = (S3ResponseException) e
+		    .getCause();
+	    if (responseException.getResponse().getStatusCode() == 404) {
+		return true;
+	    } else if ("BucketNotEmpty".equals(responseException.getError()
+		    .getCode())) {
+		return false;
+	    }
+	}
+	throw e;
+    }
+
+    @Override
+    public Boolean get(long l, TimeUnit timeUnit) throws InterruptedException,
+	    ExecutionException, TimeoutException {
+	try {
+	    return super.get(l, timeUnit);
+	} catch (ExecutionException e) {
+	    return attemptNotFound(e);
+	}
+    }
 }

s3/src/main/java/org/jclouds/aws/s3/commands/DeleteObject.java

@@ -25,7 +25,7 @@ package org.jclouds.aws.s3.commands;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import org.jclouds.aws.s3.commands.callables.DeleteCallable;
+import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
 
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
@@ -35,7 +35,7 @@ public class DeleteObject extends S3FutureCommand<Boolean> {
 
     @Inject
     public DeleteObject(@Named("jclouds.http.address") String amazonHost,
-	    DeleteCallable callable, @Assisted("bucketName") String bucket,
+	    ReturnTrueIf2xx callable, @Assisted("bucketName") String bucket,
 	    @Assisted("key") String key) {
 	super("DELETE", "/" + checkNotNull(key), callable, amazonHost, bucket);
     }

s3/src/main/java/org/jclouds/aws/s3/commands/GetObject.java

@@ -30,7 +30,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 import org.jclouds.aws.s3.S3ResponseException;
-import org.jclouds.aws.s3.commands.callables.GetObjectCallable;
+import org.jclouds.aws.s3.commands.callables.ParseObjectFromHeadersAndHttpContent;
 import org.jclouds.aws.s3.commands.options.GetObjectOptions;
 import org.jclouds.aws.s3.domain.S3Object;
 
@@ -50,7 +50,7 @@ public class GetObject extends S3FutureCommand<S3Object> {
 
     @Inject
     public GetObject(@Named("jclouds.http.address") String amazonHost,
-	    GetObjectCallable callable,
+	    ParseObjectFromHeadersAndHttpContent callable,
 	    @Assisted("bucketName") String s3Bucket,
 	    @Assisted("key") String key, @Assisted GetObjectOptions options) {
 	super("GET", "/" + checkNotNull(key), callable, amazonHost, s3Bucket);

s3/src/main/java/org/jclouds/aws/s3/commands/HeadMetaData.java

@@ -25,9 +25,15 @@ package org.jclouds.aws.s3.commands;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import org.jclouds.aws.s3.commands.callables.HeadMetaDataCallable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.jclouds.aws.s3.S3ResponseException;
+import org.jclouds.aws.s3.commands.callables.ParseMetadataFromHeaders;
 import org.jclouds.aws.s3.domain.S3Object;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
 import com.google.inject.name.Named;
@@ -43,9 +49,42 @@ public class HeadMetaData extends S3FutureCommand<S3Object.Metadata> {
 
     @Inject
     public HeadMetaData(@Named("jclouds.http.address") String amazonHost,
-	    HeadMetaDataCallable callable,
+	    ParseMetadataFromHeaders callable,
 	    @Assisted("bucketName") String bucket, @Assisted("key") String key) {
 	super("HEAD", "/" + checkNotNull(key), callable, amazonHost, bucket);
 	callable.setKey(key);
     }
+
+    @Override
+    public S3Object.Metadata get() throws InterruptedException,
+	    ExecutionException {
+	try {
+	    return super.get();
+	} catch (ExecutionException e) {
+	    return attemptNotFound(e);
+	}
+    }
+
+    @VisibleForTesting
+    S3Object.Metadata attemptNotFound(ExecutionException e)
+	    throws ExecutionException {
+	if (e.getCause() != null && e.getCause() instanceof S3ResponseException) {
+	    S3ResponseException responseException = (S3ResponseException) e
+		    .getCause();
+	    if (responseException.getResponse().getStatusCode() == 404) {
+		return S3Object.Metadata.NOT_FOUND;
+	    }
+	}
+	throw e;
+    }
+
+    @Override
+    public S3Object.Metadata get(long l, TimeUnit timeUnit)
+	    throws InterruptedException, ExecutionException, TimeoutException {
+	try {
+	    return super.get(l, timeUnit);
+	} catch (ExecutionException e) {
+	    return attemptNotFound(e);
+	}
+    }
 }

s3/src/main/java/org/jclouds/aws/s3/commands/PutBucket.java

@@ -23,9 +23,9 @@
  */
 package org.jclouds.aws.s3.commands;
 
-import org.jclouds.aws.s3.commands.callables.PutBucketCallable;
 import org.jclouds.aws.s3.commands.options.PutBucketOptions;
 import org.jclouds.http.HttpHeaders;
+import org.jclouds.http.commands.callables.ReturnTrueIf2xx;
 
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
@@ -41,7 +41,7 @@ public class PutBucket extends S3FutureCommand<Boolean> {
 
     @Inject
     public PutBucket(@Named("jclouds.http.address") String amazonHost,
-	    PutBucketCallable callable, @Assisted String s3Bucket,
+	    ReturnTrueIf2xx callable, @Assisted String s3Bucket,
 	    @Assisted PutBucketOptions options) {
 	super("PUT", "/", callable, amazonHost, s3Bucket);
 	getRequest().getHeaders().putAll(options.buildRequestHeaders());

s3/src/main/java/org/jclouds/aws/s3/commands/PutObject.java

@@ -23,10 +23,10 @@
  */
 package org.jclouds.aws.s3.commands;
 
-import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.*;
 
 import org.jclouds.aws.s3.S3Utils;
-import org.jclouds.aws.s3.commands.callables.PutObjectCallable;
+import org.jclouds.aws.s3.commands.callables.ParseMd5FromETagHeader;
 import org.jclouds.aws.s3.commands.options.PutObjectOptions;
 import org.jclouds.aws.s3.domain.S3Object;
 import org.jclouds.http.HttpHeaders;
@@ -39,10 +39,12 @@ public class PutObject extends S3FutureCommand<byte[]> {
 
     @Inject
     public PutObject(@Named("jclouds.http.address") String amazonHost,
-	    PutObjectCallable callable, @Assisted String s3Bucket,
+	    ParseMd5FromETagHeader callable, @Assisted String s3Bucket,
 	    @Assisted S3Object object, @Assisted PutObjectOptions options) {
 	super("PUT", "/" + checkNotNull(object.getKey()), callable, amazonHost,
 		s3Bucket);
+	checkArgument(object.getMetaData().getSize() >=0,"size must be set");
+	
 	getRequest().setPayload(
 		checkNotNull(object.getData(), "object.getContent()"));
 
@@ -50,6 +52,7 @@ public class PutObject extends S3FutureCommand<byte[]> {
 		HttpHeaders.CONTENT_TYPE,
 		checkNotNull(object.getMetaData().getContentType(),
 			"object.metaData.contentType()"));
+
 	getRequest().getHeaders().put(HttpHeaders.CONTENT_LENGTH,
 		object.getMetaData().getSize() + "");
 
@@ -75,5 +78,4 @@ public class PutObject extends S3FutureCommand<byte[]> {
 	getRequest().getHeaders().putAll(options.buildRequestHeaders());
 
     }
-
 }

s3/src/main/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallable.java

@@ -1,60 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.commands.callables;
-
-import java.io.IOException;
-
-import org.jclouds.aws.s3.S3Utils;
-import org.jclouds.http.HttpException;
-
-/**
- * // TODO: Adrian: Document this!
- * 
- * @author Adrian Cole
- */
-public class DeleteBucketCallable extends DeleteCallable {
-
-    public Boolean call() throws HttpException {
-	if (getResponse().getStatusCode() == 404) {
-	    return true;
-	} else if (getResponse().getStatusCode() == 409) {
-	    try {
-		if (getResponse().getContent() == null) {
-		    throw new HttpException(
-			    "cannot determine error as there is no content");
-		}
-		String reason = S3Utils.toStringAndClose(getResponse()
-			.getContent());
-		if (reason.indexOf("BucketNotEmpty") >= 0)
-		    return false;
-		else
-		    throw new HttpException("Error deleting bucket.\n" + reason);
-	    } catch (IOException e) {
-		throw new HttpException("Error deleting bucket", e);
-	    }
-	} else {
-	    return super.call();
-	}
-    }
-}

s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMd5FromETagHeader.java

@@ -23,24 +23,28 @@
  */
 package org.jclouds.aws.s3.commands.callables;
 
+import org.apache.commons.io.IOUtils;
+import org.jclouds.aws.s3.S3Headers;
+import org.jclouds.aws.s3.S3Utils;
 import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpFutureCommand;
 
 /**
- * Returns true, if the bucket was created.
+ * Parses an MD5 checksum from the header {@link S3Headers#ETAG}.
- *  
+ * 
  * @author Adrian Cole
  */
-public class PutBucketCallable extends
+public class ParseMd5FromETagHeader extends
-	HttpFutureCommand.ResponseCallable<Boolean> {
+	HttpFutureCommand.ResponseCallable<byte[]> {
 
-    public Boolean call() throws HttpException {
+    public byte[] call() throws HttpException {
-	if (getResponse().getStatusCode() == 200) {
+	checkCode();
-	    return true;
+	IOUtils.closeQuietly(getResponse().getContent());
-	} else if (getResponse().getStatusCode() == 404) {
+
-	    return false;
+	String eTag = getResponse().getFirstHeaderOrNull(S3Headers.ETAG);
-	} else {
+	if (eTag != null) {
-	    throw new HttpException("Error checking bucket " + getResponse());
+	    return S3Utils.fromHexString(eTag.replaceAll("\"", ""));
 	}
+	throw new HttpException("did not receive ETag");
     }
 }

s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseMetadataFromHeaders.java

@@ -23,10 +23,8 @@
  */
 package org.jclouds.aws.s3.commands.callables;
 
-import java.io.IOException;
 import java.util.Map.Entry;
 
-import org.jclouds.Utils;
 import org.jclouds.aws.s3.DateService;
 import org.jclouds.aws.s3.S3Headers;
 import org.jclouds.aws.s3.S3Utils;
@@ -38,58 +36,65 @@ import org.jclouds.http.HttpHeaders;
 import com.google.inject.Inject;
 
 /**
- * This parses @{link {@link S3Object.Metadata} from http headers or returns
+ * This parses @{link {@link S3Object.Metadata} from http headers.
- * {@link S3Object.Metadata#NOT_FOUND} on 404.
  * 
  * @author Adrian Cole
  */
-public class HeadMetaDataCallable extends
+public class ParseMetadataFromHeaders extends
 	HttpFutureCommand.ResponseCallable<S3Object.Metadata> {
     private final DateService dateParser;
     private String key;
 
     @Inject
-    public HeadMetaDataCallable(DateService dateParser) {
+    public ParseMetadataFromHeaders(DateService dateParser) {
 	this.dateParser = dateParser;
     }
 
     /**
-     * @return S3Content.NOT_FOUND, if not found.
+     * parses the http response headers to create a new
-     * @throws org.jclouds.http.HttpException
+     * {@link S3Object.MetaData} object.
      */
     public S3Object.Metadata call() throws HttpException {
-	if (getResponse().getStatusCode() == 200) {
+	checkCode();
-	    S3Object.Metadata metaData = new S3Object.Metadata(key);
+	
+	S3Object.Metadata metaData = new S3Object.Metadata(key);
 
-	    extractUserMetadata(metaData);
+	extractUserMetadata(metaData);
-	    addMd5(metaData);
+	addMd5(metaData);
 
-	    metaData.setLastModified(dateParser
+	parseLastModifiedOrThrowException(metaData);
-		    .dateTimeFromHeaderFormat(getResponse()
+	setContentTypeOrThrowException(metaData);
-			    .getFirstHeaderOrNull(HttpHeaders.LAST_MODIFIED)));
+
-	    metaData.setContentType(getResponse().getFirstHeaderOrNull(
+	metaData.setCacheControl(getResponse().getFirstHeaderOrNull(
-		    HttpHeaders.CONTENT_TYPE));
+		HttpHeaders.CACHE_CONTROL));
-	    metaData.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull(
+	metaData.setContentDisposition(getResponse().getFirstHeaderOrNull(
-		    HttpHeaders.CONTENT_LENGTH)));
+		HttpHeaders.CONTENT_DISPOSITION));
-	    metaData.setCacheControl(getResponse().getFirstHeaderOrNull(
+	metaData.setContentEncoding(getResponse().getFirstHeaderOrNull(
-		    HttpHeaders.CACHE_CONTROL));
+		HttpHeaders.CONTENT_ENCODING));
-	    metaData.setContentDisposition(getResponse().getFirstHeaderOrNull(
+	return metaData;
-		    HttpHeaders.CONTENT_DISPOSITION));
+
-	    metaData.setContentEncoding(getResponse().getFirstHeaderOrNull(
+    }
-		    HttpHeaders.CONTENT_ENCODING));
+
-	    return metaData;
+    private void setContentTypeOrThrowException(S3Object.Metadata metaData)
-	} else if (getResponse().getStatusCode() == 404) {
+	    throws HttpException {
-	    return S3Object.Metadata.NOT_FOUND;
+	String contentType = getResponse().getFirstHeaderOrNull(
-	} else {
+		HttpHeaders.CONTENT_TYPE);
-	    String reason = null;
+	if (contentType == null)
-	    try {
+	    throw new HttpException(HttpHeaders.CONTENT_TYPE
-		reason = Utils.toStringAndClose(getResponse().getContent());
+		    + " not found in headers");
-	    } catch (IOException e) {
+	else
-		logger.error(e, "error parsing reason");
+	    metaData.setContentType(contentType);
-	    }
+    }
-	    throw new HttpException("Error parsing object " + getResponse()
+
-		    + " reason: " + reason);
+    private void parseLastModifiedOrThrowException(S3Object.Metadata metaData)
-	}
+	    throws HttpException {
+	String lastModified = getResponse().getFirstHeaderOrNull(
+		HttpHeaders.LAST_MODIFIED);
+	metaData.setLastModified(dateParser
+		.dateTimeFromHeaderFormat(lastModified));
+	if (metaData.getLastModified() == null)
+	    throw new HttpException("could not parse: "
+		    + HttpHeaders.LAST_MODIFIED + ": " + lastModified);
     }
 
     private void addMd5(S3Object.Metadata metaData) {

s3/src/main/java/org/jclouds/aws/s3/commands/callables/ParseObjectFromHeadersAndHttpContent.java

@@ -23,51 +23,55 @@
  */
 package org.jclouds.aws.s3.commands.callables;
 
-import java.io.IOException;
-
-import org.jclouds.Utils;
 import org.jclouds.aws.s3.domain.S3Object;
+import org.jclouds.aws.s3.domain.S3Object.Metadata;
 import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpFutureCommand;
+import org.jclouds.http.HttpHeaders;
 
 import com.google.inject.Inject;
 
 /**
- * // TODO: Adrian: Document this!
+ * Parses response headers and creates a new S3Object from them and the http
+ * content.
  * 
  * @author Adrian Cole
  */
-public class GetObjectCallable extends
+public class ParseObjectFromHeadersAndHttpContent extends
 	HttpFutureCommand.ResponseCallable<S3Object> {
-    private final HeadMetaDataCallable metaDataParser;
+    private final ParseMetadataFromHeaders metaDataParser;
-    private String key;
 
     @Inject
-    public GetObjectCallable(HeadMetaDataCallable metaDataParser) {
+    public ParseObjectFromHeadersAndHttpContent(ParseMetadataFromHeaders metaDataParser) {
 	this.metaDataParser = metaDataParser;
     }
 
     /**
-     * @return S3Content.NOT_FOUND, if not found.
+     * First, calls {@link ParseMetadataFromHeaders}.
+     * 
+     * Then, sets the object size based on the Content-Length header and adds
+     * the content to the {@link S3Object} result.
+     * 
      * @throws org.jclouds.http.HttpException
      */
     public S3Object call() throws HttpException {
+	checkCode();
 	metaDataParser.setResponse(getResponse());
 	S3Object.Metadata metaData = metaDataParser.call();
-	if (metaData == S3Object.Metadata.NOT_FOUND)
+
-	    return S3Object.NOT_FOUND;
+	parseContentLengthOrThrowException(metaData);
-	if (getResponse().getContent() != null) {
+	return new S3Object(metaData, getResponse().getContent());
-	    return new S3Object(metaData, getResponse().getContent());
+    }
-	} else {
+
-	    String reason = null;
+    private void parseContentLengthOrThrowException(Metadata metaData)
-	    try {
+	    throws HttpException {
-		reason = Utils.toStringAndClose(getResponse().getContent());
+	String contentLength = getResponse().getFirstHeaderOrNull(
-	    } catch (IOException e) {
+		HttpHeaders.CONTENT_LENGTH);
-		logger.error(e, "error parsing reason");
+	if (contentLength == null)
-	    }
+	    throw new HttpException(HttpHeaders.CONTENT_LENGTH
-	    throw new HttpException("No content retrieving object " + key + ":"
+		    + " header not present in headers: "
-		    + getResponse() + " reason: " + reason);
+		    + getResponse().getHeaders());
-	}
+	metaData.setSize(Long.parseLong(contentLength));
     }
 
     public void setKey(String key) {

s3/src/main/java/org/jclouds/aws/s3/commands/callables/PutObjectCallable.java

@@ -1,65 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.commands.callables;
-
-import java.io.IOException;
-
-import org.jclouds.aws.s3.S3Utils;
-import org.jclouds.http.HttpException;
-import org.jclouds.http.HttpFutureCommand;
-
-/**
- * // TODO: Adrian: Document this!
- * 
- * @author Adrian Cole
- */
-public class PutObjectCallable extends
-	HttpFutureCommand.ResponseCallable<byte []> {
-
-    public byte [] call() throws HttpException {
-	if (getResponse().getStatusCode() == 200) {
-	    try {
-		getResponse().getContent().close();
-	    } catch (IOException e) {
-		logger.error(e, "error consuming content");
-	    }
-	    String eTag = getResponse().getFirstHeaderOrNull("ETag");
-	    if (eTag != null) {
-		return S3Utils
-			.fromHexString(eTag.replaceAll("\"", ""));
-	    }
-	    throw new HttpException("did not receive ETag");
-	} else {
-	    try {
-		String reason = S3Utils.toStringAndClose(getResponse()
-			.getContent());
-		throw new HttpException(getResponse().getStatusCode()
-			+ ": Problem uploading content.\n" + reason);
-	    } catch (IOException e) {
-		throw new HttpException(getResponse().getStatusCode()
-			+ ": Problem uploading content", e);
-	    }
-	}
-    }
-}

s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java

@@ -26,22 +26,33 @@ package org.jclouds.aws.s3.config;
 import java.util.ArrayList;
 import java.util.List;
 
+import javax.annotation.Resource;
+
 import org.jclouds.aws.s3.S3Connection;
 import org.jclouds.aws.s3.S3Context;
 import org.jclouds.aws.s3.commands.config.S3CommandsModule;
+import org.jclouds.aws.s3.filters.ParseS3ErrorFromXmlContent;
 import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader;
 import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
 import org.jclouds.aws.s3.internal.GuiceS3Context;
 import org.jclouds.aws.s3.internal.LiveS3Connection;
 import org.jclouds.aws.s3.internal.LiveS3InputStreamMap;
 import org.jclouds.aws.s3.internal.LiveS3ObjectMap;
+import org.jclouds.http.HttpConstants;
 import org.jclouds.http.HttpRequestFilter;
+import org.jclouds.http.HttpResponseHandler;
+import org.jclouds.http.annotation.ClientErrorHandler;
+import org.jclouds.http.annotation.RedirectHandler;
+import org.jclouds.http.annotation.ServerErrorHandler;
+import org.jclouds.logging.Logger;
 
 import com.google.inject.AbstractModule;
+import com.google.inject.Inject;
 import com.google.inject.Provides;
 import com.google.inject.Scopes;
 import com.google.inject.Singleton;
 import com.google.inject.assistedinject.FactoryProvider;
+import com.google.inject.name.Named;
 
 /**
  * // TODO: Adrian: Document this!
@@ -49,6 +60,18 @@ import com.google.inject.assistedinject.FactoryProvider;
  * @author Adrian Cole
  */
 public class S3ContextModule extends AbstractModule {
+    @Resource
+    protected Logger logger = Logger.NULL;
+
+    @Inject
+    @Named(HttpConstants.PROPERTY_HTTP_ADDRESS)
+    String address;
+    @Inject
+    @Named(HttpConstants.PROPERTY_HTTP_PORT)
+    int port;
+    @Inject
+    @Named(HttpConstants.PROPERTY_HTTP_SECURE)
+    boolean isSecure;
 
     @Override
     protected void configure() {
@@ -64,6 +87,15 @@ public class S3ContextModule extends AbstractModule {
 			GuiceS3Context.S3InputStreamMapFactory.class,
 			LiveS3InputStreamMap.class));
 	bind(S3Context.class).to(GuiceS3Context.class);
+	bind(HttpResponseHandler.class).annotatedWith(RedirectHandler.class)
+		.to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON);
+	bind(HttpResponseHandler.class).annotatedWith(ClientErrorHandler.class)
+		.to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON);
+	bind(HttpResponseHandler.class).annotatedWith(ServerErrorHandler.class)
+		.to(ParseS3ErrorFromXmlContent.class).in(Scopes.SINGLETON);
+	requestInjection(this);
+	logger.info("S3 Context = %1s://%2s:%3s",
+		(isSecure ? "https" : "http"), address, port);
     }
 
     @Provides
@@ -76,4 +108,5 @@ public class S3ContextModule extends AbstractModule {
 	filters.add(requestAuthorizeSignature);
 	return filters;
     }
+
 }

s3/src/main/java/org/jclouds/aws/s3/config/S3JavaUrlHttpFutureCommandClientModule.java

@@ -1,46 +0,0 @@
-/**
- *
- * Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
- *
- * ====================================================================
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ====================================================================
- */
-package org.jclouds.aws.s3.config;
-
-import org.jclouds.aws.s3.internal.S3JavaUrlHttpFutureCommandClient;
-import org.jclouds.http.HttpFutureCommandClient;
-import org.jclouds.http.config.HttpFutureCommandClientModule;
-import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
-
-/**
- * Configures {@link S3JavaUrlHttpFutureCommandClient}.
- * 
- * @author Adrian Cole
- */
-@HttpFutureCommandClientModule
-public class S3JavaUrlHttpFutureCommandClientModule extends
-	JavaUrlHttpFutureCommandClientModule {
-
-    @Override
-    protected void bindClient() {
-	// note this is not threadsafe, so it cannot be singleton
-	bind(HttpFutureCommandClient.class).to(
-		S3JavaUrlHttpFutureCommandClient.class);
-    }
-}

s3/src/main/java/org/jclouds/aws/s3/domain/S3Object.java

@@ -27,14 +27,12 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Arrays;
 
-import org.apache.commons.io.IOUtils;
 import org.jclouds.aws.s3.S3Utils;
+import org.jclouds.aws.s3.S3Utils.Md5InputStreamResult;
 import org.jclouds.http.ContentTypes;
 import org.joda.time.DateTime;
 
@@ -62,12 +60,12 @@ public class S3Object {
 
     public S3Object(Metadata metaData, Object data) {
 	this(metaData);
-	this.data = data;
+	setData(data);
     }
 
     public S3Object(String key, Object data) {
 	this(key);
-	this.data = data;
+	setData(data);
     }
 
     public static class Metadata {
@@ -76,7 +74,7 @@ public class S3Object {
 	// parsed during list, head, or get
 	private final String key;
 	private byte[] md5;
-	private long size = -1;
+	private volatile long size = -1;
 
 	// only parsed during head or get
 	private Multimap<String, String> userMetadata = HashMultimap.create();
@@ -238,44 +236,22 @@ public class S3Object {
     }
 
     public void setData(Object data) {
-	this.data = data;
+	this.data = checkNotNull(data, "data");
+	if (getMetaData().getSize() == -1)
+	    this.getMetaData().setSize(S3Utils.calculateSize(data));
     }
 
-    /**
-     * converts String or InputStreams to byte [] type before generating an md5;
-     * 
-     * @throws IOException
-     */
     public void generateMd5() throws IOException {
-	checkState(data != null,
+	checkState(data != null, "data");
-		"data must be set before calling generateMd5()");
+	if (data instanceof InputStream) {
-	byte[] md5;
+	    Md5InputStreamResult result = S3Utils
-	if (data == null || data instanceof byte[]) {
+		    .generateMd5Result((InputStream) data);
-	    md5 = S3Utils.md5((byte[]) data);
+	    getMetaData().setSize(result.length);
-	} else if (data instanceof String) {
+	    getMetaData().setMd5(result.md5);
-	    this.data = ((String) data).getBytes();
+	    setData(result.data);
-	    md5 = S3Utils.md5((byte[]) data);
-	} else if (data instanceof File) {
-	    InputStream i = new FileInputStream((File) data);
-	    try {
-		md5 = S3Utils.md5(i);
-	    } finally {
-		IOUtils.closeQuietly(i);
-	    }
-	} else if (data instanceof InputStream) {
-	    InputStream i = (InputStream) data;
-	    try {
-		this.data = (IOUtils.toByteArray(i));
-		md5 = S3Utils.md5(i);
-	    } finally {
-		IOUtils.closeQuietly(i);
-
-	    }
 	} else {
-	    throw new UnsupportedOperationException("Content not supported "
+	    getMetaData().setMd5(S3Utils.md5(data));
-		    + data.getClass());
 	}
-	getMetaData().setMd5(md5);
     }
 
     public Object getData() {

s3/src/main/java/org/jclouds/aws/s3/filters/ParseS3ErrorFromXmlContent.java

@@ -21,40 +21,42 @@
  * under the License.
  * ====================================================================
  */
-package org.jclouds.aws.s3.internal;
+package org.jclouds.aws.s3.filters;
 
-import java.io.IOException;
 import java.io.InputStream;
-import java.net.HttpURLConnection;
+
-import java.net.MalformedURLException;
+import javax.annotation.Resource;
-import java.net.URL;
 
 import org.apache.commons.io.IOUtils;
 import org.jclouds.aws.s3.S3ResponseException;
 import org.jclouds.aws.s3.domain.S3Error;
 import org.jclouds.aws.s3.xml.S3ParserFactory;
 import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpFutureCommand;
 import org.jclouds.http.HttpResponse;
-import org.jclouds.http.JavaUrlHttpFutureCommandClient;
+import org.jclouds.http.HttpResponseHandler;
+import org.jclouds.logging.Logger;
 
 import com.google.inject.Inject;
 
-public class S3JavaUrlHttpFutureCommandClient extends
+/**
-	JavaUrlHttpFutureCommandClient {
+ * This will parse and set an appropriate exception on the command object.
+ * 
+ * @author Adrian Cole
+ * 
+ */
+public class ParseS3ErrorFromXmlContent implements HttpResponseHandler {
+    @Resource
+    protected Logger logger = Logger.NULL;
 
-    private S3ParserFactory parserFactory;
+    private final S3ParserFactory parserFactory;
 
     @Inject
-    public S3JavaUrlHttpFutureCommandClient(S3ParserFactory parserFactory,
+    public ParseS3ErrorFromXmlContent(S3ParserFactory parserFactory) {
-	    URL target) throws MalformedURLException {
-	super(target);
 	this.parserFactory = parserFactory;
     }
 
-    @Override
+    public void handle(HttpFutureCommand<?> command, HttpResponse response) {
-    protected HttpResponse getResponse(HttpURLConnection connection)
-	    throws IOException {
-	HttpResponse response = super.getResponse(connection);
 	int code = response.getStatusCode();
 	if (code >= 300) {
 	    InputStream errorStream = response.getContent();
@@ -62,16 +64,20 @@ public class S3JavaUrlHttpFutureCommandClient extends
 		try {
 		    S3Error error = parserFactory.createErrorParser().parse(
 			    errorStream);
-		    logger.error("received the following error from s3: %1s",
+		    logger.trace("received the following error from s3: %1s",
 			    error);
-		    throw new S3ResponseException(error, response);
+		    command.setException(new S3ResponseException(command,
+			    response, error));
 		} catch (HttpException he) {
-		    logger.error(he, "error parsing response");
+		    logger.error(he, "error parsing response %1s", response);
 		} finally {
 		    IOUtils.closeQuietly(errorStream);
 		}
+	    } else {
+		command
+			.setException(new S3ResponseException(command, response));
 	    }
 	}
-	return response;
     }
-}
+
+}

s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java

@@ -24,8 +24,6 @@
 package org.jclouds.aws.s3.filters;
 
 import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -36,20 +34,17 @@ import org.jclouds.http.HttpException;
 import org.jclouds.http.HttpHeaders;
 import org.jclouds.http.HttpRequest;
 import org.jclouds.http.HttpRequestFilter;
-import org.joda.time.DateTime;
 
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
 
 public class RequestAuthorizeSignature implements HttpRequestFilter {
     private static final String[] firstHeadersToSign = new String[] {
-	    HttpHeaders.CONTENT_MD5, HttpHeaders.CONTENT_TYPE,
+	    HttpHeaders.CONTENT_MD5, HttpHeaders.CONTENT_TYPE, HttpHeaders.DATE };
-	    HttpHeaders.DATE };
 
     private final String accessKey;
     private final String secretKey;
     private final DateService dateService;
-    private Map<String, DateTime> dateCache = new ConcurrentHashMap<String, DateTime>();
 
     public static final long BILLION = 1000000000;
     private final AtomicReference<String> timeStamp;
@@ -80,24 +75,6 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
 	return timeStamp.get();
     }
 
-    public DateTime dateTimeFromXMLFormat(String toParse) {
-	DateTime time = dateCache.get(toParse);
-	if (time == null) {
-	    time = dateService.dateTimeFromXMLFormat(toParse);
-	    dateCache.put(toParse, time);
-	}
-	return time;
-    }
-
-    public DateTime dateTimeFromHeaderFormat(String toParse) {
-	DateTime time = dateCache.get(toParse);
-	if (time == null) {
-	    time = dateService.dateTimeFromHeaderFormat(toParse);
-	    dateCache.put(toParse, time);
-	}
-	return time;
-    }
-
     @Inject
     public RequestAuthorizeSignature(
 	    @Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey,
@@ -149,8 +126,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
     }
 
     private void addDateHeader(HttpRequest request) {
-	request.getHeaders().put(HttpHeaders.DATE,
+	request.getHeaders().put(HttpHeaders.DATE, timestampAsHeaderString());
-		dateService.timestampAsHeaderString());
     }
 
     private void appendAmzHeaders(HttpRequest request, StringBuilder toSign) {

s3/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java

@@ -25,11 +25,8 @@ package org.jclouds.aws.s3.internal;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -45,7 +42,6 @@ import org.jclouds.Utils;
 import org.jclouds.aws.s3.S3Connection;
 import org.jclouds.aws.s3.S3Constants;
 import org.jclouds.aws.s3.S3Map;
-import org.jclouds.aws.s3.S3Utils;
 import org.jclouds.aws.s3.domain.S3Bucket;
 import org.jclouds.aws.s3.domain.S3Object;
 
@@ -62,7 +58,7 @@ public abstract class BaseS3Map<T> implements Map<String, T>, S3Map {
      * maximum duration of an S3 Request
      */
     @Inject(optional = true)
-    @Named(S3Constants.PROPERTY_AWS_MAP_TIMEOUT)
+    @Named(S3Constants.PROPERTY_S3_MAP_TIMEOUT)
     protected long requestTimeoutMilliseconds = 10000;
 
     @Inject
@@ -71,6 +67,13 @@ public abstract class BaseS3Map<T> implements Map<String, T>, S3Map {
 	this.bucket = checkNotNull(bucket, "bucket");
     }
 
+    /**
+     * {@inheritDoc}
+     * <p/>
+     * This returns the number of keys in the {@link S3Bucket}
+     * 
+     * @see S3Bucket#getContents()
+     */
     public int size() {
 	try {
 	    S3Bucket bucket = refreshBucket();
@@ -95,28 +98,15 @@ public abstract class BaseS3Map<T> implements Map<String, T>, S3Map {
     protected byte[] getMd5(Object value) throws IOException,
 	    FileNotFoundException, InterruptedException, ExecutionException,
 	    TimeoutException {
-	byte[] md5;
+	S3Object object = null;
-
+	if (value instanceof S3Object) {
-	if (value instanceof InputStream) {
+	    object = (S3Object) value;
-	    md5 = S3Utils.md5((InputStream) value);
-	} else if (value instanceof byte[]) {
-	    md5 = S3Utils.md5((byte[]) value);
-	} else if (value instanceof String) {
-	    md5 = S3Utils.md5(((String) value).getBytes());
-	} else if (value instanceof File) {
-	    md5 = S3Utils.md5(new FileInputStream((File) value));
-	} else if (value instanceof S3Object) {
-	    S3Object object = (S3Object) value;
-	    object = connection.getObject(bucket, object.getKey()).get(
-		    requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
-	    if (S3Object.NOT_FOUND.equals(object))
-		throw new FileNotFoundException("not found: " + object.getKey());
-	    md5 = object.getMetaData().getMd5();
 	} else {
-	    throw new IllegalArgumentException("unsupported value type: "
+	    object = new S3Object("dummy", value);
-		    + value.getClass());
 	}
-	return md5;
+	if (object.getMetaData().getMd5() == null)
+	    object.generateMd5();
+	return object.getMetaData().getMd5();
     }
 
     protected Set<S3Object> getAllObjects() {
@@ -141,13 +131,14 @@ public abstract class BaseS3Map<T> implements Map<String, T>, S3Map {
 	return objects;
     }
 
-    /*
+    /**
-     * (non-Javadoc)
+     * {@inheritDoc}
      * 
-     * @see org.jclouds.aws.s3.S3ObjectMapi#containsValue(java.lang.Object)
+     * Note that if value is an instance of InputStream, it will be read and
+     * closed following this method. To reuse data from InputStreams, pass
+     * {@link InputStream}s inside {@link S3Object}s
      */
     public boolean containsValue(Object value) {
-
 	try {
 	    byte[] md5 = getMd5(value);
 	    return containsMd5(md5);

s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3InputStreamMap.java

@@ -23,9 +23,7 @@
  */
 package org.jclouds.aws.s3.internal;
 
-import java.io.ByteArrayInputStream;
 import java.io.File;
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -37,7 +35,6 @@ import java.util.Set;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.io.IOUtils;
 import org.jclouds.Utils;
 import org.jclouds.aws.s3.S3Connection;
 import org.jclouds.aws.s3.S3InputStreamMap;
@@ -151,7 +148,7 @@ public class LiveS3InputStreamMap extends BaseS3Map<InputStream> implements
 	try {
 	    InputStream returnVal = containsKey(s) ? get(s) : null;
 	    object.setData(o);
-	    setSizeIfContentIsInputStream(object);
+	    object.generateMd5();
 	    connection.putObject(bucket, object).get(
 		    requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
 	    return returnVal;
@@ -189,7 +186,7 @@ public class LiveS3InputStreamMap extends BaseS3Map<InputStream> implements
 	    for (String key : map.keySet()) {
 		S3Object object = new S3Object(key);
 		object.setData(map.get(key));
-		setSizeIfContentIsInputStream(object);
+		object.generateMd5();
 		puts.add(connection.putObject(bucket, object));
 	    }
 	    for (Future<byte[]> put : puts)
@@ -202,15 +199,6 @@ public class LiveS3InputStreamMap extends BaseS3Map<InputStream> implements
 	}
     }
 
-    private void setSizeIfContentIsInputStream(S3Object object)
-	    throws IOException {
-	if (object.getData() instanceof InputStream) {
-	    byte[] buffer = IOUtils.toByteArray((InputStream) object.getData());
-	    object.getMetaData().setSize(buffer.length);
-	    object.setData(new ByteArrayInputStream(buffer));
-	}
-    }
-
     /*
      * (non-Javadoc)
      * 

s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java

@@ -44,10 +44,10 @@ import java.util.logging.Level;
 import java.util.logging.LogRecord;
 import java.util.logging.Logger;
 
-import org.jclouds.aws.s3.config.S3JavaUrlHttpFutureCommandClientModule;
 import org.jclouds.aws.s3.domain.S3Bucket;
 import org.jclouds.aws.s3.domain.S3Object;
 import org.jclouds.http.HttpConstants;
+import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
 import org.testng.annotations.AfterTest;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Optional;
@@ -90,6 +90,7 @@ public class S3IntegrationTest {
 		.getContents().size(), 1);
 	S3Object newObject = client.getObject(sourceBucket, key).get(10,
 		TimeUnit.SECONDS);
+	assert newObject != S3Object.NOT_FOUND;
 	assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING);
 	return newObject;
     }
@@ -147,7 +148,7 @@ public class S3IntegrationTest {
     }
 
     protected boolean debugEnabled() {
-	return true;
+	return false;
     }
 
     protected S3Context createS3Context(String AWSAccessKeyId,
@@ -171,7 +172,7 @@ public class S3IntegrationTest {
     }
 
     protected Module createHttpModule() {
-	return new S3JavaUrlHttpFutureCommandClientModule();
+	return new JavaUrlHttpFutureCommandClientModule();
     }
 
     protected void deleteEverything() throws Exception {

s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java

@@ -126,11 +126,11 @@ public class S3ObjectMapTest extends BaseS3MapTest<S3Object> {
     public void testPut() throws IOException {
 	S3Object object = new S3Object("one");
 	object.setData(IOUtils.toInputStream("apple"));
-	object.getMetaData().setSize("apple".getBytes().length);
+	object.generateMd5();
 	S3Object old = map.put(object.getKey(), object);
 	getOneReturnsAppleAndOldValueIsNull(old);
 	object.setData(IOUtils.toInputStream("bear"));
-	object.getMetaData().setSize("bear".getBytes().length);
+	object.generateMd5();
 	S3Object apple = map.put(object.getKey(), object);
 	getOneReturnsBearAndOldValueIsApple(apple);
     }

s3/src/test/java/org/jclouds/aws/s3/commands/PutObjectIntegrationTest.java

@@ -30,6 +30,7 @@ import static org.testng.Assert.assertNotNull;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URL;
 import java.util.concurrent.TimeUnit;
 
@@ -77,6 +78,9 @@ public class PutObjectIntegrationTest extends S3IntegrationTest {
 	S3Object object = new S3Object(key);
 	object.getMetaData().setContentType(type);
 	object.setData(content);
+	if (content instanceof InputStream) {
+	    object.generateMd5();
+	}
 	assertNotNull(client.putObject(bucketName, object).get(10,
 		TimeUnit.SECONDS));
 	object = client.getObject(bucketName, object.getKey()).get(10,
@@ -105,19 +109,19 @@ public class PutObjectIntegrationTest extends S3IntegrationTest {
 	object.getMetaData().setMd5(S3Utils.md5(TEST_STRING.getBytes()));
 
 	addObjectToBucket(bucketName, object);
-	object = validateContent(bucketName, key);
+	S3Object newObject = validateContent(bucketName, key);
 
 	// TODO.. why does this come back as binary/octetstring
-	assertEquals(object.getMetaData().getContentType(),
+	assertEquals(newObject.getMetaData().getContentType(),
 		"binary/octet-stream");
-	assertEquals(object.getMetaData().getContentEncoding(), "x-compress");
+	assertEquals(newObject.getMetaData().getContentEncoding(), "x-compress");
-	assertEquals(object.getMetaData().getContentDisposition(),
+	assertEquals(newObject.getMetaData().getContentDisposition(),
 		"attachment; filename=hello.txt");
-	assertEquals(object.getMetaData().getCacheControl(), "no-cache");
+	assertEquals(newObject.getMetaData().getCacheControl(), "no-cache");
-	assertEquals(object.getMetaData().getSize(), TEST_STRING.length());
+	assertEquals(newObject.getMetaData().getSize(), TEST_STRING.length());
-	assertEquals(object.getMetaData().getUserMetadata().values().iterator()
+	assertEquals(newObject.getMetaData().getUserMetadata().values()
-		.next(), "powderpuff");
+		.iterator().next(), "powderpuff");
-	assertEquals(object.getMetaData().getMd5(), S3Utils.md5(TEST_STRING
+	assertEquals(newObject.getMetaData().getMd5(), S3Utils.md5(TEST_STRING
 		.getBytes()));
     }
 

s3/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java

@@ -118,19 +118,21 @@ public class S3CommandFactoryTest {
     void testCreatePutObject() {
 	S3Object.Metadata metaData = createMock(S3Object.Metadata.class);
 	S3Object object = new S3Object(metaData);
-	object.setData("<a></a>");
+	expect(metaData.getSize()).andReturn(4L).atLeastOnce();
 	expect(metaData.getKey()).andReturn("rawr");
 	expect(metaData.getContentType()).andReturn("text/xml").atLeastOnce();
-	expect(metaData.getSize()).andReturn(4L);
 	expect(metaData.getCacheControl()).andReturn("no-cache").atLeastOnce();
-	expect(metaData.getContentDisposition()).andReturn("disposition").atLeastOnce();
+	expect(metaData.getContentDisposition()).andReturn("disposition")
-	expect(metaData.getContentEncoding()).andReturn("encoding").atLeastOnce();
+		.atLeastOnce();
-	expect(metaData.getMd5()).andReturn("encoding".getBytes()).atLeastOnce();
+	expect(metaData.getContentEncoding()).andReturn("encoding")
-	Multimap<String,String> userMdata = HashMultimap.create();
+		.atLeastOnce();
+	expect(metaData.getMd5()).andReturn("encoding".getBytes())
+		.atLeastOnce();
+	Multimap<String, String> userMdata = HashMultimap.create();
 	expect(metaData.getUserMetadata()).andReturn(userMdata).atLeastOnce();
 
-
 	replay(metaData);
+	object.setData("<a></a>");
 
 	assert commandFactory.createPutObject("test", object,
 		PutObjectOptions.NONE) != null;

s3/src/test/java/org/jclouds/aws/s3/commands/callables/DeleteBucketCallableTest.java

@@ -23,85 +23,70 @@
  */
 package org.jclouds.aws.s3.commands.callables;
 
-import static org.easymock.EasyMock.expect;
-import static org.easymock.classextension.EasyMock.createMock;
-import static org.easymock.classextension.EasyMock.replay;
-import static org.easymock.classextension.EasyMock.verify;
-import static org.testng.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.commons.io.IOUtils;
-import org.jclouds.http.HttpException;
-import org.jclouds.http.HttpFutureCommand;
-import org.jclouds.http.HttpResponse;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 @Test
 public class DeleteBucketCallableTest {
 
-    private HttpFutureCommand.ResponseCallable<Boolean> callable = null;
+    // private HttpFutureCommand.ResponseCallable<Boolean> callable = null;
-
+    //
-    @BeforeMethod
+    // @BeforeMethod
-    void setUp() {
+    // void setUp() {
-	callable = new DeleteBucketCallable();
+    // callable = new DeleteBucketCallable();
-    }
+    // }
-
+    //
-    @AfterMethod
+    // @AfterMethod
-    void tearDown() {
+    // void tearDown() {
-	callable = null;
+    // callable = null;
-    }
+    // }
-
+    //
-    @Test(expectedExceptions = HttpException.class)
+    // @Test(expectedExceptions = HttpException.class)
-    public void testExceptionWhenNoContentOn409() throws Exception {
+    // public void testExceptionWhenNoContentOn409() throws Exception {
-	HttpResponse response = createMock(HttpResponse.class);
+    // HttpResponse response = createMock(HttpResponse.class);
-	expect(response.getStatusCode()).andReturn(409).atLeastOnce();
+    // expect(response.getStatusCode()).andReturn(409).atLeastOnce();
-	expect(response.getContent()).andReturn(null);
+    // expect(response.getContent()).andReturn(null);
-	replay(response);
+    // replay(response);
-	callable.setResponse(response);
+    // callable.setResponse(response);
-	callable.call();
+    // callable.call();
-    }
+    // }
-
+    //
-    @Test
+    // @Test
-    public void testExceptionWhenIOExceptionOn409() throws ExecutionException,
+    // public void testExceptionWhenIOExceptionOn409() throws
-	    InterruptedException, TimeoutException, IOException {
+    // ExecutionException,
-	HttpResponse response = createMock(HttpResponse.class);
+    // InterruptedException, TimeoutException, IOException {
-	expect(response.getStatusCode()).andReturn(409).atLeastOnce();
+    // HttpResponse response = createMock(HttpResponse.class);
-	RuntimeException exception = new RuntimeException("bad");
+    // expect(response.getStatusCode()).andReturn(409).atLeastOnce();
-	expect(response.getContent()).andThrow(exception);
+    // RuntimeException exception = new RuntimeException("bad");
-	replay(response);
+    // expect(response.getContent()).andThrow(exception);
-	callable.setResponse(response);
+    // replay(response);
-	try {
+    // callable.setResponse(response);
-	    callable.call();
+    // try {
-	} catch (Exception e) {
+    // callable.call();
-	    assert e.equals(exception);
+    // } catch (Exception e) {
-	}
+    // assert e.equals(exception);
-	verify(response);
+    // }
-    }
+    // verify(response);
-
+    // }
-    @Test
+    //
-    public void testFalseWhenBucketNotEmptyOn409() throws Exception {
+    // @Test
-	HttpResponse response = createMock(HttpResponse.class);
+    // public void testFalseWhenBucketNotEmptyOn409() throws Exception {
-	expect(response.getStatusCode()).andReturn(409).atLeastOnce();
+    // HttpResponse response = createMock(HttpResponse.class);
-	expect(response.getContent()).andReturn(
+    // expect(response.getStatusCode()).andReturn(409).atLeastOnce();
-		IOUtils.toInputStream("BucketNotEmpty")).atLeastOnce();
+    // expect(response.getContent()).andReturn(
-	replay(response);
+    // IOUtils.toInputStream("BucketNotEmpty")).atLeastOnce();
-	callable.setResponse(response);
+    // replay(response);
-	assert !callable.call().booleanValue();
+    // callable.setResponse(response);
-	verify(response);
+    // assert !callable.call().booleanValue();
-    }
+    // verify(response);
-
+    // }
-    @Test
+    //
-    public void testResponseOk() throws Exception {
+    // @Test
-	HttpResponse response = createMock(HttpResponse.class);
+    // public void testResponseOk() throws Exception {
-	expect(response.getStatusCode()).andReturn(204).atLeastOnce();
+    // HttpResponse response = createMock(HttpResponse.class);
-	replay(response);
+    // expect(response.getStatusCode()).andReturn(204).atLeastOnce();
-	callable.setResponse(response);
+    // replay(response);
-	assertEquals(callable.call(), new Boolean(true));
+    // callable.setResponse(response);
-	verify(response);
+    // assertEquals(callable.call(), new Boolean(true));
-    }
+    // verify(response);
+    // }
 }

samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java

@@ -80,8 +80,17 @@ public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
 	assert string.indexOf("Hello World!") >= 0 : string;
     }
 
-    @Test(invocationCount = 50, enabled = true, threadPoolSize = 10)
+    @Test(invocationCount = 5, enabled = true)
-    public void testGuiceJCloudsServed() throws InterruptedException,
+    public void testGuiceJCloudsSerial() throws InterruptedException,
+	    IOException {
+	URL gurl = new URL(url, "/guice/listbuckets.s3");
+	InputStream i = gurl.openStream();
+	String string = IOUtils.toString(i);
+	assert string.indexOf("List") >= 0 : string;
+    }
+
+    @Test(invocationCount = 50, enabled = false, threadPoolSize = 10)
+    public void testGuiceJCloudsParallel() throws InterruptedException,
 	    IOException {
 	URL gurl = new URL(url, "/guice/listbuckets.s3");
 	InputStream i = gurl.openStream();

samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java

@@ -75,7 +75,6 @@ public class GuiceServletConfig extends GuiceServletContextListener {
 	} finally {
 	    IOUtils.closeQuietly(input);
 	}
-	props.setProperty("jclouds.http.sax.debug", "true");
 	return props;
     }