diff --git a/core/src/main/java/org/jclouds/command/FutureCommandClient.java b/core/src/main/java/org/jclouds/command/FutureCommandClient.java index 8cd1b1f120..0af69706d2 100644 --- a/core/src/main/java/org/jclouds/command/FutureCommandClient.java +++ b/core/src/main/java/org/jclouds/command/FutureCommandClient.java @@ -25,11 +25,10 @@ package org.jclouds.command; /** * // TODO: Adrian: Document this! - * + * * @author Adrian Cole */ public interface FutureCommandClient { @SuppressWarnings("unchecked") void submit(O operation); - void close(); } diff --git a/core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPoolClient.java b/core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPoolClient.java index cb451c918e..71ae7f346e 100644 --- a/core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPoolClient.java +++ b/core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPoolClient.java @@ -23,116 +23,122 @@ */ package org.jclouds.command.pool; -import com.google.inject.Inject; -import org.jclouds.Logger; -import org.jclouds.Utils; -import org.jclouds.command.FutureCommand; -import org.jclouds.command.FutureCommandClient; -import org.jclouds.lifecycle.BaseLifeCycle; -import org.jclouds.lifecycle.Closer; - -import java.io.IOException; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.jclouds.Logger; +import org.jclouds.Utils; +import org.jclouds.command.FutureCommand; +import org.jclouds.command.FutureCommandClient; +import org.jclouds.lifecycle.BaseLifeCycle; + +import com.google.inject.Inject; + /** * // TODO: Adrian: Document this! - * + * * @author Adrian Cole */ -public class FutureCommandConnectionPoolClient extends BaseLifeCycle implements FutureCommandClient { - @Inject private Closer closer; +public class FutureCommandConnectionPoolClient extends BaseLifeCycle + implements FutureCommandClient { private final FutureCommandConnectionPool futureCommandConnectionPool; private final BlockingQueue commandQueue; @Inject - public FutureCommandConnectionPoolClient(java.util.logging.Logger logger, ExecutorService executor, FutureCommandConnectionPool futureCommandConnectionPool, BlockingQueue commandQueue) { - super(new Logger(logger), executor, futureCommandConnectionPool); - this.futureCommandConnectionPool = futureCommandConnectionPool; - this.commandQueue = commandQueue; + public FutureCommandConnectionPoolClient(java.util.logging.Logger logger, + ExecutorService executor, + FutureCommandConnectionPool futureCommandConnectionPool, + BlockingQueue commandQueue) { + super(new Logger(logger), executor, futureCommandConnectionPool); + this.futureCommandConnectionPool = futureCommandConnectionPool; + this.commandQueue = commandQueue; } - @Override protected boolean shouldDoWork() { - return super.shouldDoWork() && futureCommandConnectionPool.getStatus().equals(Status.ACTIVE); + return super.shouldDoWork() + && futureCommandConnectionPool.getStatus() + .equals(Status.ACTIVE); } @Override protected void doShutdown() { - if (exception == null && futureCommandConnectionPool.getException() != null) - exception = futureCommandConnectionPool.getException(); - while (!commandQueue.isEmpty()) { - FutureCommand command = commandQueue.remove(); - if (command != null) { - if (exception != null) - command.setException(exception); - else - command.cancel(true); - } - } + if (exception == null + && futureCommandConnectionPool.getException() != null) + exception = futureCommandConnectionPool.getException(); + while (!commandQueue.isEmpty()) { + FutureCommand command = commandQueue.remove(); + if (command != null) { + if (exception != null) + command.setException(exception); + else + command.cancel(true); + } + } } + @Override protected void doWork() throws InterruptedException { - FutureCommand command = commandQueue.poll(1, TimeUnit.SECONDS); - if (command != null) { - try { - invoke(command); - } catch (Exception e) { - Utils.rethrowIfRuntimeOrSameType(e); - logger.error(e, "Error processing command %s", command); - } - } + FutureCommand command = commandQueue.poll(1, TimeUnit.SECONDS); + if (command != null) { + try { + invoke(command); + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + logger.error(e, "Error processing command %s", command); + } + } } - public void submit(O operation) { - exceptionIfNotActive(); - commandQueue.add(operation); + exceptionIfNotActive(); + commandQueue.add(operation); } protected void invoke(O operation) { - exceptionIfNotActive(); - FutureCommandConnectionHandle connectionHandle = null; - try { - connectionHandle = futureCommandConnectionPool.getHandle(operation); - } catch (InterruptedException e) { - logger.warn(e, "Interrupted getting a connection for operation %1s; retrying", operation); - commandQueue.add(operation); - return; - } catch (TimeoutException e) { - logger.warn(e, "Timeout getting a connection for operation %1s; retrying", operation); - commandQueue.add(operation); - return; - } + exceptionIfNotActive(); + FutureCommandConnectionHandle connectionHandle = null; + try { + connectionHandle = futureCommandConnectionPool.getHandle(operation); + } catch (InterruptedException e) { + logger + .warn( + e, + "Interrupted getting a connection for operation %1s; retrying", + operation); + commandQueue.add(operation); + return; + } catch (TimeoutException e) { + logger.warn(e, + "Timeout getting a connection for operation %1s; retrying", + operation); + commandQueue.add(operation); + return; + } - if (connectionHandle == null) { - logger.error("Failed to obtain connection for operation %1s; retrying", operation); - commandQueue.add(operation); - return; - } - connectionHandle.startConnection(); + if (connectionHandle == null) { + logger.error( + "Failed to obtain connection for operation %1s; retrying", + operation); + commandQueue.add(operation); + return; + } + connectionHandle.startConnection(); } @Override public String toString() { - final StringBuilder sb = new StringBuilder(); - sb.append("FutureCommandConnectionPoolClient"); - sb.append("{status=").append(status); - sb.append(", commandQueue=").append((commandQueue != null) ? commandQueue.size() : 0); - sb.append(", futureCommandConnectionPool=").append(futureCommandConnectionPool); - sb.append('}'); - return sb.toString(); + final StringBuilder sb = new StringBuilder(); + sb.append("FutureCommandConnectionPoolClient"); + sb.append("{status=").append(status); + sb.append(", commandQueue=").append( + (commandQueue != null) ? commandQueue.size() : 0); + sb.append(", futureCommandConnectionPool=").append( + futureCommandConnectionPool); + sb.append('}'); + return sb.toString(); } - public void close(){ - try { - closer.close(); - } catch (IOException e) { - e.printStackTrace(); // TODO: Adrian: Customise this generated block - } - } } - diff --git a/core/src/main/java/org/jclouds/command/pool/PoolConstants.java b/core/src/main/java/org/jclouds/command/pool/PoolConstants.java new file mode 100644 index 0000000000..2e27309b02 --- /dev/null +++ b/core/src/main/java/org/jclouds/command/pool/PoolConstants.java @@ -0,0 +1,37 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.command.pool; + +/** + * // TODO: Adrian: Document this! + * + * @author Adrian Cole + */ +public interface PoolConstants { + public static final String PROPERTY_POOL_MAX_CONNECTIONS = "jclouds.pool.max_connections"; + public static final String PROPERTY_POOL_IO_WORKER_THREADS = "jclouds.http.pool.io_worker_threads"; + public static final String PROPERTY_POOL_REQUEST_INVOKER_THREADS = "jclouds.http.pool.request_invoker_threads"; + public static final String PROPERTY_POOL_MAX_SESSION_FAILURES = "jclouds.http.pool.max_session_failures"; + public static final String PROPERTY_POOL_MAX_CONNECTION_REUSE = "jclouds.http.pool.max_connection_reuse"; +} diff --git a/core/src/main/java/org/jclouds/command/pool/config/FutureCommandConnectionPoolClientModule.java b/core/src/main/java/org/jclouds/command/pool/config/FutureCommandConnectionPoolClientModule.java index 3132a8477c..05f05e4696 100644 --- a/core/src/main/java/org/jclouds/command/pool/config/FutureCommandConnectionPoolClientModule.java +++ b/core/src/main/java/org/jclouds/command/pool/config/FutureCommandConnectionPoolClientModule.java @@ -27,6 +27,7 @@ import com.google.inject.*; import com.google.inject.name.Named; import org.jclouds.command.FutureCommand; import org.jclouds.lifecycle.config.LifeCycleModule; +import org.jclouds.command.pool.PoolConstants; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -50,8 +51,7 @@ public abstract class FutureCommandConnectionPoolClientModule extends Abstrac @Provides @Singleton - public abstract BlockingQueue provideAvailablePool(@Named("jclouds.pool.max_connections") int max) throws Exception; - + public abstract BlockingQueue provideAvailablePool(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception; /** * controls production and destruction of real connections. *

@@ -64,7 +64,7 @@ public abstract class FutureCommandConnectionPoolClientModule extends Abstrac */ @Provides @Singleton - public Semaphore provideTotalConnectionSemaphore(@Named("jclouds.pool.max_connections") int max) throws Exception { + public Semaphore provideTotalConnectionSemaphore(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception { return new Semaphore(max, true); } } diff --git a/core/src/main/java/org/jclouds/http/HttpConstants.java b/core/src/main/java/org/jclouds/http/HttpConstants.java index c7416b4f0e..71d176fada 100644 --- a/core/src/main/java/org/jclouds/http/HttpConstants.java +++ b/core/src/main/java/org/jclouds/http/HttpConstants.java @@ -25,10 +25,10 @@ package org.jclouds.http; /** * // TODO: Adrian: Document this! - * + * * @author Adrian Cole */ -public class HttpConstants { +public interface HttpConstants { public static final String CONTENT_LENGTH = "Content-Length"; public static final String CONTENT_TYPE = "Content-Type"; public static final String HOST = "Host"; @@ -36,4 +36,7 @@ public class HttpConstants { public static final String BINARY = "application/octet-stream"; public static final String PLAIN = "text/plain"; public static final String TRANSFER_ENCODING = "Transfer-Encoding"; + public static final String PROPERTY_HTTP_SECURE = "jclouds.http.secure"; + public static final String PROPERTY_HTTP_PORT = "jclouds.http.port"; + public static final String PROPERTY_HTTP_ADDRESS = "jclouds.http.address"; } diff --git a/core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java b/core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java index 0e1afdc833..a412732f8f 100644 --- a/core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java +++ b/core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java @@ -23,15 +23,16 @@ */ package org.jclouds.http; -import com.google.inject.Inject; +import java.util.List; + import org.jclouds.command.FutureCommand; import org.jclouds.command.FutureCommandClient; -import java.util.List; +import com.google.inject.Inject; /** * // TODO: Adrian: Document this! - * + * * @author Adrian Cole */ public interface HttpFutureCommandClient extends FutureCommandClient { @@ -41,6 +42,4 @@ public interface HttpFutureCommandClient extends FutureCommandClient { void setRequestFilters(List requestFilters); void submit(O operation); - - void close(); } diff --git a/core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java b/core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java index e17177ddda..87fd56409e 100644 --- a/core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java +++ b/core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java @@ -25,6 +25,7 @@ package org.jclouds.http; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -82,35 +83,40 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient { HttpResponse response = getResponse(connection); logger.trace("%1s - received response %2s", target, response); + operation.getResponseFuture().setResponse(response); + operation.getResponseFuture().run(); + } catch (FileNotFoundException e) { + HttpResponse response = new HttpResponse(); + response.setStatusCode(404); operation.getResponseFuture().setResponse(response); operation.getResponseFuture().run(); } catch (Exception e) { if (connection != null) { - InputStream errorStream = connection.getErrorStream(); - if (errorStream != null) { - try { - String errorMessage = Utils.toStringAndClose(connection - .getErrorStream()); - logger.error(e, - "error encountered during the exception: %1s", - errorMessage); - } catch (IOException e1) { + StringBuilder errors = new StringBuilder(); + try { + for (InputStream in : new InputStream[] { + connection.getErrorStream(), + connection.getInputStream() }) { + if (in != null) { + errors.append(Utils.toStringAndClose(in)).append( + "\n"); + } } + logger.error(e, + "error encountered during the exception: %1s", + errors.toString()); + } catch (IOException e2) { } - } operation.setException(e); } finally { // DO NOT disconnect, as it will also close the unconsumed // outputStream from above. - // connection.disconnect(); + if (request.getMethod().equals("HEAD")) + connection.disconnect(); } } - public void close() { - // Nothing to stop; - } - private HttpResponse getResponse(HttpURLConnection connection) throws IOException { HttpResponse response = new HttpResponse(); @@ -121,9 +127,11 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient { } response.setMessage(connection.getResponseMessage()); - response.setContent(connection.getInputStream()); - response.setContentType(connection - .getHeaderField(HttpConstants.CONTENT_TYPE)); + if (!connection.getRequestMethod().equals("HEAD")) { + response.setContent(connection.getInputStream()); + response.setContentType(connection + .getHeaderField(HttpConstants.CONTENT_TYPE)); + } return response; } diff --git a/core/src/main/java/org/jclouds/http/config/JavaUrlHttpFutureCommandClientModule.java b/core/src/main/java/org/jclouds/http/config/JavaUrlHttpFutureCommandClientModule.java index 2f4fb68f18..a833a8050c 100644 --- a/core/src/main/java/org/jclouds/http/config/JavaUrlHttpFutureCommandClientModule.java +++ b/core/src/main/java/org/jclouds/http/config/JavaUrlHttpFutureCommandClientModule.java @@ -23,36 +23,41 @@ */ package org.jclouds.http.config; -import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import org.jclouds.http.HttpFutureCommandClient; -import org.jclouds.http.JavaUrlHttpFutureCommandClient; - import java.net.MalformedURLException; import java.net.URL; +import org.jclouds.http.HttpConstants; +import org.jclouds.http.HttpFutureCommandClient; +import org.jclouds.http.JavaUrlHttpFutureCommandClient; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.name.Named; + /** - * // TODO: Adrian: Document this! - * + * Configures {@link JavaUrlHttpFutureCommandClient}. + * * @author Adrian Cole */ public class JavaUrlHttpFutureCommandClientModule extends AbstractModule { @Override protected void configure() { - //note this is not threadsafe, so it cannot be singleton - bind(HttpFutureCommandClient.class).to(JavaUrlHttpFutureCommandClient.class); + // note this is not threadsafe, so it cannot be singleton + bind(HttpFutureCommandClient.class).to( + JavaUrlHttpFutureCommandClient.class); } @Singleton @Provides - protected URL provideAddress(@Named("jclouds.http.address") String address, @Named("jclouds.http.port") int port, @Named("jclouds.http.secure") boolean isSecure) throws MalformedURLException { + protected URL provideAddress( + @Named(HttpConstants.PROPERTY_HTTP_ADDRESS) String address, + @Named(HttpConstants.PROPERTY_HTTP_PORT) int port, + @Named(HttpConstants.PROPERTY_HTTP_SECURE) boolean isSecure) + throws MalformedURLException { - return new URL(isSecure ? "https" : "http", address, port, "/"); + return new URL(isSecure ? "https" : "http", address, port, "/"); } - } \ No newline at end of file diff --git a/core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java b/core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java index 4a79d0eaf7..ab006cd589 100644 --- a/core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java +++ b/core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java @@ -40,6 +40,7 @@ import org.jclouds.http.commands.GetString; import org.jclouds.http.commands.Head; import org.jclouds.http.commands.callables.xml.ParseSax; import org.jclouds.http.commands.config.HttpCommandsModule; +import org.jclouds.lifecycle.Closer; import org.mortbay.jetty.Handler; import org.mortbay.jetty.Request; import org.mortbay.jetty.Server; @@ -69,6 +70,7 @@ public abstract class BaseHttpFutureCommandClientTest { protected CommandFactory factory; protected HttpFutureCommandClient client; protected Injector injector; + private Closer closer; @BeforeClass @Parameters( { "test-jetty-port" }) @@ -95,9 +97,9 @@ public abstract class BaseHttpFutureCommandClientTest { server.setHandler(handler); server.start(); final Properties properties = new Properties(); - properties.put("jclouds.http.address", "localhost"); - properties.put("jclouds.http.port", testPort + ""); - properties.put("jclouds.http.secure", "false"); + properties.put(HttpConstants.PROPERTY_HTTP_ADDRESS, "localhost"); + properties.put(HttpConstants.PROPERTY_HTTP_PORT, testPort + ""); + properties.put(HttpConstants.PROPERTY_HTTP_SECURE, "false"); addConnectionProperties(properties); final List filters = new ArrayList( 1); @@ -123,6 +125,7 @@ public abstract class BaseHttpFutureCommandClientTest { }); factory = injector.getInstance(CommandFactory.class); client = injector.getInstance(HttpFutureCommandClient.class); + closer = injector.getInstance(Closer.class); assert client != null; } @@ -132,7 +135,7 @@ public abstract class BaseHttpFutureCommandClientTest { @AfterClass public void tearDownJetty() throws Exception { - client.close(); + closer.close(); server.stop(); } diff --git a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java index 02e862dd7c..1e51b20263 100644 --- a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java +++ b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java @@ -61,7 +61,7 @@ public class HttpNioUtils { String contentType, long length) { if (content instanceof InputStream) { InputStream inputStream = (InputStream) content; - if (length <= 0) + if (length == -1) throw new IllegalArgumentException( "you must specify size when content is an InputStream"); InputStreamEntity entity = new InputStreamEntity(inputStream, diff --git a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/HttpNioConnectionPoolClientModule.java b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/HttpNioConnectionPoolClientModule.java index 2a96619e50..769add32e2 100644 --- a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/HttpNioConnectionPoolClientModule.java +++ b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/HttpNioConnectionPoolClientModule.java @@ -23,49 +23,52 @@ */ package org.jclouds.http.httpnio.config; -import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; -import com.google.inject.TypeLiteral; -import com.google.inject.name.Named; +import java.net.InetSocketAddress; import org.apache.http.nio.NHttpConnection; import org.jclouds.command.pool.FutureCommandConnectionRetry; +import org.jclouds.http.HttpConstants; import org.jclouds.http.HttpFutureCommandClient; import org.jclouds.http.httpnio.config.internal.NonSSLHttpNioConnectionPoolClientModule; import org.jclouds.http.httpnio.config.internal.SSLHttpNioConnectionPoolClientModule; import org.jclouds.http.httpnio.pool.HttpNioConnectionPoolClient; import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionRetry; -import java.net.InetSocketAddress; +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.TypeLiteral; +import com.google.inject.name.Named; /** - * // TODO: Adrian: Document this! - * + * Configures {@link HttpNioConnectionPoolClient} + * * @author Adrian Cole */ public class HttpNioConnectionPoolClientModule extends AbstractModule { - @Named("jclouds.http.secure") + @Named(HttpConstants.PROPERTY_HTTP_SECURE) boolean isSecure; @Override protected void configure() { - requestInjection(this); - //TODO test... - if (isSecure) - install(new SSLHttpNioConnectionPoolClientModule()); - else - install(new NonSSLHttpNioConnectionPoolClientModule()); - bind(new TypeLiteral>(){}).to(HttpNioFutureCommandConnectionRetry.class); - bind(HttpFutureCommandClient.class).to(HttpNioConnectionPoolClient.class); + requestInjection(this); + if (isSecure) + install(new SSLHttpNioConnectionPoolClientModule()); + else + install(new NonSSLHttpNioConnectionPoolClientModule()); + bind(new TypeLiteral>() { + }).to(HttpNioFutureCommandConnectionRetry.class); + bind(HttpFutureCommandClient.class).to( + HttpNioConnectionPoolClient.class); } @Singleton @Provides - protected InetSocketAddress provideAddress(@Named("jclouds.http.address") String address, @Named("jclouds.http.port") int port) { - return new InetSocketAddress(address, port); + protected InetSocketAddress provideAddress( + @Named(HttpConstants.PROPERTY_HTTP_ADDRESS) String address, + @Named(HttpConstants.PROPERTY_HTTP_PORT) int port) { + return new InetSocketAddress(address, port); } } diff --git a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/internal/BaseHttpNioConnectionPoolClientModule.java b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/internal/BaseHttpNioConnectionPoolClientModule.java index e72cfa8acb..5935fbbc1f 100644 --- a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/internal/BaseHttpNioConnectionPoolClientModule.java +++ b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/internal/BaseHttpNioConnectionPoolClientModule.java @@ -45,6 +45,7 @@ import org.apache.http.params.CoreProtocolPNames; import org.apache.http.params.HttpParams; import org.apache.http.protocol.*; import org.jclouds.command.pool.FutureCommandConnectionRetry; +import org.jclouds.command.pool.PoolConstants; import org.jclouds.command.pool.config.FutureCommandConnectionPoolClientModule; import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionHandle; import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionPool; @@ -114,7 +115,7 @@ public abstract class BaseHttpNioConnectionPoolClientModule extends FutureComman } @Override - public BlockingQueue provideAvailablePool(@Named("jclouds.pool.max_connections") int max) throws Exception { + public BlockingQueue provideAvailablePool(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception { return new ArrayBlockingQueue(max, true); } @@ -125,7 +126,7 @@ public abstract class BaseHttpNioConnectionPoolClientModule extends FutureComman @Provides @Singleton - public DefaultConnectingIOReactor provideDefaultConnectingIOReactor(@Named("jclouds.http.pool.io_worker_threads") int ioWorkerThreads, HttpParams params) throws IOReactorException { + public DefaultConnectingIOReactor provideDefaultConnectingIOReactor(@Named(PoolConstants.PROPERTY_POOL_IO_WORKER_THREADS) int ioWorkerThreads, HttpParams params) throws IOReactorException { return new DefaultConnectingIOReactor(ioWorkerThreads, params); } diff --git a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandConnectionPool.java b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandConnectionPool.java index 8cae28d82b..0192c1897f 100644 --- a/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandConnectionPool.java +++ b/extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandConnectionPool.java @@ -23,8 +23,13 @@ */ package org.jclouds.http.httpnio.pool; -import com.google.inject.Inject; -import com.google.inject.name.Named; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + import org.apache.http.HttpException; import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; import org.apache.http.nio.NHttpConnection; @@ -38,21 +43,18 @@ import org.jclouds.Logger; import org.jclouds.command.FutureCommand; import org.jclouds.command.pool.FutureCommandConnectionPool; import org.jclouds.command.pool.FutureCommandConnectionRetry; +import org.jclouds.command.pool.PoolConstants; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; - +import com.google.inject.Inject; +import com.google.inject.name.Named; /** - * // TODO: Adrian: Document this! - * + * Connection Pool for HTTP requests that utilizes Apache HTTPNio + * * @author Adrian Cole */ -public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionPool implements EventListener { +public class HttpNioFutureCommandConnectionPool extends + FutureCommandConnectionPool implements EventListener { private final NHttpClientConnectionPoolSessionRequestCallback sessionCallback; private final DefaultConnectingIOReactor ioReactor; @@ -61,153 +63,182 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP private final int maxSessionFailures; @Inject - public HttpNioFutureCommandConnectionPool(java.util.logging.Logger logger, ExecutorService executor, Semaphore allConnections, BlockingQueue available, AsyncNHttpClientHandler clientHandler, DefaultConnectingIOReactor ioReactor, IOEventDispatch dispatch, FutureCommandConnectionHandleFactory requestHandleFactory, InetSocketAddress target, FutureCommandConnectionRetry futureCommandConnectionRetry, @Named("jclouds.http.pool.max_connection_reuse") int maxConnectionReuse, @Named("jclouds.http.pool.max_session_failures") int maxSessionFailures) { - super(new Logger(logger), executor, futureCommandConnectionRetry, allConnections, requestHandleFactory, maxConnectionReuse, available); - this.ioReactor = ioReactor; - this.dispatch = dispatch; - this.target = target; - this.maxSessionFailures = maxSessionFailures; - this.sessionCallback = new NHttpClientConnectionPoolSessionRequestCallback(); - clientHandler.setEventListener(this); + public HttpNioFutureCommandConnectionPool( + java.util.logging.Logger logger, + ExecutorService executor, + Semaphore allConnections, + BlockingQueue available, + AsyncNHttpClientHandler clientHandler, + DefaultConnectingIOReactor ioReactor, + IOEventDispatch dispatch, + FutureCommandConnectionHandleFactory requestHandleFactory, + InetSocketAddress target, + FutureCommandConnectionRetry futureCommandConnectionRetry, + @Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTION_REUSE) int maxConnectionReuse, + @Named(PoolConstants.PROPERTY_POOL_MAX_SESSION_FAILURES) int maxSessionFailures) { + super(new Logger(logger), executor, futureCommandConnectionRetry, + allConnections, requestHandleFactory, maxConnectionReuse, + available); + this.ioReactor = ioReactor; + this.dispatch = dispatch; + this.target = target; + this.maxSessionFailures = maxSessionFailures; + this.sessionCallback = new NHttpClientConnectionPoolSessionRequestCallback(); + clientHandler.setEventListener(this); } @Override public void start() { - synchronized (this.statusLock) { - if (this.status.compareTo(Status.INACTIVE) == 0) { - executor.execute(new Runnable() { - public void run() { - try { - ioReactor.execute(dispatch); - } catch (IOException e) { - exception = e; - logger.error(e, "Error dispatching %1s", dispatch); - status = Status.SHUTDOWN_REQUEST; - } - } - }); - } - super.start(); - } + synchronized (this.statusLock) { + if (this.status.compareTo(Status.INACTIVE) == 0) { + executor.execute(new Runnable() { + public void run() { + try { + ioReactor.execute(dispatch); + } catch (IOException e) { + exception = e; + logger.error(e, "Error dispatching %1s", dispatch); + status = Status.SHUTDOWN_REQUEST; + } + } + }); + } + super.start(); + } } public void shutdownReactor(long waitMs) { - try { - this.ioReactor.shutdown(waitMs); - } catch (IOException e) { - logger.error(e, "Error shutting down reactor"); - } + try { + this.ioReactor.shutdown(waitMs); + } catch (IOException e) { + logger.error(e, "Error shutting down reactor"); + } } - public boolean connectionValid(NHttpConnection conn) { - return conn.isOpen() && !conn.isStale() && conn.getMetrics().getRequestCount() < maxConnectionReuse; + return conn.isOpen() && !conn.isStale() + && conn.getMetrics().getRequestCount() < maxConnectionReuse; } protected void doWork() throws Exception { - createNewConnection(); + createNewConnection(); } @Override protected void doShutdown() { - // Give the I/O reactor 10 sec to shut down - shutdownReactor(10000); + // Give the I/O reactor 10 sec to shut down + shutdownReactor(10000); } protected void createNewConnection() throws InterruptedException { - boolean acquired = allConnections.tryAcquire(1, TimeUnit.SECONDS); - if (acquired) { - if (shouldDoWork()) { - logger.debug("%1s - opening new connection", target); - ioReactor.connect(target, null, null, sessionCallback); - } else { - allConnections.release(); - } - } + boolean acquired = allConnections.tryAcquire(1, TimeUnit.SECONDS); + if (acquired) { + if (shouldDoWork()) { + logger.debug("%1s - opening new connection", target); + ioReactor.connect(target, null, null, sessionCallback); + } else { + allConnections.release(); + } + } } @Override protected boolean shouldDoWork() { - return super.shouldDoWork() && ioReactor.getStatus().equals(IOReactorStatus.ACTIVE); + return super.shouldDoWork() + && ioReactor.getStatus().equals(IOReactorStatus.ACTIVE); } - class NHttpClientConnectionPoolSessionRequestCallback implements SessionRequestCallback { + class NHttpClientConnectionPoolSessionRequestCallback implements + SessionRequestCallback { - public void completed(SessionRequest request) { - logger.trace("%1s - %2s - operation complete", request, request.getAttachment()); - } + public void completed(SessionRequest request) { + logger.trace("%1s - %2s - operation complete", request, request + .getAttachment()); + } - public void cancelled(SessionRequest request) { - logger.info("%1s - %2s - operation cancelled", request, request.getAttachment()); - releaseConnectionAndCancelResponse(request); - } + public void cancelled(SessionRequest request) { + logger.info("%1s - %2s - operation cancelled", request, request + .getAttachment()); + releaseConnectionAndCancelResponse(request); + } - private void releaseConnectionAndCancelResponse(SessionRequest request) { - allConnections.release(); - FutureCommand frequest = (FutureCommand) request.getAttachment(); - if (frequest != null) { - frequest.cancel(true); - } - } + private void releaseConnectionAndCancelResponse(SessionRequest request) { + allConnections.release(); + FutureCommand frequest = (FutureCommand) request + .getAttachment(); + if (frequest != null) { + frequest.cancel(true); + } + } - private void releaseConnectionAndSetResponseException(SessionRequest request, Exception e) { - allConnections.release(); - FutureCommand frequest = (FutureCommand) request.getAttachment(); - if (frequest != null) { - frequest.setException(e); - } - } + private void releaseConnectionAndSetResponseException( + SessionRequest request, Exception e) { + allConnections.release(); + FutureCommand frequest = (FutureCommand) request + .getAttachment(); + if (frequest != null) { + frequest.setException(e); + } + } - public void failed(SessionRequest request) { - int count = currentSessionFailures.getAndIncrement(); - logger.error(request.getException(), "%1s - %2s - operation failed", request, request.getAttachment()); - releaseConnectionAndSetResponseException(request, request.getException()); - if (count >= maxSessionFailures) { - exception = request.getException(); - } + public void failed(SessionRequest request) { + int count = currentSessionFailures.getAndIncrement(); + logger.error(request.getException(), + "%1s - %2s - operation failed", request, request + .getAttachment()); + releaseConnectionAndSetResponseException(request, request + .getException()); + if (count >= maxSessionFailures) { + exception = request.getException(); + } - } + } - public void timeout(SessionRequest request) { - logger.warn("%1s - %2s - operation timed out", request, request.getAttachment()); - releaseConnectionAndCancelResponse(request); - } + public void timeout(SessionRequest request) { + logger.warn("%1s - %2s - operation timed out", request, request + .getAttachment()); + releaseConnectionAndCancelResponse(request); + } } - public void connectionOpen(NHttpConnection conn) { - conn.setSocketTimeout(0); - available.offer(conn); - logger.trace("%1s - %2d - open", conn, conn.hashCode()); + conn.setSocketTimeout(0); + available.offer(conn); + logger.trace("%1s - %2d - open", conn, conn.hashCode()); } - public void connectionTimeout(NHttpConnection conn) { - logger.warn("%1s - %2d - timeout %2d", conn, conn.hashCode(), conn.getSocketTimeout()); - allConnections.release(); - futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn); + logger.warn("%1s - %2d - timeout %2d", conn, conn.hashCode(), conn + .getSocketTimeout()); + allConnections.release(); + futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn); } public void connectionClosed(NHttpConnection conn) { - allConnections.release(); - logger.trace("%1s - %2d - closed", conn, conn.hashCode()); + allConnections.release(); + logger.trace("%1s - %2d - closed", conn, conn.hashCode()); } public void fatalIOException(IOException ex, NHttpConnection conn) { - exception = ex; - logger.error(ex, "%1s - %2d - %3s - pool error", conn, conn.hashCode(), target); - futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn); + exception = ex; + logger.error(ex, "%1s - %2d - %3s - pool error", conn, conn.hashCode(), + target); + futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn); } public void fatalProtocolException(HttpException ex, NHttpConnection conn) { - exception = ex; - logger.error(ex, "%1s - %2d - %3s - http error", conn, conn.hashCode(), target); - fatalException(ex, conn); + exception = ex; + logger.error(ex, "%1s - %2d - %3s - http error", conn, conn.hashCode(), + target); + fatalException(ex, conn); } - public static interface FutureCommandConnectionHandleFactory extends FutureCommandConnectionPool.FutureCommandConnectionHandleFactory { - HttpNioFutureCommandConnectionHandle create(FutureCommand command, NHttpConnection conn); + public static interface FutureCommandConnectionHandleFactory + extends + FutureCommandConnectionPool.FutureCommandConnectionHandleFactory { + HttpNioFutureCommandConnectionHandle create(FutureCommand command, + NHttpConnection conn); } } \ No newline at end of file diff --git a/extensions/httpnio/src/test/java/org/jclouds/http/httpnio/pool/HttpNioConnectionPoolFutureCommandClientTest.java b/extensions/httpnio/src/test/java/org/jclouds/http/httpnio/pool/HttpNioConnectionPoolFutureCommandClientTest.java index ea5a0af925..51510f2820 100644 --- a/extensions/httpnio/src/test/java/org/jclouds/http/httpnio/pool/HttpNioConnectionPoolFutureCommandClientTest.java +++ b/extensions/httpnio/src/test/java/org/jclouds/http/httpnio/pool/HttpNioConnectionPoolFutureCommandClientTest.java @@ -24,6 +24,8 @@ package org.jclouds.http.httpnio.pool; import com.google.inject.Module; + +import org.jclouds.command.pool.PoolConstants; import org.jclouds.http.BaseHttpFutureCommandClientTest; import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule; import org.testng.annotations.Test; @@ -32,21 +34,27 @@ import java.util.Properties; /** * // TODO: Adrian: Document this! - * + * * @author Adrian Cole */ @Test -public class HttpNioConnectionPoolFutureCommandClientTest extends BaseHttpFutureCommandClientTest { +public class HttpNioConnectionPoolFutureCommandClientTest extends + BaseHttpFutureCommandClientTest { protected void addConnectionProperties(Properties properties) { - properties.setProperty("jclouds.http.pool.max_connection_reuse", "75"); - properties.setProperty("jclouds.http.pool.max_session_failures", "2"); - properties.setProperty("jclouds.http.pool.request_invoker_threads", "1"); - properties.setProperty("jclouds.http.pool.io_worker_threads", "2"); - properties.setProperty("jclouds.pool.max_connections", "12"); + properties.setProperty( + PoolConstants.PROPERTY_POOL_MAX_CONNECTION_REUSE, "75"); + properties.setProperty( + PoolConstants.PROPERTY_POOL_MAX_SESSION_FAILURES, "2"); + properties.setProperty( + PoolConstants.PROPERTY_POOL_REQUEST_INVOKER_THREADS, "1"); + properties.setProperty(PoolConstants.PROPERTY_POOL_IO_WORKER_THREADS, + "2"); + properties.setProperty(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS, + "12"); } protected Module createClientModule() { - return new HttpNioConnectionPoolClientModule(); + return new HttpNioConnectionPoolClientModule(); } } \ No newline at end of file diff --git a/extensions/s3nio/pom.xml b/extensions/s3nio/pom.xml index ddd5993fd7..82042a4a2e 100644 --- a/extensions/s3nio/pom.xml +++ b/extensions/s3nio/pom.xml @@ -46,11 +46,48 @@ http://jclouds.googlecode.com/svn/trunk/extensions/s3nio + + + + + ${project.groupId} jclouds-httpnio ${project.version} + + ${project.groupId} + jclouds-s3 + ${project.version} + + + ${project.groupId} + jclouds-s3 + ${project.version} + test-jar + test + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + jclouds.aws.accesskeyid + ${jclouds.aws.accesskeyid} + + + jclouds.aws.secretaccesskey + ${jclouds.aws.secretaccesskey} + + + + + + diff --git a/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java b/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java new file mode 100644 index 0000000000..edebe44425 --- /dev/null +++ b/extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java @@ -0,0 +1,61 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.aws.s3.nio; + +import java.util.Properties; + +import org.jclouds.aws.s3.S3ObjectMapTest; +import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule; +import org.testng.annotations.Test; + +import com.google.inject.Module; + +/** + * // TODO: Adrian: Document this! + * + * @author Adrian Cole + */ +@Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest") +public class NioS3ObjectMapTest extends S3ObjectMapTest { + + @Override + protected Properties buildS3Properties(String AWSAccessKeyId, + String AWSSecretAccessKey) { + Properties properties = super.buildS3Properties(AWSAccessKeyId, + AWSSecretAccessKey); + properties.setProperty("jclouds.http.pool.max_connection_reuse", "75"); + properties.setProperty("jclouds.http.pool.max_session_failures", "2"); + properties + .setProperty("jclouds.http.pool.request_invoker_threads", "1"); + properties.setProperty("jclouds.http.pool.io_worker_threads", "2"); + properties.setProperty("jclouds.pool.max_connections", "12"); + return properties; + } + + @Override + protected Module createHttpModule() { + return new S3HttpNioConnectionPoolClientModule(); + } + +} diff --git a/pom.xml b/pom.xml index 31c3c539f6..80bc03e86d 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,6 @@ extensions/s3nio s3 s3/perftest - samples/googleappengine diff --git a/repo/com/google/code/guice/guice-assistedinject/maven-metadata-local.xml b/repo/com/google/code/guice/guice-assistedinject/maven-metadata-local.xml index 931b9a48d7..a04703a131 100644 --- a/repo/com/google/code/guice/guice-assistedinject/maven-metadata-local.xml +++ b/repo/com/google/code/guice/guice-assistedinject/maven-metadata-local.xml @@ -1,4 +1,29 @@ + com.google.code.guice diff --git a/repo/com/google/code/guice/guice-multibindings/maven-metadata-local.xml b/repo/com/google/code/guice/guice-multibindings/maven-metadata-local.xml index 4f2c466a25..e6d532eab1 100644 --- a/repo/com/google/code/guice/guice-multibindings/maven-metadata-local.xml +++ b/repo/com/google/code/guice/guice-multibindings/maven-metadata-local.xml @@ -1,4 +1,29 @@ + com.google.code.guice diff --git a/repo/com/google/code/guice/guice-servlet/maven-metadata-local.xml b/repo/com/google/code/guice/guice-servlet/maven-metadata-local.xml index 4967398d31..a422bf4438 100644 --- a/repo/com/google/code/guice/guice-servlet/maven-metadata-local.xml +++ b/repo/com/google/code/guice/guice-servlet/maven-metadata-local.xml @@ -1,4 +1,29 @@ + com.google.code.guice diff --git a/repo/com/google/code/guice/guice/maven-metadata-local.xml b/repo/com/google/code/guice/guice/maven-metadata-local.xml index d5a58d0acf..bab56ada4f 100644 --- a/repo/com/google/code/guice/guice/maven-metadata-local.xml +++ b/repo/com/google/code/guice/guice/maven-metadata-local.xml @@ -1,4 +1,29 @@ + com.google.code.guice diff --git a/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java b/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java index 9eae6ed982..cd04270e22 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java +++ b/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ExecutionException; +import org.jclouds.aws.s3.S3Constants; import org.testng.annotations.BeforeTest; import org.testng.annotations.Optional; import org.testng.annotations.Parameters; @@ -46,7 +47,7 @@ public class AmazonPerformance extends BasePerformance { @Override @BeforeTest - @Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" }) + @Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY }) protected void setUpClient(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception { super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey); amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey, diff --git a/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java b/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java index 04d4531778..c57d9914fb 100644 --- a/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java +++ b/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java @@ -34,6 +34,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; +import org.jclouds.aws.s3.S3Constants; import org.jclouds.aws.s3.S3IntegrationTest; import org.jclouds.aws.s3.domain.S3Bucket; import org.testng.annotations.AfterTest; @@ -79,7 +80,7 @@ public abstract class BasePerformance extends S3IntegrationTest { @Override @BeforeTest - @Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" }) + @Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY }) protected void setUpClient(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception { super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey); diff --git a/s3/pom.xml b/s3/pom.xml index a1f697ad97..49731d6b75 100644 --- a/s3/pom.xml +++ b/s3/pom.xml @@ -67,6 +67,11 @@ bcprov-jdk15 140 + + eu.medsea.mimeutil + mime-util + 1.3 + diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3Connection.java b/s3/src/main/java/org/jclouds/aws/s3/S3Connection.java index 318a7c6fd3..27a931280a 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3Connection.java +++ b/s3/src/main/java/org/jclouds/aws/s3/S3Connection.java @@ -23,20 +23,18 @@ */ package org.jclouds.aws.s3; -import org.jclouds.aws.s3.domain.S3Object; -import org.jclouds.aws.s3.domain.S3Bucket; - -import java.io.IOException; -import java.io.Closeable; import java.util.List; import java.util.concurrent.Future; +import org.jclouds.aws.s3.domain.S3Bucket; +import org.jclouds.aws.s3.domain.S3Object; + /** * // TODO: Adrian: Document this! * * @author Adrian Cole */ -public interface S3Connection extends Closeable { +public interface S3Connection { Future getObject(S3Bucket s3Bucket, String key); Future headObject(S3Bucket s3Bucket, String key); @@ -56,6 +54,4 @@ public interface S3Connection extends Closeable { Future getBucket(S3Bucket s3Bucket); Future> getBuckets(); - - public void close() throws IOException; } diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3ConnectionFactory.java b/s3/src/main/java/org/jclouds/aws/s3/S3ConnectionFactory.java deleted file mode 100644 index 208cf2da3a..0000000000 --- a/s3/src/main/java/org/jclouds/aws/s3/S3ConnectionFactory.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3; - -import com.google.inject.AbstractModule; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.name.Names; -import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; - -import java.util.Properties; - -/** - * // TODO: Adrian: Document return getConnection! - * - * @author Adrian Cole - */ -public class S3ConnectionFactory { - public static final Properties DEFAULT_PROPERTIES; - - static { - DEFAULT_PROPERTIES = new Properties(); - DEFAULT_PROPERTIES.setProperty("jclouds.http.address", "s3.amazonaws.com"); - DEFAULT_PROPERTIES.setProperty("jclouds.http.port", "443"); - DEFAULT_PROPERTIES.setProperty("jclouds.http.secure", "true"); - DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.max_connection_reuse", "75"); - DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.max_session_failures", "2"); - DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.request_invoker_threads", "1"); - DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.io_worker_threads", "2"); - DEFAULT_PROPERTIES.setProperty("jclouds.pool.max_connections", "12"); - } - - public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey) { - Properties properties = new Properties(DEFAULT_PROPERTIES); - properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId); - properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey); - return getConnection(properties, new JavaUrlHttpFutureCommandClientModule()); - } - - public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure) { - Properties properties = new Properties(DEFAULT_PROPERTIES); - properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId); - properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey); - properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure)); - if (!isSecure) - properties.setProperty("jclouds.http.port", "80"); - - return getConnection(properties, new JavaUrlHttpFutureCommandClientModule()); - } - - public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, - String server) { - Properties properties = new Properties(DEFAULT_PROPERTIES); - properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId); - properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey); - properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure)); - properties.setProperty("jclouds.http.address", server); - if (!isSecure) - properties.setProperty("jclouds.http.port", "80"); - return getConnection(properties, new JavaUrlHttpFutureCommandClientModule()); - } - - public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, - String server, int port) { - Properties properties = new Properties(DEFAULT_PROPERTIES); - properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId); - properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey); - properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure)); - properties.setProperty("jclouds.http.address", server); - properties.setProperty("jclouds.http.port", port + ""); - return getConnection(properties, new JavaUrlHttpFutureCommandClientModule()); - } - - /** - * Create a new interface to interact with S3 with the given credential and connection - * parameters - */ - public static synchronized S3Connection getConnection(final Properties properties, Module httpModule) { - return getInjector(properties, httpModule).getInstance(S3Connection.class); - } - - /** - * Create a new interface to interact with S3 with the given credential and connection - * parameters - */ - public static synchronized Injector getInjector(final Properties properties, Module httpModule) { - return Guice.createInjector(new AbstractModule() { - @Override - protected void configure() { - Names.bindProperties(binder(), properties); - } - }, httpModule, new S3ConnectionModule()); - } -} diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3Constants.java b/s3/src/main/java/org/jclouds/aws/s3/S3Constants.java index 0c47eacc0a..e9cc371499 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3Constants.java +++ b/s3/src/main/java/org/jclouds/aws/s3/S3Constants.java @@ -23,13 +23,16 @@ */ package org.jclouds.aws.s3; +import org.jclouds.command.pool.PoolConstants; import org.jclouds.http.HttpConstants; /** * // TODO: Adrian: Document this! - * + * * @author Adrian Cole */ -public class S3Constants extends HttpConstants { +public interface S3Constants extends HttpConstants, PoolConstants { public static final String AUTH = "Authorization"; + public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey"; + public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid"; } diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3Context.java b/s3/src/main/java/org/jclouds/aws/s3/S3Context.java new file mode 100644 index 0000000000..5b3b338cc0 --- /dev/null +++ b/s3/src/main/java/org/jclouds/aws/s3/S3Context.java @@ -0,0 +1,49 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.aws.s3; + +import org.jclouds.aws.s3.domain.S3Bucket; + +public interface S3Context { + + /** + * @return a connection to S3 + */ + S3Connection getConnection(); + + /** + * Creates a Map view of the specified + * bucket. + * + * @param bucket + * @return + */ + S3ObjectMap createMapView(S3Bucket bucket); + + /** + * Closes all connections to S3. + */ + void close(); + +} \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java b/s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java new file mode 100644 index 0000000000..fea5cdf13c --- /dev/null +++ b/s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java @@ -0,0 +1,148 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.aws.s3; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import org.jclouds.aws.s3.config.S3ContextModule; +import org.jclouds.http.HttpFutureCommandClient; +import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.name.Names; +import static org.jclouds.aws.s3.S3Constants.*; + +/** + * // TODO: Adrian: Document return getConnection! + * + * @author Adrian Cole + */ +public class S3ContextFactory { + + public static final Properties DEFAULT_PROPERTIES; + + static { + DEFAULT_PROPERTIES = new Properties(); + DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_ADDRESS, + "s3.amazonaws.com"); + DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_PORT, "443"); + DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_SECURE, "true"); + DEFAULT_PROPERTIES + .setProperty(PROPERTY_POOL_MAX_CONNECTION_REUSE, "75"); + DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_MAX_SESSION_FAILURES, "2"); + DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_REQUEST_INVOKER_THREADS, + "1"); + DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_IO_WORKER_THREADS, "2"); + DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_MAX_CONNECTIONS, "12"); + } + + public static S3Context createS3Context(String awsAccessKeyId, + String awsSecretAccessKey, Module... modules) { + Properties properties = new Properties(DEFAULT_PROPERTIES); + properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId); + properties + .setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey); + return createS3Context(properties, modules); + } + + public static S3Context createS3Context(String awsAccessKeyId, + String awsSecretAccessKey, boolean isSecure, Module... modules) { + Properties properties = new Properties(DEFAULT_PROPERTIES); + properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId); + properties + .setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey); + properties + .setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure)); + if (!isSecure) + properties.setProperty(PROPERTY_HTTP_PORT, "80"); + return createS3Context(properties, modules); + } + + public static S3Context createS3Context(String awsAccessKeyId, + String awsSecretAccessKey, boolean isSecure, String server, + Module... modules) { + Properties properties = new Properties(DEFAULT_PROPERTIES); + properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId); + properties + .setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey); + properties + .setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure)); + properties.setProperty(PROPERTY_HTTP_ADDRESS, server); + if (!isSecure) + properties.setProperty(PROPERTY_HTTP_PORT, "80"); + return createS3Context(properties, modules); + } + + public static S3Context createS3Context(String awsAccessKeyId, + String awsSecretAccessKey, boolean isSecure, String server, + int port, Module... modules) { + Properties properties = new Properties(DEFAULT_PROPERTIES); + properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId); + properties + .setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey); + properties + .setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure)); + properties.setProperty(PROPERTY_HTTP_ADDRESS, server); + properties.setProperty(PROPERTY_HTTP_PORT, port + ""); + return createS3Context(properties, modules); + } + + public static S3Context createS3Context(Properties properties, + Module... modules) { + return getInjector(properties, modules).getInstance(S3Context.class); + } + + /** + * Bind the given properties and install the list of modules. If no modules + * are specified, install the default + * {@link JavaUrlHttpFutureCommandClientModule} + * + * @param properties + * - contains constants used by jclouds + * {@link #DEFAULT_PROPERTIES} + * @param httpModules + * - modules that must bind {@link HttpFutureCommandClient} if + * specified + * */ + public static Injector getInjector(final Properties properties, + Module... httpModules) { + final List modules = httpModules.length != 0 ? Arrays + .asList(httpModules) : Collections + .singletonList(new JavaUrlHttpFutureCommandClientModule()); + return Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + Names.bindProperties(binder(), properties); + for (Module module : modules) + install(module); + } + }, new S3ContextModule()); + } +} diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3ObjectMap.java b/s3/src/main/java/org/jclouds/aws/s3/S3ObjectMap.java index e6932a0036..529ba95e05 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3ObjectMap.java +++ b/s3/src/main/java/org/jclouds/aws/s3/S3ObjectMap.java @@ -23,138 +23,32 @@ */ package org.jclouds.aws.s3; -import com.google.inject.Inject; -import org.jclouds.Logger; -import org.jclouds.aws.s3.domain.S3Object; +import java.io.File; +import java.io.InputStream; +import java.util.Map; +import java.util.Set; + import org.jclouds.aws.s3.domain.S3Bucket; +import org.jclouds.aws.s3.domain.S3Object; -import java.util.*; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; +public interface S3ObjectMap extends Map { -/** - * // TODO: Adrian: Document this! - * - * @author Adrian Cole - */ -public class S3ObjectMap implements ConcurrentMap { - private Logger logger; - private S3Connection connection; - private S3Bucket bucket; - private S3Utils utils; + InputStream putString(String key, String value); - @Inject - public S3ObjectMap(java.util.logging.Logger logger, S3Connection connection, S3Bucket bucket, S3Utils utils) { - this.logger = new Logger(logger); - this.connection = connection; - this.bucket = bucket; - this.utils = utils; - } + InputStream putFile(String key, File value); + InputStream putBytes(String key, byte[] value); + + void putAllStrings(Map map); - public Object putIfAbsent(String s, Object o) { - return null; // TODO: Adrian: Customise this generated block - } + void putAllBytes(Map map); - public boolean remove(Object o, Object o1) { - return false; // TODO: Adrian: Customise this generated block - } + void putAllFiles(Map map); - public boolean replace(String s, Object o, Object o1) { - return false; // TODO: Adrian: Customise this generated block - } + InputStream put(S3Object object); + + void putAll(Set objects); - public Object replace(String s, Object o) { - return null; // TODO: Adrian: Customise this generated block - } + S3Bucket getBucket(); - public int size() { - try { - bucket = connection.getBucket(bucket).get(); - return bucket.getContents().size(); - } catch (Exception e) { - S3Utils.rethrowIfRuntimeOrSameType(e); - throw new S3RuntimeException("Error clearing bucket" + bucket, e); - } - } - - public boolean isEmpty() { - return false; // TODO: Adrian: Customise this generated block - } - - public boolean containsKey(Object o) { - return false; // TODO: Adrian: Customise this generated block - } - - public boolean containsValue(Object o) { - return false; // TODO: Adrian: Customise this generated block - } - - public Object get(Object o) { - try { - return connection.getObject(bucket, o.toString()).get(); - } catch (Exception e) { - S3Utils.rethrowIfRuntimeOrSameType(e); - throw new S3RuntimeException(String.format("Error geting object %1s:%2s", bucket, o), e); - } - } - - public Object put(String s, Object o) { - S3Object object = new S3Object(); - try { - object.setKey(s); - object.setContent(o); - return connection.addObject(bucket, object).get(); - } catch (Exception e) { - S3Utils.rethrowIfRuntimeOrSameType(e); - throw new S3RuntimeException(String.format("Error adding object %1s:%2s", bucket, object), e); - } - } - - public Object remove(Object o) { - return null; // TODO: Adrian: Customise this generated block - } - - public void putAll(Map map) { - // TODO: Adrian: Customise this generated block - } - - private class S3RuntimeException extends RuntimeException { - public S3RuntimeException(String s) { - super(s); // TODO: Adrian: Customise this generated block - } - - public S3RuntimeException(String s, Throwable throwable) { - super(s, throwable); // TODO: Adrian: Customise this generated block - } - } - - public void clear() { - try { - bucket = connection.getBucket(bucket).get(); - List> deletes = new ArrayList>(); - for (S3Object object : bucket.getContents()) { - deletes.add(connection.deleteObject(bucket, object.getKey())); - } - for (Future isdeleted : deletes) - if (!isdeleted.get()) { - throw new S3RuntimeException("failed to delete entry"); - } - } catch (Exception e) { - S3Utils.rethrowIfRuntimeOrSameType(e); - throw new S3RuntimeException("Error clearing bucket" + bucket, e); - } - } - - public Set keySet() { - return null; // TODO: Adrian: Customise this generated block - } - - public Collection values() { - return null; // TODO: Adrian: Customise this generated block - } - - public Set> entrySet() { - return null; // TODO: Adrian: Customise this generated block - } -} +} \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3Utils.java b/s3/src/main/java/org/jclouds/aws/s3/S3Utils.java index 7ffceb7c48..d7159ccc74 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3Utils.java +++ b/s3/src/main/java/org/jclouds/aws/s3/S3Utils.java @@ -23,6 +23,14 @@ */ package org.jclouds.aws.s3; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; + +import org.bouncycastle.crypto.digests.MD5Digest; import org.bouncycastle.crypto.digests.SHA1Digest; import org.bouncycastle.crypto.macs.HMac; import org.bouncycastle.crypto.params.KeyParameter; @@ -30,38 +38,89 @@ import org.bouncycastle.util.encoders.Base64; import org.jclouds.Utils; import org.jclouds.aws.s3.domain.S3Object; -import java.io.IOException; -import java.io.InputStream; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; - public class S3Utils extends Utils { + static final byte[] HEX_CHAR_TABLE = { (byte) '0', (byte) '1', (byte) '2', + (byte) '3', (byte) '4', (byte) '5', (byte) '6', (byte) '7', + (byte) '8', (byte) '9', (byte) 'a', (byte) 'b', (byte) 'c', + (byte) 'd', (byte) 'e', (byte) 'f' }; - public static String digest(String toEncode, byte[] key) throws NoSuchAlgorithmException, NoSuchProviderException, InvalidKeyException { - HMac hmac = new HMac(new SHA1Digest()); - byte[] resBuf = new byte[hmac.getMacSize()]; - byte[] plainBytes = toEncode.getBytes(); - byte[] keyBytes = key; - hmac.init(new KeyParameter(keyBytes)); - hmac.update(plainBytes, 0, plainBytes.length); - hmac.doFinal(resBuf, 0); - return new String(Base64.encode(resBuf)); + public static String getHexString(byte[] raw) + throws UnsupportedEncodingException { + byte[] hex = new byte[2 * raw.length]; + int index = 0; + + for (byte b : raw) { + int v = b & 0xFF; + hex[index++] = HEX_CHAR_TABLE[v >>> 4]; + hex[index++] = HEX_CHAR_TABLE[v & 0xF]; + } + return new String(hex, "ASCII"); } + public static String hmacSha1Base64(String toEncode, byte[] key) + throws NoSuchAlgorithmException, NoSuchProviderException, + InvalidKeyException { + HMac hmac = new HMac(new SHA1Digest()); + byte[] resBuf = new byte[hmac.getMacSize()]; + byte[] plainBytes = toEncode.getBytes(); + byte[] keyBytes = key; + hmac.init(new KeyParameter(keyBytes)); + hmac.update(plainBytes, 0, plainBytes.length); + hmac.doFinal(resBuf, 0); + return new String(Base64.encode(resBuf)); + } - public static String getContentAsStringAndClose(S3Object object) throws IOException { - Object o = object.getContent(); + public static String md5Hex(byte [] toEncode) + throws NoSuchAlgorithmException, NoSuchProviderException, + InvalidKeyException, UnsupportedEncodingException { + byte[] resBuf = md5(toEncode); + return getHexString(resBuf); + } - if (o instanceof InputStream) { - String returnVal = toStringAndClose((InputStream) o); - if (object.getContentType().indexOf("xml") >= 0) { + public static String md5Base64(byte [] toEncode) + throws NoSuchAlgorithmException, NoSuchProviderException, + InvalidKeyException { + byte[] resBuf = md5(toEncode); + return new String(Base64.encode(resBuf)); + } - } - return returnVal; - } else { - throw new IllegalArgumentException("Object type not supported: " + o.getClass().getName()); - } + public static byte[] md5(byte[] plainBytes) { + MD5Digest md5 = new MD5Digest(); + byte[] resBuf = new byte[md5.getDigestSize()]; + md5.update(plainBytes, 0, plainBytes.length); + md5.doFinal(resBuf, 0); + return resBuf; + } + + public static byte[] md5(InputStream toEncode) throws IOException { + MD5Digest md5 = new MD5Digest(); + byte[] resBuf = new byte[md5.getDigestSize()]; + byte[] buffer = new byte[1024]; + int numRead = -1; + do { + numRead = toEncode.read(buffer); + if (numRead > 0) { + md5.update(buffer, 0, numRead); + } + } while (numRead != -1); + md5.doFinal(resBuf, 0); + return resBuf; + } + + public static String getContentAsStringAndClose(S3Object object) + throws IOException { + Object o = object.getContent(); + + if (o instanceof InputStream) { + String returnVal = toStringAndClose((InputStream) o); + if (object.getContentType().indexOf("xml") >= 0) { + + } + return returnVal; + } else { + throw new IllegalArgumentException("Object type not supported: " + + o.getClass().getName()); + } } } \ No newline at end of file diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/RetrieveObjectCallable.java b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/RetrieveObjectCallable.java index 4021c5d95a..e0674da727 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/RetrieveObjectCallable.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/RetrieveObjectCallable.java @@ -67,7 +67,10 @@ public class RetrieveObjectCallable extends object.setLastModified(dateParser .dateTimeFromHeaderFormat(getResponse() .getFirstHeaderOrNull("Last-Modified"))); - object.setETag(getResponse().getFirstHeaderOrNull("ETag")); + String eTag = getResponse().getFirstHeaderOrNull("ETag"); + if (eTag != null) { + object.setETag(eTag.replaceAll("\"", "")); + } object.setContentType(getResponse().getFirstHeaderOrNull( "Content-Type")); object.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull( diff --git a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/xml/ListBucketHandler.java b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/xml/ListBucketHandler.java index ff064cfa78..4b8a676926 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/commands/callables/xml/ListBucketHandler.java +++ b/s3/src/main/java/org/jclouds/aws/s3/commands/callables/xml/ListBucketHandler.java @@ -68,7 +68,7 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult { } else if (qName.equals("LastModified")) { currentObject.setLastModified(dateParser.dateTimeFromXMLFormat(currentText.toString())); } else if (qName.equals("ETag")) { - currentObject.setETag(currentText.toString()); + currentObject.setETag(currentText.toString().replaceAll("\"", "")); } else if (qName.equals("Size")) { currentObject.setSize(Long.parseLong(currentText.toString())); } else if (qName.equals("Owner")) { diff --git a/s3/src/main/java/org/jclouds/aws/s3/S3ConnectionModule.java b/s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java similarity index 58% rename from s3/src/main/java/org/jclouds/aws/s3/S3ConnectionModule.java rename to s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java index 37b04bbc7c..135d69fccb 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/S3ConnectionModule.java +++ b/s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java @@ -21,39 +21,55 @@ * under the License. * ==================================================================== */ -package org.jclouds.aws.s3; - -import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import com.google.inject.Scopes; -import org.jclouds.aws.s3.commands.config.S3CommandsModule; -import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader; -import org.jclouds.aws.s3.filters.RequestAuthorizeSignature; -import org.jclouds.http.HttpRequestFilter; +package org.jclouds.aws.s3.config; import java.util.ArrayList; import java.util.List; +import org.jclouds.aws.s3.S3Connection; +import org.jclouds.aws.s3.S3Context; +import org.jclouds.aws.s3.commands.config.S3CommandsModule; +import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader; +import org.jclouds.aws.s3.filters.RequestAuthorizeSignature; +import org.jclouds.aws.s3.internal.GuiceS3Context; +import org.jclouds.aws.s3.internal.LiveS3Connection; +import org.jclouds.aws.s3.internal.LiveS3ObjectMap; +import org.jclouds.aws.s3.internal.GuiceS3Context.S3ObjectMapFactory; +import org.jclouds.http.HttpRequestFilter; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import com.google.inject.assistedinject.FactoryProvider; + /** * // TODO: Adrian: Document this! - * + * * @author Adrian Cole */ -public class S3ConnectionModule extends AbstractModule { +public class S3ContextModule extends AbstractModule { + @Override protected void configure() { - install(new S3CommandsModule()); - bind(S3Connection.class).to(LiveS3Connection.class).in(Scopes.SINGLETON); + install(new S3CommandsModule()); + bind(S3Connection.class).to(LiveS3Connection.class) + .in(Scopes.SINGLETON); + bind(GuiceS3Context.S3ObjectMapFactory.class).toProvider( + FactoryProvider.newFactory( + GuiceS3Context.S3ObjectMapFactory.class, + LiveS3ObjectMap.class)); + bind(S3Context.class).to(GuiceS3Context.class); } @Provides @Singleton - List provideRequestFilters(RemoveTransferEncodingHeader removTransferEncodingHeader, RequestAuthorizeSignature requestAuthorizeSignature) { - List filters = new ArrayList(); - filters.add(removTransferEncodingHeader); - filters.add(requestAuthorizeSignature); - return filters; + List provideRequestFilters( + RemoveTransferEncodingHeader removTransferEncodingHeader, + RequestAuthorizeSignature requestAuthorizeSignature) { + List filters = new ArrayList(); + filters.add(removTransferEncodingHeader); + filters.add(requestAuthorizeSignature); + return filters; } } - diff --git a/s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java b/s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java index 8fd87018c2..a291b17989 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java +++ b/s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java @@ -99,8 +99,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter { @Inject public RequestAuthorizeSignature( - @Named("jclouds.aws.accesskeyid") String accessKey, - @Named("jclouds.aws.secretaccesskey") String secretKey, + @Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey, + @Named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY) String secretKey, DateService dateService) { this.accessKey = accessKey; this.secretKey = secretKey; @@ -142,7 +142,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter { toSign.append(request.getUri()); String signature; try { - signature = S3Utils.digest(toSign.toString(), secretKey.getBytes()); + signature = S3Utils.hmacSha1Base64(toSign.toString(), secretKey.getBytes()); } catch (Exception e) { throw new HttpException("error signing request", e); } diff --git a/s3/src/main/java/org/jclouds/aws/s3/internal/GuiceS3Context.java b/s3/src/main/java/org/jclouds/aws/s3/internal/GuiceS3Context.java new file mode 100644 index 0000000000..3fef28739c --- /dev/null +++ b/s3/src/main/java/org/jclouds/aws/s3/internal/GuiceS3Context.java @@ -0,0 +1,88 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.aws.s3.internal; + +import java.io.IOException; + +import org.jclouds.Logger; +import org.jclouds.aws.s3.S3Connection; +import org.jclouds.aws.s3.S3Context; +import org.jclouds.aws.s3.S3ObjectMap; +import org.jclouds.aws.s3.domain.S3Bucket; +import org.jclouds.lifecycle.Closer; + +import com.google.inject.Inject; +import com.google.inject.Injector; + +/** + * // TODO: Adrian: Document return getConnection! + * + * @author Adrian Cole + */ +public class GuiceS3Context implements S3Context { + public interface S3ObjectMapFactory { + S3ObjectMap createMapView(S3Bucket bucket); + } + + private final Logger logger; + private final Injector injector; + private final S3ObjectMapFactory s3ObjectMapFactory; + private final Closer closer; + + @Inject + private GuiceS3Context(java.util.logging.Logger logger, Injector injector, + Closer closer, S3ObjectMapFactory s3ObjectMapFactory) { + this.logger = new Logger(logger); + this.injector = injector; + this.s3ObjectMapFactory = s3ObjectMapFactory; + this.closer = closer; + } + + /** + * {@inheritDoc} + */ + public S3Connection getConnection() { + return injector.getInstance(S3Connection.class); + } + + /** + * {@inheritDoc} + */ + public S3ObjectMap createMapView(S3Bucket bucket) { + getConnection().createBucketIfNotExists(bucket); + return s3ObjectMapFactory.createMapView(bucket); + } + + /** + * {@inheritDoc} + */ + public void close() { + try { + closer.close(); + } catch (IOException e) { + logger.error(e, "error closing content"); + } + } + +} diff --git a/s3/src/main/java/org/jclouds/aws/s3/LiveS3Connection.java b/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3Connection.java similarity index 97% rename from s3/src/main/java/org/jclouds/aws/s3/LiveS3Connection.java rename to s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3Connection.java index 4d3aad4125..a5bc160158 100644 --- a/s3/src/main/java/org/jclouds/aws/s3/LiveS3Connection.java +++ b/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3Connection.java @@ -21,12 +21,12 @@ * under the License. * ==================================================================== */ -package org.jclouds.aws.s3; +package org.jclouds.aws.s3.internal; -import java.io.IOException; import java.util.List; import java.util.concurrent.Future; +import org.jclouds.aws.s3.S3Connection; import org.jclouds.aws.s3.commands.CopyObject; import org.jclouds.aws.s3.commands.DeleteBucket; import org.jclouds.aws.s3.commands.DeleteObject; @@ -127,9 +127,4 @@ public class LiveS3Connection implements S3Connection { client.submit(listRequest); return listRequest; } - - public void close() throws IOException { - client.close(); - } - } diff --git a/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3ObjectMap.java b/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3ObjectMap.java new file mode 100644 index 0000000000..59cc90aaf1 --- /dev/null +++ b/s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3ObjectMap.java @@ -0,0 +1,433 @@ +/** + * + * Copyright (C) 2009 Adrian Cole + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ +package org.jclouds.aws.s3.internal; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import org.apache.commons.io.IOUtils; +import org.jclouds.Utils; +import org.jclouds.aws.s3.S3Connection; +import org.jclouds.aws.s3.S3ObjectMap; +import org.jclouds.aws.s3.S3Utils; +import org.jclouds.aws.s3.domain.S3Bucket; +import org.jclouds.aws.s3.domain.S3Object; + +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; + +/** + * Map representation of a live connection to S3. + * + * @author Adrian Cole + */ +public class LiveS3ObjectMap implements S3ObjectMap { + + private final S3Connection connection; + private final S3Bucket bucket; + + @Inject + public LiveS3ObjectMap(S3Connection connection, @Assisted S3Bucket bucket) { + this.connection = connection; + this.bucket = bucket; + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#size() + */ + public int size() { + try { + return refreshBucket().getContents().size(); + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException("Error clearing bucket" + bucket, e); + } + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#get(java.lang.Object) + */ + public InputStream get(Object o) { + try { + return (InputStream) (connection.getObject(bucket, o.toString()) + .get()).getContent(); + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException(String.format( + "Error geting object %1s:%2s", bucket, o), e); + } + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#remove(java.lang.Object) + */ + public InputStream remove(Object o) { + InputStream old = get(o); + try { + connection.deleteObject(bucket, o.toString()).get(); + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException(String.format( + "Error removing object %1s:%2s", bucket, o), e); + } + return old; + } + + public class S3RuntimeException extends RuntimeException { + S3RuntimeException(String s) { + super(s); + } + + public S3RuntimeException(String s, Throwable throwable) { + super(s, throwable); + } + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#clear() + */ + public void clear() { + try { + List> deletes = new ArrayList>(); + for (String key : keySet()) { + deletes.add(connection.deleteObject(bucket, key)); + } + for (Future isdeleted : deletes) + if (!isdeleted.get()) { + throw new S3RuntimeException("failed to delete entry"); + } + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException("Error clearing bucket" + bucket, e); + } + } + + private S3Bucket refreshBucket() throws InterruptedException, + ExecutionException { + return connection.getBucket(bucket).get(); + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#keySet() + */ + public Set keySet() { + try { + Set keys = new HashSet(); + for (S3Object object : refreshBucket().getContents()) + keys.add(object.getKey()); + return keys; + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException("Error getting keys in bucket: " + + bucket, e); + } + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#values() + */ + public Collection values() { + Collection values = new LinkedList(); + Set> futureObjects = new HashSet>(); + for (String key : keySet()) { + futureObjects.add(connection.getObject(bucket, key)); + } + for (Future futureObject : futureObjects) { + S3Object object = null; + try { + object = futureObject.get(); + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException(String.format( + "Error getting value from bucket %1s:%2s", bucket, + object != null ? object.getKey() : "unknown"), e); + } + System.err.printf("key: %1s, MD5: %2s", object.getKey(), object + .getContentMD5()); + values.add((InputStream) object.getContent()); + } + return values; + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#entrySet() + */ + public Set> entrySet() { + Set> entrySet = new HashSet>(); + for (String key : keySet()) { + Map.Entry entry = new Entry(key, get(key)); + entrySet.add(entry); + } + return entrySet; + } + + public class Entry implements java.util.Map.Entry { + + private InputStream value; + private String key; + + Entry(String key, InputStream value) { + this.key = key; + this.value = value; + } + + public String getKey() { + return key; + } + + public InputStream getValue() { + return value; + } + + public InputStream setValue(InputStream value) { + return put(key, value); + } + + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#containsKey(java.lang.Object) + */ + public boolean containsKey(Object key) { + try { + return connection.headObject(bucket, key.toString()).get() != S3Object.NOT_FOUND; + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException(String.format( + "Error searching for %1s:%2s", bucket, key), e); + } + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#containsValue(java.lang.Object) + */ + public boolean containsValue(Object value) { + + try { + byte[] md5; + + if (value instanceof InputStream) { + md5 = S3Utils.md5((InputStream) value); + } else if (value instanceof byte[]) { + md5 = S3Utils.md5((byte[]) value); + } else if (value instanceof String) { + md5 = S3Utils.md5(((String) value).getBytes()); + } else if (value instanceof File) { + md5 = S3Utils.md5(new FileInputStream((File) value)); + } else { + throw new IllegalArgumentException("unsupported value type: " + + value.getClass()); + } + String eTagOfValue = S3Utils.getHexString(md5); + + for (S3Object object : refreshBucket().getContents()) { + if (object.getETag().equals(eTagOfValue)) + return true; + } + return false; + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException(String.format( + "Error searching for ETAG of value: [%2s] in bucket:%1s", + bucket, value), e); + } + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMapi#isEmpty() + */ + public boolean isEmpty() { + return keySet().size() == 0; + } + + private InputStream putInternal(String s, Object o) { + S3Object object = new S3Object(); + try { + + InputStream returnVal = containsKey(s) ? get(s) : null; + object.setKey(s); + object.setContent(o); + setSizeIfContentIsInputStream(object); + connection.addObject(bucket, object).get(); + return returnVal; + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException(String.format( + "Error adding object %1s:%2s", bucket, object), e); + } + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMap#putAll(java.util.Map) + */ + public void putAll(Map map) { + putAllInternal(map); + } + + public void putAllBytes(Map map) { + putAllInternal(map); + } + + public void putAllFiles(Map map) { + putAllInternal(map); + } + + public void putAllStrings(Map map) { + putAllInternal(map); + } + + private void putAllInternal(Map map) { + try { + List> puts = new ArrayList>(); + for (String key : map.keySet()) { + S3Object object = new S3Object(); + object.setKey(key); + object.setContent(map.get(key)); + setSizeIfContentIsInputStream(object); + puts.add(connection.addObject(bucket, object)); + } + for (Future put : puts) + put.get();// this will throw an exception if there was a problem + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException("Error putting into bucket" + bucket, + e); + } + } + + private void setSizeIfContentIsInputStream(S3Object object) + throws IOException { + if (object.getContent() instanceof InputStream) { + byte[] buffer = IOUtils.toByteArray((InputStream) object + .getContent()); + object.setSize(buffer.length); + object.setContent(new ByteArrayInputStream(buffer)); + } + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMap#putString(java.lang.String, + * java.lang.String) + */ + public InputStream putString(String key, String value) { + return putInternal(key, value); + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMap#putFile(java.lang.String, + * java.io.File) + */ + public InputStream putFile(String key, File value) { + return putInternal(key, value); + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMap#putBytes(java.lang.String, byte[]) + */ + public InputStream putBytes(String key, byte[] value) { + return putInternal(key, value); + } + + /* + * (non-Javadoc) + * + * @see org.jclouds.aws.s3.S3ObjectMap#put(java.lang.String, + * java.io.InputStream) + */ + public InputStream put(String key, InputStream value) { + return putInternal(key, value); + } + + public S3Bucket getBucket() { + return bucket; + } + + public InputStream put(S3Object object) { + InputStream returnVal = get(object.getKey()); + try { + connection.addObject(bucket, object).get(); + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException(String.format( + "Error putting object %1s:%2s", bucket, object.getKey()), e); + } + return returnVal; + } + + public void putAll(Set objects) { + try { + List> puts = new ArrayList>(); + for (S3Object object : objects) { + puts.add(connection.addObject(bucket, object)); + } + for (Future put : puts) + put.get();// this will throw an exception if there was a problem + } catch (Exception e) { + Utils. rethrowIfRuntimeOrSameType(e); + throw new S3RuntimeException("Error putting into bucket" + bucket, + e); + } + } + +} diff --git a/s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java b/s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java index efcb3cf872..154fc10781 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java @@ -41,6 +41,7 @@ import java.util.logging.Logger; import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Object; +import org.jclouds.http.HttpConstants; import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeTest; @@ -48,7 +49,6 @@ import org.testng.annotations.Optional; import org.testng.annotations.Parameters; import org.testng.annotations.Test; -import com.google.inject.Injector; import com.google.inject.Module; @Test(sequential = true) @@ -90,44 +90,45 @@ public class S3IntegrationTest { + "/adriancole.s3.amazons3test.filetestsforadrian/file0101100101001001"; String amazonHadAnError = "InternalErrorWe encountered an internal error. Please try again.EF6FA7A639CAFF15tBkX23mIeq2riHsNw2YShupMlZ9+iy3V/uN+lRhqCR4qHTE07ujFeyAUPTowvuH/"; protected S3Connection client; - Injector i = null; + protected S3Context context = null; protected String bucketPrefix = System.getProperty("user.name") + "." + this.getClass().getName(); private static final String sysAWSAccessKeyId = System - .getProperty("jclouds.aws.accesskeyid"); + .getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID); private static final String sysAWSSecretAccessKey = System - .getProperty("jclouds.aws.secretaccesskey"); + .getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY); @BeforeTest - @Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" }) + @Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, + S3Constants.PROPERTY_AWS_SECRETACCESSKEY }) protected void setUpClient(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception { - i = createInject(AWSAccessKeyId != null ? AWSAccessKeyId + context = createS3Context(AWSAccessKeyId != null ? AWSAccessKeyId : sysAWSAccessKeyId, AWSSecretAccessKey != null ? AWSSecretAccessKey : sysAWSSecretAccessKey); - client = i.getInstance(LiveS3Connection.class); + client = context.getConnection(); deleteEverything(); } - protected Injector createInject(String AWSAccessKeyId, + protected S3Context createS3Context(String AWSAccessKeyId, String AWSSecretAccessKey) { - return S3ConnectionFactory.getInjector(buildS3Properties( + return S3ContextFactory.createS3Context(buildS3Properties( AWSAccessKeyId, AWSSecretAccessKey), createHttpModule()); } protected Properties buildS3Properties(String AWSAccessKeyId, String AWSSecretAccessKey) { Properties properties = new Properties( - S3ConnectionFactory.DEFAULT_PROPERTIES); - properties.setProperty("jclouds.aws.accesskeyid", checkNotNull( - AWSAccessKeyId, "AWSAccessKeyId")); - properties.setProperty("jclouds.aws.secretaccesskey", checkNotNull( - AWSSecretAccessKey, "AWSSecretAccessKey")); - properties.setProperty("jclouds.http.secure", "false"); - properties.setProperty("jclouds.http.port", "80"); + S3ContextFactory.DEFAULT_PROPERTIES); + properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID, + checkNotNull(AWSAccessKeyId, "AWSAccessKeyId")); + properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, + checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey")); + properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false"); + properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80"); return properties; } @@ -163,8 +164,8 @@ public class S3IntegrationTest { @AfterTest protected void tearDownClient() throws Exception { deleteEverything(); - client.close(); - i = null; + context.close(); + context = null; } } \ No newline at end of file diff --git a/s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java b/s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java index 256acdf162..259c38d6e6 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java @@ -23,59 +23,307 @@ */ package org.jclouds.aws.s3; +import static org.testng.Assert.assertEquals; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; + +import org.apache.commons.io.IOUtils; +import org.jclouds.Utils; +import org.jclouds.aws.s3.domain.S3Bucket; +import org.jclouds.aws.s3.domain.S3Object; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; -import org.jclouds.aws.s3.domain.S3Bucket; -import java.util.Map; -import java.util.logging.Logger; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; /** - * // TODO: Adrian: Document this! - * + * Tests to cover @{link LiveS3ObjectMap} + * * @author Adrian Cole */ @Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest") -public class S3ObjectMapTest { - private S3Connection connection; +public class S3ObjectMapTest extends S3IntegrationTest { + private S3Bucket bucket; - private Map map; + private S3ObjectMap map; + + private Map fiveStrings = ImmutableMap.of("one", "apple", + "two", "bear", "three", "candy", "four", "dogma", "five", "emma"); + private Map fiveBytes = ImmutableMap.of("one", "apple" + .getBytes(), "two", "bear".getBytes(), "three", "candy".getBytes(), + "four", "dogma".getBytes(), "five", "emma".getBytes()); + private Map fiveInputs; + private Map fiveFiles; + + String tmpDirectory; @BeforeMethod - public void setupExecutorService() throws Exception { - connection = new StubS3Connection(); - bucket = new S3Bucket(); - bucket.setName("mimi"); - connection.createBucketIfNotExists(bucket).get(); - map = new S3ObjectMap(Logger.getLogger("test"), connection, bucket, new S3Utils()); + @Parameters( { "basedir" }) + protected void setUpTempDir(String basedir) throws InterruptedException, + ExecutionException, FileNotFoundException, IOException { + tmpDirectory = basedir + File.separator + "target" + File.separator + + "testFiles" + File.separator + getClass().getSimpleName(); + new File(tmpDirectory).mkdirs(); + + fiveFiles = ImmutableMap.of("one", new File(tmpDirectory, "apple"), + "two", new File(tmpDirectory, "bear"), "three", new File( + tmpDirectory, "candy"), "four", new File(tmpDirectory, + "dogma"), "five", new File(tmpDirectory, "emma")); + + for (File file : fiveFiles.values()) { + IOUtils.write(file.getName(), new FileOutputStream(file)); + } + + fiveInputs = ImmutableMap.of("one", IOUtils.toInputStream("apple"), + "two", IOUtils.toInputStream("bear"), "three", IOUtils + .toInputStream("candy"), "four", IOUtils + .toInputStream("dogma"), "five", IOUtils + .toInputStream("emma")); + bucket = new S3Bucket(); + bucket.setName(bucketPrefix + ".mimi"); + client.createBucketIfNotExists(bucket).get(); + map = context.createMapView(bucket); + map.clear(); } @AfterMethod - public void teardownExecutorService() { - map = null; - bucket = null; - connection = null; + public void tearDown() { + map.clear(); + map = null; + bucket = null; } @Test - public void testClearWhenNothingInMap(){ - map.clear(); - assert map.size() == 0; + public void testClear() { + map.clear(); + assert map.size() == 0; + map.putString("one", "apple"); + map.clear(); + assert map.size() == 0; } -// @Test -// public void testGetReturnsWhatWasPut(){ -// map.put("hello","goodbye"); -// assert "goodbye".equals(map.get("hello")); -// assert map.size() == 0; -// } + @Test() + public void testRemove() throws IOException { + map.putString("one", "two"); + InputStream old = map.remove("one"); + assertEquals(Utils.toStringAndClose(old), "two"); + old = map.remove("one"); + assert old == null; + old = map.get("one"); + assert old == null; + assertEquals(map.keySet().size(), 0); + } + @Test() + public void testKeySet() { + assertEquals(map.keySet().size(), 0); + map.putString("one", "two"); + assertEquals(map.keySet(), ImmutableSet.of("one")); + } + + @Test() + public void testValues() throws IOException { + map.putAll(this.fiveInputs); + Collection values = map.values(); + assertEquals(values.size(), 5); + Set valuesAsString = new HashSet(); + for (InputStream stream : values) { + valuesAsString.add(Utils.toStringAndClose(stream)); + } + valuesAsString.removeAll(fiveStrings.values()); + assert valuesAsString.size() == 0; + } + + @Test() + public void testEntrySet() throws IOException { + map.putAllStrings(this.fiveStrings); + Set> entries = map.entrySet(); + assertEquals(entries.size(), 5); + for (Entry entry : entries) { + assertEquals(IOUtils.toString(entry.getValue()), fiveStrings + .get(entry.getKey())); + entry.setValue(IOUtils.toInputStream("")); + } + assertEquals(map.size(), 5); + for (InputStream value : map.values()) { + assertEquals(IOUtils.toString(value), ""); + } + } + + @Test() + public void testContainsKey() { + assert !map.containsKey("one"); + map.putString("one", "apple"); + assert map.containsKey("one"); + } + + @Test() + public void testContainsStringValue() { + map.putString("one", "apple"); + assert map.containsValue(fiveStrings.get("one")); + } + + @Test() + public void testContainsFileValue() { + map.putString("one", "apple"); + assert map.containsValue(fiveFiles.get("one")); + } + + @Test() + public void testContainsInputStreamValue() { + map.putString("one", "apple"); + assert map.containsValue(this.fiveInputs.get("one")); + } + + @Test() + public void testContainsBytesValue() { + map.putString("one", "apple"); + assert map.containsValue(this.fiveBytes.get("one")); + } + + @Test() + public void testIsEmpty() { + assert map.isEmpty(); + map.putString("one", "apple"); + assert !map.isEmpty(); + } + + @Test() + public void testPutAll() { + map.putAll(this.fiveInputs); + assertEquals(map.size(), 5); + assertEquals(new TreeSet(map.keySet()), new TreeSet( + fiveInputs.keySet())); + fourLeftRemovingOne(); + } + + @Test() + public void testPutAllBytes() { + map.putAllBytes(this.fiveBytes); + assertEquals(map.size(), 5); + assertEquals(new TreeSet(map.keySet()), new TreeSet( + fiveBytes.keySet())); + fourLeftRemovingOne(); + } @Test - public void testClearWhenSomethingInMap(){ - map.put("hello","goodbye"); - map.clear(); - assert map.size() == 0; + public void testPutAllFiles() { + map.putAllFiles(this.fiveFiles); + assertEquals(map.size(), 5); + assertEquals(new TreeSet(map.keySet()), new TreeSet( + fiveFiles.keySet())); + fourLeftRemovingOne(); + } + + void fourLeftRemovingOne() { + map.remove("one"); + assertEquals(map.size(), 4); + assertEquals(new TreeSet(map.keySet()), new TreeSet( + ImmutableSet.of("two", "three", "four", "five"))); + } + + @Test() + public void testPutAllStrings() { + map.putAllStrings(this.fiveStrings); + assertEquals(map.size(), 5); + assertEquals(new TreeSet(map.keySet()), new TreeSet( + fiveStrings.keySet())); + fourLeftRemovingOne(); + } + + @Test() + public void testPutString() throws IOException { + InputStream old = map.putString("one", "apple"); + getOneReturnsAppleAndOldValueIsNull(old); + InputStream apple = map.putString("one", "bear"); + getOneReturnsBearAndOldValueIsApple(apple); + } + + private void getOneReturnsAppleAndOldValueIsNull(InputStream old) + throws IOException { + assert old == null; + assertEquals(Utils.toStringAndClose(map.get("one")), "apple"); + assert map.size() == 1; + } + + private void getOneReturnsBearAndOldValueIsApple(InputStream oldValue) + throws IOException { + assertEquals(Utils.toStringAndClose(map.get("one")), "bear"); + assertEquals(Utils.toStringAndClose(oldValue), "apple"); + assert map.size() == 1; + } + + @Test() + public void testPutFile() throws IOException { + InputStream old = map.putFile("one", fiveFiles.get("one")); + getOneReturnsAppleAndOldValueIsNull(old); + InputStream apple = map.putFile("one", fiveFiles.get("two")); + getOneReturnsBearAndOldValueIsApple(apple); + } + + @Test() + public void testPutBytes() throws IOException { + InputStream old = map.putBytes("one", "apple".getBytes()); + getOneReturnsAppleAndOldValueIsNull(old); + InputStream apple = map.putBytes("one", "bear".getBytes()); + getOneReturnsBearAndOldValueIsApple(apple); + } + + @Test() + public void testPut() throws IOException { + InputStream old = map.put("one", IOUtils.toInputStream("apple")); + getOneReturnsAppleAndOldValueIsNull(old); + InputStream apple = map.put("one", IOUtils.toInputStream("bear")); + getOneReturnsBearAndOldValueIsApple(apple); + } + + @Test() + public void testGetBucket() { + assertEquals(map.getBucket().getName(), bucket.getName()); + } + + @Test + void testPutS3Object() throws IOException { + S3Object object = new S3Object(); + object.setKey("one"); + object.setContent(IOUtils.toInputStream("apple")); + object.setSize("apple".getBytes().length); + InputStream old = map.put(object); + getOneReturnsAppleAndOldValueIsNull(old); + object.setContent(IOUtils.toInputStream("bear")); + object.setSize("bear".getBytes().length); + InputStream apple = map.put(object); + getOneReturnsBearAndOldValueIsApple(apple); + } + + @Test + void testPutAllS3Objects() { + Set set = new HashSet(); + for (String key : fiveInputs.keySet()) { + S3Object object = new S3Object(); + object.setKey(key); + object.setContent(fiveInputs.get(key)); + object.setSize(fiveBytes.get(key).length); + set.add(object); + } + map.putAll(set); + assertEquals(map.size(), 5); + assertEquals(new TreeSet(map.keySet()), new TreeSet( + fiveInputs.keySet())); + fourLeftRemovingOne(); } } diff --git a/s3/src/test/java/org/jclouds/aws/s3/S3UtilsTest.java b/s3/src/test/java/org/jclouds/aws/s3/S3UtilsTest.java index fee3b36a93..d5aac89da6 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/S3UtilsTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/S3UtilsTest.java @@ -28,6 +28,7 @@ import org.jclouds.aws.PerformanceTest; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.io.UnsupportedEncodingException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; @@ -51,7 +52,7 @@ public class S3UtilsTest extends PerformanceTest { String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException { for (int i = 0; i < 10000; i++) - testBouncyCastleDigest(key, message, base64Digest); + testBouncyCastleHmacSha1Base64(key, message, base64Digest); } @Test(dataProvider = "hmacsha1") @@ -64,13 +65,24 @@ public class S3UtilsTest extends PerformanceTest { for (int i = 0; i < 10000; i++) completer.submit(new Callable() { public Boolean call() throws Exception { - testBouncyCastleDigest(key, message, base64Digest); + testBouncyCastleHmacSha1Base64(key, message, base64Digest); return true; } }); for (int i = 0; i < 10000; i++) assert completer.take().get(); } + + @DataProvider(name = "md5") + public Object[][] createMD5Data() { + return base64MD5MessageDigest; + } + public final static Object[][] base64MD5MessageDigest = { + { "apple", "1f3870be274f6c49b3e31a0c6728957f" }, + { "bear", "893b56e3cfe153fb770a120b83bac20c" }, + { "candy", "c48ba993d35c3abe0380f91738fe2a34" }, + { "dogma", "95eb470e4faee302e9cd3063b1923dab" }, + { "emma", "00a809937eddc44521da9521269e75c6" } }; public final static Object[][] base64KeyMessageDigest = { { Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There", @@ -96,11 +108,19 @@ public class S3UtilsTest extends PerformanceTest { } @Test(dataProvider = "hmacsha1") - public void testBouncyCastleDigest(byte[] key, String message, + public void testBouncyCastleHmacSha1Base64(byte[] key, String message, String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException { - String b64 = S3Utils.digest(message, key); + String b64 = S3Utils.hmacSha1Base64(message, key); assertEquals(b64, base64Digest); } + + @Test(dataProvider = "md5") + public void testBouncyCastleMD5Digest(String message, + String base64Digest) throws NoSuchProviderException, + NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException { + String b64 = S3Utils.md5Hex(message.getBytes()); + assertEquals(base64Digest,b64); + } } diff --git a/s3/src/test/java/org/jclouds/aws/s3/StubS3Connection.java b/s3/src/test/java/org/jclouds/aws/s3/StubS3Connection.java index 9eae5e9d26..a55483a8e9 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/StubS3Connection.java +++ b/s3/src/test/java/org/jclouds/aws/s3/StubS3Connection.java @@ -23,159 +23,181 @@ */ package org.jclouds.aws.s3; -import org.jclouds.aws.s3.domain.S3Object; -import org.jclouds.aws.s3.domain.S3Bucket; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; -import java.util.*; -import java.util.concurrent.*; -import java.io.IOException; +import org.jclouds.aws.s3.domain.S3Bucket; +import org.jclouds.aws.s3.domain.S3Object; /** * // TODO: Adrian: Document this! - * + * * @author Adrian Cole */ public class StubS3Connection implements S3Connection { private static Map> bucketToContents = new ConcurrentHashMap>(); - public Future getObject(final S3Bucket s3Bucket, final String key) { - return new FutureBase() { - public S3Object get() throws InterruptedException, ExecutionException { - if (!bucketToContents.containsKey(s3Bucket)) - return S3Object.NOT_FOUND; - Map realContents = bucketToContents.get(s3Bucket); - if (!realContents.containsKey(key)) - return S3Object.NOT_FOUND; - S3Object object = new S3Object(); - object.setKey(key); - object.setContent(realContents.get(key)); - return object; - } - }; + return new FutureBase() { + public S3Object get() throws InterruptedException, + ExecutionException { + if (!bucketToContents.containsKey(s3Bucket)) + return S3Object.NOT_FOUND; + Map realContents = bucketToContents + .get(s3Bucket); + if (!realContents.containsKey(key)) + return S3Object.NOT_FOUND; + S3Object object = new S3Object(); + object.setKey(key); + object.setContent(realContents.get(key)); + return object; + } + }; } public Future headObject(S3Bucket s3Bucket, String key) { - return getObject(s3Bucket, key); + return getObject(s3Bucket, key); } - public Future deleteObject(final S3Bucket s3Bucket, final String key) { - return new FutureBase() { - public Boolean get() throws InterruptedException, ExecutionException { - if (bucketToContents.containsKey(s3Bucket)) { - bucketToContents.get(s3Bucket).remove(key); - } - return true; - } - }; + public Future deleteObject(final S3Bucket s3Bucket, + final String key) { + return new FutureBase() { + public Boolean get() throws InterruptedException, + ExecutionException { + if (bucketToContents.containsKey(s3Bucket)) { + bucketToContents.get(s3Bucket).remove(key); + } + return true; + } + }; } - public Future addObject(final S3Bucket s3Bucket, final S3Object object) { - return new FutureBase() { - public String get() throws InterruptedException, ExecutionException { - if (!bucketToContents.containsKey(s3Bucket)) { - throw new ExecutionException(new RuntimeException("bucket not found: " + s3Bucket.getName())); - } - bucketToContents.get(s3Bucket).put(object.getKey(), object.getClass()); - return object.getKey(); - } - }; + public Future addObject(final S3Bucket s3Bucket, + final S3Object object) { + return new FutureBase() { + public String get() throws InterruptedException, ExecutionException { + if (!bucketToContents.containsKey(s3Bucket)) { + throw new ExecutionException(new RuntimeException( + "bucket not found: " + s3Bucket.getName())); + } + bucketToContents.get(s3Bucket).put(object.getKey(), + object.getClass()); + return object.getKey(); + } + }; } public Future createBucketIfNotExists(final S3Bucket s3Bucket) { - return new FutureBase() { - public Boolean get() throws InterruptedException, ExecutionException { - if (!bucketToContents.containsKey(s3Bucket)) { - bucketToContents.put(s3Bucket, new ConcurrentHashMap()); - } - return bucketToContents.containsKey(s3Bucket); - } - }; + return new FutureBase() { + public Boolean get() throws InterruptedException, + ExecutionException { + if (!bucketToContents.containsKey(s3Bucket)) { + bucketToContents.put(s3Bucket, + new ConcurrentHashMap()); + } + return bucketToContents.containsKey(s3Bucket); + } + }; } public Future deleteBucket(final S3Bucket s3Bucket) { - return new FutureBase() { - public Boolean get() throws InterruptedException, ExecutionException { - if (bucketToContents.containsKey(s3Bucket)) { - if (bucketToContents.get(s3Bucket).size() == 0) - return true; - } - return false; - } - }; + return new FutureBase() { + public Boolean get() throws InterruptedException, + ExecutionException { + if (bucketToContents.containsKey(s3Bucket)) { + if (bucketToContents.get(s3Bucket).size() == 0) + return true; + } + return false; + } + }; } - public Future copyObject(final S3Bucket sourceBucket, final S3Object sourceObject, final S3Bucket destinationBucket, final S3Object destinationObject) { - return new FutureBase() { - public Boolean get() throws InterruptedException, ExecutionException { - Map source = bucketToContents.get(sourceBucket); - Map dest = bucketToContents.get(destinationBucket); - if (source.containsKey(sourceObject.getKey())) { - dest.put(destinationObject.getKey(), source.get(sourceObject.getKey())); - return true; - } - return false; - } - }; + public Future copyObject(final S3Bucket sourceBucket, + final S3Object sourceObject, final S3Bucket destinationBucket, + final S3Object destinationObject) { + return new FutureBase() { + public Boolean get() throws InterruptedException, + ExecutionException { + Map source = bucketToContents.get(sourceBucket); + Map dest = bucketToContents + .get(destinationBucket); + if (source.containsKey(sourceObject.getKey())) { + dest.put(destinationObject.getKey(), source + .get(sourceObject.getKey())); + return true; + } + return false; + } + }; } public Future bucketExists(final S3Bucket s3Bucket) { - return new FutureBase() { - public Boolean get() throws InterruptedException, ExecutionException { - return bucketToContents.containsKey(s3Bucket); - } - }; + return new FutureBase() { + public Boolean get() throws InterruptedException, + ExecutionException { + return bucketToContents.containsKey(s3Bucket); + } + }; } public Future getBucket(final S3Bucket s3Bucket) { - return new FutureBase() { - public S3Bucket get() throws InterruptedException, ExecutionException { - Set contents = new HashSet(); - Map realContents = bucketToContents.get(s3Bucket); - if (realContents != null) { - for (String key : realContents.keySet()) { - S3Object object = new S3Object(); - object.setKey(key); - object.setContent(realContents.get(key)); - contents.add(object); - } - } - s3Bucket.setContents(contents); - return s3Bucket; - } - } + return new FutureBase() { + public S3Bucket get() throws InterruptedException, + ExecutionException { + Set contents = new HashSet(); + Map realContents = bucketToContents + .get(s3Bucket); + if (realContents != null) { + for (String key : realContents.keySet()) { + S3Object object = new S3Object(); + object.setKey(key); + object.setContent(realContents.get(key)); + contents.add(object); + } + } + s3Bucket.setContents(contents); + return s3Bucket; + } + } - ; + ; } public Future> getBuckets() { - return new FutureBase>() { - public List get() throws InterruptedException, ExecutionException { - return new ArrayList(bucketToContents.keySet()); - } - }; - } - - public void close() throws IOException { - // nothing to close + return new FutureBase>() { + public List get() throws InterruptedException, + ExecutionException { + return new ArrayList(bucketToContents.keySet()); + } + }; } private abstract class FutureBase implements Future { - public boolean cancel(boolean b) { - return false; - } + public boolean cancel(boolean b) { + return false; + } - public boolean isCancelled() { - return false; - } + public boolean isCancelled() { + return false; + } - public boolean isDone() { - return true; - } + public boolean isDone() { + return true; + } - public V get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { - return get(); - } + public V get(long l, TimeUnit timeUnit) throws InterruptedException, + ExecutionException, TimeoutException { + return get(); + } } } diff --git a/s3/src/test/java/org/jclouds/aws/s3/commands/S3ParserTest.java b/s3/src/test/java/org/jclouds/aws/s3/commands/S3ParserTest.java index 820086f3d9..1d2ec8f0dc 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/commands/S3ParserTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/commands/S3ParserTest.java @@ -23,6 +23,8 @@ */ package org.jclouds.aws.s3.commands; +import static org.testng.Assert.assertEquals; + import java.io.IOException; import java.util.List; import java.util.concurrent.Callable; @@ -52,7 +54,7 @@ import com.google.inject.name.Names; * * @author Adrian Cole */ -@Test(groups = "unit", sequential= true, testName = "s3.S3ParserTest") +@Test(groups = "unit", sequential = true, testName = "s3.S3ParserTest") public class S3ParserTest extends PerformanceTest { Injector injector = null; S3CommandFactory commandFactory = null; @@ -70,7 +72,7 @@ public class S3ParserTest extends PerformanceTest { } }); commandFactory = injector.getInstance(S3CommandFactory.class); - assert commandFactory != null; + assert commandFactory != null; } @AfterMethod @@ -148,7 +150,7 @@ public class S3ParserTest extends PerformanceTest { assert object.getLastModified().equals(expected) : String .format("expected %1s, but got %1s", expected, object .getLastModified()); - assert object.getETag().equals("\"9d7bb64e8e18ee34eec06dd2cf37b766\""); + assertEquals(object.getETag(), "9d7bb64e8e18ee34eec06dd2cf37b766"); assert object.getSize() == 136; S3Owner owner = new S3Owner(); owner diff --git a/s3/src/test/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignatureTest.java b/s3/src/test/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignatureTest.java index 7ab2399c1c..8f2a46baa3 100644 --- a/s3/src/test/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignatureTest.java +++ b/s3/src/test/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignatureTest.java @@ -27,6 +27,7 @@ import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.name.Names; import org.jclouds.aws.s3.DateService; +import org.jclouds.aws.s3.S3Constants; import org.testng.annotations.Test; @@ -40,8 +41,8 @@ public class RequestAuthorizeSignatureTest { filter = Guice.createInjector(new AbstractModule() { protected void configure() { - bindConstant().annotatedWith(Names.named("jclouds.aws.accesskeyid")).to("foo"); - bindConstant().annotatedWith(Names.named("jclouds.aws.secretaccesskey")).to("bar"); + bindConstant().annotatedWith(Names.named(S3Constants.PROPERTY_AWS_ACCESSKEYID)).to("foo"); + bindConstant().annotatedWith(Names.named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY)).to("bar"); bind(DateService.class); } diff --git a/samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java b/samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java index 80e8f98129..2f1c172e8f 100644 --- a/samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java +++ b/samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java @@ -31,6 +31,7 @@ import java.net.URL; import java.util.Properties; import org.apache.commons.io.IOUtils; +import org.jclouds.aws.s3.S3Constants; import org.testng.annotations.BeforeTest; import org.testng.annotations.Optional; import org.testng.annotations.Parameters; @@ -47,13 +48,14 @@ import org.testng.annotations.Test; public class GoogleAppEngineTest extends BaseGoogleAppEngineTest { private static final String sysAWSAccessKeyId = System - .getProperty("jclouds.aws.accesskeyid"); + .getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID); private static final String sysAWSSecretAccessKey = System - .getProperty("jclouds.aws.secretaccesskey"); + .getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY); @BeforeTest @Parameters( { "warfile", "devappserver.address", "devappserver.port", - "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" }) + S3Constants.PROPERTY_AWS_ACCESSKEYID, + S3Constants.PROPERTY_AWS_SECRETACCESSKEY }) public void startDevAppServer(final String warfile, final String address, final String port, @Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception { @@ -66,8 +68,8 @@ public class GoogleAppEngineTest extends BaseGoogleAppEngineTest { checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey"); Properties props = new Properties(); - props.put("jclouds.aws.accesskeyid", AWSAccessKeyId); - props.put("jclouds.aws.secretaccesskey", AWSSecretAccessKey); + props.put(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId); + props.put(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey); writePropertiesAndStartServer(address, port, warfile, props); } diff --git a/samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java b/samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java index c89779fec6..57bf8cffb8 100644 --- a/samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java +++ b/samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java @@ -31,8 +31,8 @@ import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; import org.apache.commons.io.IOUtils; -import org.jclouds.aws.s3.S3ConnectionFactory; -import org.jclouds.aws.s3.S3ConnectionModule; +import org.jclouds.aws.s3.S3ContextFactory; +import org.jclouds.aws.s3.config.S3ContextModule; import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; import org.jclouds.lifecycle.Closer; import org.jclouds.samples.googleappengine.JCloudsServlet; @@ -84,11 +84,11 @@ public class GuiceServletConfig extends GuiceServletContextListener { } finally { IOUtils.closeQuietly(input); } - props.putAll(S3ConnectionFactory.DEFAULT_PROPERTIES); + props.putAll(S3ContextFactory.DEFAULT_PROPERTIES); Names.bindProperties(binder(), props); } }, new JavaUrlHttpFutureCommandClientModule(), - new S3ConnectionModule(), new ServletModule() { + new S3ContextModule(), new ServletModule() { @Override protected void configureServlets() { serve("*.s3").with(JCloudsServlet.class);