removed google from default module list

git-svn-id: http://jclouds.googlecode.com/svn/trunk@72 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-04-30 13:28:24 +00:00
parent 9dfac47f05
commit ca8f278580
46 changed files with 1924 additions and 753 deletions

View File

@ -31,5 +31,4 @@ package org.jclouds.command;
public interface FutureCommandClient { public interface FutureCommandClient {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
<O extends FutureCommand> void submit(O operation); <O extends FutureCommand> void submit(O operation);
void close();
} }

View File

@ -23,46 +23,50 @@
*/ */
package org.jclouds.command.pool; package org.jclouds.command.pool;
import com.google.inject.Inject; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.Logger; import org.jclouds.Logger;
import org.jclouds.Utils; import org.jclouds.Utils;
import org.jclouds.command.FutureCommand; import org.jclouds.command.FutureCommand;
import org.jclouds.command.FutureCommandClient; import org.jclouds.command.FutureCommandClient;
import org.jclouds.lifecycle.BaseLifeCycle; import org.jclouds.lifecycle.BaseLifeCycle;
import org.jclouds.lifecycle.Closer;
import java.io.IOException; import com.google.inject.Inject;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/** /**
* // TODO: Adrian: Document this! * // TODO: Adrian: Document this!
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implements FutureCommandClient { public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle
@Inject private Closer closer; implements FutureCommandClient {
private final FutureCommandConnectionPool<C> futureCommandConnectionPool; private final FutureCommandConnectionPool<C> futureCommandConnectionPool;
private final BlockingQueue<FutureCommand> commandQueue; private final BlockingQueue<FutureCommand> commandQueue;
@Inject @Inject
public FutureCommandConnectionPoolClient(java.util.logging.Logger logger, ExecutorService executor, FutureCommandConnectionPool<C> futureCommandConnectionPool, BlockingQueue<FutureCommand> commandQueue) { public FutureCommandConnectionPoolClient(java.util.logging.Logger logger,
ExecutorService executor,
FutureCommandConnectionPool<C> futureCommandConnectionPool,
BlockingQueue<FutureCommand> commandQueue) {
super(new Logger(logger), executor, futureCommandConnectionPool); super(new Logger(logger), executor, futureCommandConnectionPool);
this.futureCommandConnectionPool = futureCommandConnectionPool; this.futureCommandConnectionPool = futureCommandConnectionPool;
this.commandQueue = commandQueue; this.commandQueue = commandQueue;
} }
@Override @Override
protected boolean shouldDoWork() { protected boolean shouldDoWork() {
return super.shouldDoWork() && futureCommandConnectionPool.getStatus().equals(Status.ACTIVE); return super.shouldDoWork()
&& futureCommandConnectionPool.getStatus()
.equals(Status.ACTIVE);
} }
@Override @Override
protected void doShutdown() { protected void doShutdown() {
if (exception == null && futureCommandConnectionPool.getException() != null) if (exception == null
&& futureCommandConnectionPool.getException() != null)
exception = futureCommandConnectionPool.getException(); exception = futureCommandConnectionPool.getException();
while (!commandQueue.isEmpty()) { while (!commandQueue.isEmpty()) {
FutureCommand command = commandQueue.remove(); FutureCommand command = commandQueue.remove();
@ -75,6 +79,7 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
} }
} }
@Override
protected void doWork() throws InterruptedException { protected void doWork() throws InterruptedException {
FutureCommand command = commandQueue.poll(1, TimeUnit.SECONDS); FutureCommand command = commandQueue.poll(1, TimeUnit.SECONDS);
if (command != null) { if (command != null) {
@ -87,7 +92,6 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
} }
} }
public <O extends FutureCommand> void submit(O operation) { public <O extends FutureCommand> void submit(O operation) {
exceptionIfNotActive(); exceptionIfNotActive();
commandQueue.add(operation); commandQueue.add(operation);
@ -99,17 +103,25 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
try { try {
connectionHandle = futureCommandConnectionPool.getHandle(operation); connectionHandle = futureCommandConnectionPool.getHandle(operation);
} catch (InterruptedException e) { } catch (InterruptedException e) {
logger.warn(e, "Interrupted getting a connection for operation %1s; retrying", operation); logger
.warn(
e,
"Interrupted getting a connection for operation %1s; retrying",
operation);
commandQueue.add(operation); commandQueue.add(operation);
return; return;
} catch (TimeoutException e) { } catch (TimeoutException e) {
logger.warn(e, "Timeout getting a connection for operation %1s; retrying", operation); logger.warn(e,
"Timeout getting a connection for operation %1s; retrying",
operation);
commandQueue.add(operation); commandQueue.add(operation);
return; return;
} }
if (connectionHandle == null) { if (connectionHandle == null) {
logger.error("Failed to obtain connection for operation %1s; retrying", operation); logger.error(
"Failed to obtain connection for operation %1s; retrying",
operation);
commandQueue.add(operation); commandQueue.add(operation);
return; return;
} }
@ -121,18 +133,12 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
final StringBuilder sb = new StringBuilder(); final StringBuilder sb = new StringBuilder();
sb.append("FutureCommandConnectionPoolClient"); sb.append("FutureCommandConnectionPoolClient");
sb.append("{status=").append(status); sb.append("{status=").append(status);
sb.append(", commandQueue=").append((commandQueue != null) ? commandQueue.size() : 0); sb.append(", commandQueue=").append(
sb.append(", futureCommandConnectionPool=").append(futureCommandConnectionPool); (commandQueue != null) ? commandQueue.size() : 0);
sb.append(", futureCommandConnectionPool=").append(
futureCommandConnectionPool);
sb.append('}'); sb.append('}');
return sb.toString(); return sb.toString();
} }
public void close(){
try {
closer.close();
} catch (IOException e) {
e.printStackTrace(); // TODO: Adrian: Customise this generated block
} }
}
}

View File

@ -0,0 +1,37 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.command.pool;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public interface PoolConstants {
public static final String PROPERTY_POOL_MAX_CONNECTIONS = "jclouds.pool.max_connections";
public static final String PROPERTY_POOL_IO_WORKER_THREADS = "jclouds.http.pool.io_worker_threads";
public static final String PROPERTY_POOL_REQUEST_INVOKER_THREADS = "jclouds.http.pool.request_invoker_threads";
public static final String PROPERTY_POOL_MAX_SESSION_FAILURES = "jclouds.http.pool.max_session_failures";
public static final String PROPERTY_POOL_MAX_CONNECTION_REUSE = "jclouds.http.pool.max_connection_reuse";
}

View File

@ -27,6 +27,7 @@ import com.google.inject.*;
import com.google.inject.name.Named; import com.google.inject.name.Named;
import org.jclouds.command.FutureCommand; import org.jclouds.command.FutureCommand;
import org.jclouds.lifecycle.config.LifeCycleModule; import org.jclouds.lifecycle.config.LifeCycleModule;
import org.jclouds.command.pool.PoolConstants;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
@ -50,8 +51,7 @@ public abstract class FutureCommandConnectionPoolClientModule<C> extends Abstrac
@Provides @Provides
@Singleton @Singleton
public abstract BlockingQueue<C> provideAvailablePool(@Named("jclouds.pool.max_connections") int max) throws Exception; public abstract BlockingQueue<C> provideAvailablePool(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception;
/** /**
* controls production and destruction of real connections. * controls production and destruction of real connections.
* <p/> * <p/>
@ -64,7 +64,7 @@ public abstract class FutureCommandConnectionPoolClientModule<C> extends Abstrac
*/ */
@Provides @Provides
@Singleton @Singleton
public Semaphore provideTotalConnectionSemaphore(@Named("jclouds.pool.max_connections") int max) throws Exception { public Semaphore provideTotalConnectionSemaphore(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception {
return new Semaphore(max, true); return new Semaphore(max, true);
} }
} }

View File

@ -28,7 +28,7 @@ package org.jclouds.http;
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
public class HttpConstants { public interface HttpConstants {
public static final String CONTENT_LENGTH = "Content-Length"; public static final String CONTENT_LENGTH = "Content-Length";
public static final String CONTENT_TYPE = "Content-Type"; public static final String CONTENT_TYPE = "Content-Type";
public static final String HOST = "Host"; public static final String HOST = "Host";
@ -36,4 +36,7 @@ public class HttpConstants {
public static final String BINARY = "application/octet-stream"; public static final String BINARY = "application/octet-stream";
public static final String PLAIN = "text/plain"; public static final String PLAIN = "text/plain";
public static final String TRANSFER_ENCODING = "Transfer-Encoding"; public static final String TRANSFER_ENCODING = "Transfer-Encoding";
public static final String PROPERTY_HTTP_SECURE = "jclouds.http.secure";
public static final String PROPERTY_HTTP_PORT = "jclouds.http.port";
public static final String PROPERTY_HTTP_ADDRESS = "jclouds.http.address";
} }

View File

@ -23,11 +23,12 @@
*/ */
package org.jclouds.http; package org.jclouds.http;
import com.google.inject.Inject; import java.util.List;
import org.jclouds.command.FutureCommand; import org.jclouds.command.FutureCommand;
import org.jclouds.command.FutureCommandClient; import org.jclouds.command.FutureCommandClient;
import java.util.List; import com.google.inject.Inject;
/** /**
* // TODO: Adrian: Document this! * // TODO: Adrian: Document this!
@ -41,6 +42,4 @@ public interface HttpFutureCommandClient extends FutureCommandClient {
void setRequestFilters(List<HttpRequestFilter> requestFilters); void setRequestFilters(List<HttpRequestFilter> requestFilters);
<O extends FutureCommand> void submit(O operation); <O extends FutureCommand> void submit(O operation);
void close();
} }

View File

@ -25,6 +25,7 @@ package org.jclouds.http;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
@ -82,35 +83,40 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
HttpResponse response = getResponse(connection); HttpResponse response = getResponse(connection);
logger.trace("%1s - received response %2s", target, response); logger.trace("%1s - received response %2s", target, response);
operation.getResponseFuture().setResponse(response);
operation.getResponseFuture().run();
} catch (FileNotFoundException e) {
HttpResponse response = new HttpResponse();
response.setStatusCode(404);
operation.getResponseFuture().setResponse(response); operation.getResponseFuture().setResponse(response);
operation.getResponseFuture().run(); operation.getResponseFuture().run();
} catch (Exception e) { } catch (Exception e) {
if (connection != null) { if (connection != null) {
InputStream errorStream = connection.getErrorStream(); StringBuilder errors = new StringBuilder();
if (errorStream != null) {
try { try {
String errorMessage = Utils.toStringAndClose(connection for (InputStream in : new InputStream[] {
.getErrorStream()); connection.getErrorStream(),
connection.getInputStream() }) {
if (in != null) {
errors.append(Utils.toStringAndClose(in)).append(
"\n");
}
}
logger.error(e, logger.error(e,
"error encountered during the exception: %1s", "error encountered during the exception: %1s",
errorMessage); errors.toString());
} catch (IOException e1) { } catch (IOException e2) {
} }
} }
}
operation.setException(e); operation.setException(e);
} finally { } finally {
// DO NOT disconnect, as it will also close the unconsumed // DO NOT disconnect, as it will also close the unconsumed
// outputStream from above. // outputStream from above.
// connection.disconnect(); if (request.getMethod().equals("HEAD"))
connection.disconnect();
} }
} }
public void close() {
// Nothing to stop;
}
private HttpResponse getResponse(HttpURLConnection connection) private HttpResponse getResponse(HttpURLConnection connection)
throws IOException { throws IOException {
HttpResponse response = new HttpResponse(); HttpResponse response = new HttpResponse();
@ -121,9 +127,11 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
} }
response.setMessage(connection.getResponseMessage()); response.setMessage(connection.getResponseMessage());
if (!connection.getRequestMethod().equals("HEAD")) {
response.setContent(connection.getInputStream()); response.setContent(connection.getInputStream());
response.setContentType(connection response.setContentType(connection
.getHeaderField(HttpConstants.CONTENT_TYPE)); .getHeaderField(HttpConstants.CONTENT_TYPE));
}
return response; return response;
} }

View File

@ -23,19 +23,20 @@
*/ */
package org.jclouds.http.config; package org.jclouds.http.config;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.JavaUrlHttpFutureCommandClient;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.JavaUrlHttpFutureCommandClient;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
/** /**
* // TODO: Adrian: Document this! * Configures {@link JavaUrlHttpFutureCommandClient}.
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@ -44,15 +45,19 @@ public class JavaUrlHttpFutureCommandClientModule extends AbstractModule {
@Override @Override
protected void configure() { protected void configure() {
// note this is not threadsafe, so it cannot be singleton // note this is not threadsafe, so it cannot be singleton
bind(HttpFutureCommandClient.class).to(JavaUrlHttpFutureCommandClient.class); bind(HttpFutureCommandClient.class).to(
JavaUrlHttpFutureCommandClient.class);
} }
@Singleton @Singleton
@Provides @Provides
protected URL provideAddress(@Named("jclouds.http.address") String address, @Named("jclouds.http.port") int port, @Named("jclouds.http.secure") boolean isSecure) throws MalformedURLException { protected URL provideAddress(
@Named(HttpConstants.PROPERTY_HTTP_ADDRESS) String address,
@Named(HttpConstants.PROPERTY_HTTP_PORT) int port,
@Named(HttpConstants.PROPERTY_HTTP_SECURE) boolean isSecure)
throws MalformedURLException {
return new URL(isSecure ? "https" : "http", address, port, "/"); return new URL(isSecure ? "https" : "http", address, port, "/");
} }
} }

View File

@ -40,6 +40,7 @@ import org.jclouds.http.commands.GetString;
import org.jclouds.http.commands.Head; import org.jclouds.http.commands.Head;
import org.jclouds.http.commands.callables.xml.ParseSax; import org.jclouds.http.commands.callables.xml.ParseSax;
import org.jclouds.http.commands.config.HttpCommandsModule; import org.jclouds.http.commands.config.HttpCommandsModule;
import org.jclouds.lifecycle.Closer;
import org.mortbay.jetty.Handler; import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Request; import org.mortbay.jetty.Request;
import org.mortbay.jetty.Server; import org.mortbay.jetty.Server;
@ -69,6 +70,7 @@ public abstract class BaseHttpFutureCommandClientTest {
protected CommandFactory factory; protected CommandFactory factory;
protected HttpFutureCommandClient client; protected HttpFutureCommandClient client;
protected Injector injector; protected Injector injector;
private Closer closer;
@BeforeClass @BeforeClass
@Parameters( { "test-jetty-port" }) @Parameters( { "test-jetty-port" })
@ -95,9 +97,9 @@ public abstract class BaseHttpFutureCommandClientTest {
server.setHandler(handler); server.setHandler(handler);
server.start(); server.start();
final Properties properties = new Properties(); final Properties properties = new Properties();
properties.put("jclouds.http.address", "localhost"); properties.put(HttpConstants.PROPERTY_HTTP_ADDRESS, "localhost");
properties.put("jclouds.http.port", testPort + ""); properties.put(HttpConstants.PROPERTY_HTTP_PORT, testPort + "");
properties.put("jclouds.http.secure", "false"); properties.put(HttpConstants.PROPERTY_HTTP_SECURE, "false");
addConnectionProperties(properties); addConnectionProperties(properties);
final List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>( final List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>(
1); 1);
@ -123,6 +125,7 @@ public abstract class BaseHttpFutureCommandClientTest {
}); });
factory = injector.getInstance(CommandFactory.class); factory = injector.getInstance(CommandFactory.class);
client = injector.getInstance(HttpFutureCommandClient.class); client = injector.getInstance(HttpFutureCommandClient.class);
closer = injector.getInstance(Closer.class);
assert client != null; assert client != null;
} }
@ -132,7 +135,7 @@ public abstract class BaseHttpFutureCommandClientTest {
@AfterClass @AfterClass
public void tearDownJetty() throws Exception { public void tearDownJetty() throws Exception {
client.close(); closer.close();
server.stop(); server.stop();
} }

View File

@ -61,7 +61,7 @@ public class HttpNioUtils {
String contentType, long length) { String contentType, long length) {
if (content instanceof InputStream) { if (content instanceof InputStream) {
InputStream inputStream = (InputStream) content; InputStream inputStream = (InputStream) content;
if (length <= 0) if (length == -1)
throw new IllegalArgumentException( throw new IllegalArgumentException(
"you must specify size when content is an InputStream"); "you must specify size when content is an InputStream");
InputStreamEntity entity = new InputStreamEntity(inputStream, InputStreamEntity entity = new InputStreamEntity(inputStream,

View File

@ -23,48 +23,51 @@
*/ */
package org.jclouds.http.httpnio.config; package org.jclouds.http.httpnio.config;
import com.google.inject.AbstractModule; import java.net.InetSocketAddress;
import com.google.inject.Provides;
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
import org.apache.http.nio.NHttpConnection; import org.apache.http.nio.NHttpConnection;
import org.jclouds.command.pool.FutureCommandConnectionRetry; import org.jclouds.command.pool.FutureCommandConnectionRetry;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.HttpFutureCommandClient; import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.httpnio.config.internal.NonSSLHttpNioConnectionPoolClientModule; import org.jclouds.http.httpnio.config.internal.NonSSLHttpNioConnectionPoolClientModule;
import org.jclouds.http.httpnio.config.internal.SSLHttpNioConnectionPoolClientModule; import org.jclouds.http.httpnio.config.internal.SSLHttpNioConnectionPoolClientModule;
import org.jclouds.http.httpnio.pool.HttpNioConnectionPoolClient; import org.jclouds.http.httpnio.pool.HttpNioConnectionPoolClient;
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionRetry; import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionRetry;
import java.net.InetSocketAddress; import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
/** /**
* // TODO: Adrian: Document this! * Configures {@link HttpNioConnectionPoolClient}
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
public class HttpNioConnectionPoolClientModule extends AbstractModule { public class HttpNioConnectionPoolClientModule extends AbstractModule {
@Named("jclouds.http.secure") @Named(HttpConstants.PROPERTY_HTTP_SECURE)
boolean isSecure; boolean isSecure;
@Override @Override
protected void configure() { protected void configure() {
requestInjection(this); requestInjection(this);
//TODO test...
if (isSecure) if (isSecure)
install(new SSLHttpNioConnectionPoolClientModule()); install(new SSLHttpNioConnectionPoolClientModule());
else else
install(new NonSSLHttpNioConnectionPoolClientModule()); install(new NonSSLHttpNioConnectionPoolClientModule());
bind(new TypeLiteral<FutureCommandConnectionRetry<NHttpConnection>>(){}).to(HttpNioFutureCommandConnectionRetry.class); bind(new TypeLiteral<FutureCommandConnectionRetry<NHttpConnection>>() {
bind(HttpFutureCommandClient.class).to(HttpNioConnectionPoolClient.class); }).to(HttpNioFutureCommandConnectionRetry.class);
bind(HttpFutureCommandClient.class).to(
HttpNioConnectionPoolClient.class);
} }
@Singleton @Singleton
@Provides @Provides
protected InetSocketAddress provideAddress(@Named("jclouds.http.address") String address, @Named("jclouds.http.port") int port) { protected InetSocketAddress provideAddress(
@Named(HttpConstants.PROPERTY_HTTP_ADDRESS) String address,
@Named(HttpConstants.PROPERTY_HTTP_PORT) int port) {
return new InetSocketAddress(address, port); return new InetSocketAddress(address, port);
} }

View File

@ -45,6 +45,7 @@ import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpParams; import org.apache.http.params.HttpParams;
import org.apache.http.protocol.*; import org.apache.http.protocol.*;
import org.jclouds.command.pool.FutureCommandConnectionRetry; import org.jclouds.command.pool.FutureCommandConnectionRetry;
import org.jclouds.command.pool.PoolConstants;
import org.jclouds.command.pool.config.FutureCommandConnectionPoolClientModule; import org.jclouds.command.pool.config.FutureCommandConnectionPoolClientModule;
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionHandle; import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionHandle;
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionPool; import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionPool;
@ -114,7 +115,7 @@ public abstract class BaseHttpNioConnectionPoolClientModule extends FutureComman
} }
@Override @Override
public BlockingQueue<NHttpConnection> provideAvailablePool(@Named("jclouds.pool.max_connections") int max) throws Exception { public BlockingQueue<NHttpConnection> provideAvailablePool(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception {
return new ArrayBlockingQueue<NHttpConnection>(max, true); return new ArrayBlockingQueue<NHttpConnection>(max, true);
} }
@ -125,7 +126,7 @@ public abstract class BaseHttpNioConnectionPoolClientModule extends FutureComman
@Provides @Provides
@Singleton @Singleton
public DefaultConnectingIOReactor provideDefaultConnectingIOReactor(@Named("jclouds.http.pool.io_worker_threads") int ioWorkerThreads, HttpParams params) throws IOReactorException { public DefaultConnectingIOReactor provideDefaultConnectingIOReactor(@Named(PoolConstants.PROPERTY_POOL_IO_WORKER_THREADS) int ioWorkerThreads, HttpParams params) throws IOReactorException {
return new DefaultConnectingIOReactor(ioWorkerThreads, params); return new DefaultConnectingIOReactor(ioWorkerThreads, params);
} }

View File

@ -23,8 +23,13 @@
*/ */
package org.jclouds.http.httpnio.pool; package org.jclouds.http.httpnio.pool;
import com.google.inject.Inject; import java.io.IOException;
import com.google.inject.name.Named; import java.net.InetSocketAddress;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.apache.http.HttpException; import org.apache.http.HttpException;
import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
import org.apache.http.nio.NHttpConnection; import org.apache.http.nio.NHttpConnection;
@ -38,21 +43,18 @@ import org.jclouds.Logger;
import org.jclouds.command.FutureCommand; import org.jclouds.command.FutureCommand;
import org.jclouds.command.pool.FutureCommandConnectionPool; import org.jclouds.command.pool.FutureCommandConnectionPool;
import org.jclouds.command.pool.FutureCommandConnectionRetry; import org.jclouds.command.pool.FutureCommandConnectionRetry;
import org.jclouds.command.pool.PoolConstants;
import java.io.IOException; import com.google.inject.Inject;
import java.net.InetSocketAddress; import com.google.inject.name.Named;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
/** /**
* // TODO: Adrian: Document this! * Connection Pool for HTTP requests that utilizes Apache HTTPNio
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionPool<NHttpConnection> implements EventListener { public class HttpNioFutureCommandConnectionPool extends
FutureCommandConnectionPool<NHttpConnection> implements EventListener {
private final NHttpClientConnectionPoolSessionRequestCallback sessionCallback; private final NHttpClientConnectionPoolSessionRequestCallback sessionCallback;
private final DefaultConnectingIOReactor ioReactor; private final DefaultConnectingIOReactor ioReactor;
@ -61,8 +63,22 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
private final int maxSessionFailures; private final int maxSessionFailures;
@Inject @Inject
public HttpNioFutureCommandConnectionPool(java.util.logging.Logger logger, ExecutorService executor, Semaphore allConnections, BlockingQueue<NHttpConnection> available, AsyncNHttpClientHandler clientHandler, DefaultConnectingIOReactor ioReactor, IOEventDispatch dispatch, FutureCommandConnectionHandleFactory requestHandleFactory, InetSocketAddress target, FutureCommandConnectionRetry<NHttpConnection> futureCommandConnectionRetry, @Named("jclouds.http.pool.max_connection_reuse") int maxConnectionReuse, @Named("jclouds.http.pool.max_session_failures") int maxSessionFailures) { public HttpNioFutureCommandConnectionPool(
super(new Logger(logger), executor, futureCommandConnectionRetry, allConnections, requestHandleFactory, maxConnectionReuse, available); java.util.logging.Logger logger,
ExecutorService executor,
Semaphore allConnections,
BlockingQueue<NHttpConnection> available,
AsyncNHttpClientHandler clientHandler,
DefaultConnectingIOReactor ioReactor,
IOEventDispatch dispatch,
FutureCommandConnectionHandleFactory requestHandleFactory,
InetSocketAddress target,
FutureCommandConnectionRetry<NHttpConnection> futureCommandConnectionRetry,
@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTION_REUSE) int maxConnectionReuse,
@Named(PoolConstants.PROPERTY_POOL_MAX_SESSION_FAILURES) int maxSessionFailures) {
super(new Logger(logger), executor, futureCommandConnectionRetry,
allConnections, requestHandleFactory, maxConnectionReuse,
available);
this.ioReactor = ioReactor; this.ioReactor = ioReactor;
this.dispatch = dispatch; this.dispatch = dispatch;
this.target = target; this.target = target;
@ -99,9 +115,9 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
} }
} }
public boolean connectionValid(NHttpConnection conn) { public boolean connectionValid(NHttpConnection conn) {
return conn.isOpen() && !conn.isStale() && conn.getMetrics().getRequestCount() < maxConnectionReuse; return conn.isOpen() && !conn.isStale()
&& conn.getMetrics().getRequestCount() < maxConnectionReuse;
} }
protected void doWork() throws Exception { protected void doWork() throws Exception {
@ -128,31 +144,38 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
@Override @Override
protected boolean shouldDoWork() { protected boolean shouldDoWork() {
return super.shouldDoWork() && ioReactor.getStatus().equals(IOReactorStatus.ACTIVE); return super.shouldDoWork()
&& ioReactor.getStatus().equals(IOReactorStatus.ACTIVE);
} }
class NHttpClientConnectionPoolSessionRequestCallback implements SessionRequestCallback { class NHttpClientConnectionPoolSessionRequestCallback implements
SessionRequestCallback {
public void completed(SessionRequest request) { public void completed(SessionRequest request) {
logger.trace("%1s - %2s - operation complete", request, request.getAttachment()); logger.trace("%1s - %2s - operation complete", request, request
.getAttachment());
} }
public void cancelled(SessionRequest request) { public void cancelled(SessionRequest request) {
logger.info("%1s - %2s - operation cancelled", request, request.getAttachment()); logger.info("%1s - %2s - operation cancelled", request, request
.getAttachment());
releaseConnectionAndCancelResponse(request); releaseConnectionAndCancelResponse(request);
} }
private void releaseConnectionAndCancelResponse(SessionRequest request) { private void releaseConnectionAndCancelResponse(SessionRequest request) {
allConnections.release(); allConnections.release();
FutureCommand frequest = (FutureCommand) request.getAttachment(); FutureCommand<?, ?, ?> frequest = (FutureCommand<?, ?, ?>) request
.getAttachment();
if (frequest != null) { if (frequest != null) {
frequest.cancel(true); frequest.cancel(true);
} }
} }
private void releaseConnectionAndSetResponseException(SessionRequest request, Exception e) { private void releaseConnectionAndSetResponseException(
SessionRequest request, Exception e) {
allConnections.release(); allConnections.release();
FutureCommand frequest = (FutureCommand) request.getAttachment(); FutureCommand<?, ?, ?> frequest = (FutureCommand<?, ?, ?>) request
.getAttachment();
if (frequest != null) { if (frequest != null) {
frequest.setException(e); frequest.setException(e);
} }
@ -160,8 +183,11 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
public void failed(SessionRequest request) { public void failed(SessionRequest request) {
int count = currentSessionFailures.getAndIncrement(); int count = currentSessionFailures.getAndIncrement();
logger.error(request.getException(), "%1s - %2s - operation failed", request, request.getAttachment()); logger.error(request.getException(),
releaseConnectionAndSetResponseException(request, request.getException()); "%1s - %2s - operation failed", request, request
.getAttachment());
releaseConnectionAndSetResponseException(request, request
.getException());
if (count >= maxSessionFailures) { if (count >= maxSessionFailures) {
exception = request.getException(); exception = request.getException();
} }
@ -169,22 +195,22 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
} }
public void timeout(SessionRequest request) { public void timeout(SessionRequest request) {
logger.warn("%1s - %2s - operation timed out", request, request.getAttachment()); logger.warn("%1s - %2s - operation timed out", request, request
.getAttachment());
releaseConnectionAndCancelResponse(request); releaseConnectionAndCancelResponse(request);
} }
} }
public void connectionOpen(NHttpConnection conn) { public void connectionOpen(NHttpConnection conn) {
conn.setSocketTimeout(0); conn.setSocketTimeout(0);
available.offer(conn); available.offer(conn);
logger.trace("%1s - %2d - open", conn, conn.hashCode()); logger.trace("%1s - %2d - open", conn, conn.hashCode());
} }
public void connectionTimeout(NHttpConnection conn) { public void connectionTimeout(NHttpConnection conn) {
logger.warn("%1s - %2d - timeout %2d", conn, conn.hashCode(), conn.getSocketTimeout()); logger.warn("%1s - %2d - timeout %2d", conn, conn.hashCode(), conn
.getSocketTimeout());
allConnections.release(); allConnections.release();
futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn); futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn);
} }
@ -196,18 +222,23 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
public void fatalIOException(IOException ex, NHttpConnection conn) { public void fatalIOException(IOException ex, NHttpConnection conn) {
exception = ex; exception = ex;
logger.error(ex, "%1s - %2d - %3s - pool error", conn, conn.hashCode(), target); logger.error(ex, "%1s - %2d - %3s - pool error", conn, conn.hashCode(),
target);
futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn); futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn);
} }
public void fatalProtocolException(HttpException ex, NHttpConnection conn) { public void fatalProtocolException(HttpException ex, NHttpConnection conn) {
exception = ex; exception = ex;
logger.error(ex, "%1s - %2d - %3s - http error", conn, conn.hashCode(), target); logger.error(ex, "%1s - %2d - %3s - http error", conn, conn.hashCode(),
target);
fatalException(ex, conn); fatalException(ex, conn);
} }
public static interface FutureCommandConnectionHandleFactory extends FutureCommandConnectionPool.FutureCommandConnectionHandleFactory<NHttpConnection> { public static interface FutureCommandConnectionHandleFactory
HttpNioFutureCommandConnectionHandle create(FutureCommand command, NHttpConnection conn); extends
FutureCommandConnectionPool.FutureCommandConnectionHandleFactory<NHttpConnection> {
HttpNioFutureCommandConnectionHandle create(FutureCommand command,
NHttpConnection conn);
} }
} }

View File

@ -24,6 +24,8 @@
package org.jclouds.http.httpnio.pool; package org.jclouds.http.httpnio.pool;
import com.google.inject.Module; import com.google.inject.Module;
import org.jclouds.command.pool.PoolConstants;
import org.jclouds.http.BaseHttpFutureCommandClientTest; import org.jclouds.http.BaseHttpFutureCommandClientTest;
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule; import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -36,14 +38,20 @@ import java.util.Properties;
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test @Test
public class HttpNioConnectionPoolFutureCommandClientTest extends BaseHttpFutureCommandClientTest { public class HttpNioConnectionPoolFutureCommandClientTest extends
BaseHttpFutureCommandClientTest {
protected void addConnectionProperties(Properties properties) { protected void addConnectionProperties(Properties properties) {
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75"); properties.setProperty(
properties.setProperty("jclouds.http.pool.max_session_failures", "2"); PoolConstants.PROPERTY_POOL_MAX_CONNECTION_REUSE, "75");
properties.setProperty("jclouds.http.pool.request_invoker_threads", "1"); properties.setProperty(
properties.setProperty("jclouds.http.pool.io_worker_threads", "2"); PoolConstants.PROPERTY_POOL_MAX_SESSION_FAILURES, "2");
properties.setProperty("jclouds.pool.max_connections", "12"); properties.setProperty(
PoolConstants.PROPERTY_POOL_REQUEST_INVOKER_THREADS, "1");
properties.setProperty(PoolConstants.PROPERTY_POOL_IO_WORKER_THREADS,
"2");
properties.setProperty(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS,
"12");
} }
protected Module createClientModule() { protected Module createClientModule() {

View File

@ -46,11 +46,48 @@
<url>http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</url> <url>http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</url>
</scm> </scm>
<properties>
<jclouds.aws.accesskeyid></jclouds.aws.accesskeyid>
<jclouds.aws.secretaccesskey></jclouds.aws.secretaccesskey>
</properties>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>${project.groupId}</groupId> <groupId>${project.groupId}</groupId>
<artifactId>jclouds-httpnio</artifactId> <artifactId>jclouds-httpnio</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemProperties>
<property>
<name>jclouds.aws.accesskeyid</name>
<value>${jclouds.aws.accesskeyid}</value>
</property>
<property>
<name>jclouds.aws.secretaccesskey</name>
<value>${jclouds.aws.secretaccesskey}</value>
</property>
</systemProperties>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>

View File

@ -0,0 +1,61 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.nio;
import java.util.Properties;
import org.jclouds.aws.s3.S3ObjectMapTest;
import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test;
import com.google.inject.Module;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest")
public class NioS3ObjectMapTest extends S3ObjectMapTest {
@Override
protected Properties buildS3Properties(String AWSAccessKeyId,
String AWSSecretAccessKey) {
Properties properties = super.buildS3Properties(AWSAccessKeyId,
AWSSecretAccessKey);
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
properties.setProperty("jclouds.http.pool.max_session_failures", "2");
properties
.setProperty("jclouds.http.pool.request_invoker_threads", "1");
properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
properties.setProperty("jclouds.pool.max_connections", "12");
return properties;
}
@Override
protected Module createHttpModule() {
return new S3HttpNioConnectionPoolClientModule();
}
}

View File

@ -44,7 +44,6 @@
<module>extensions/s3nio</module> <module>extensions/s3nio</module>
<module>s3</module> <module>s3</module>
<module>s3/perftest</module> <module>s3/perftest</module>
<module>samples/googleappengine</module>
</modules> </modules>
<build> <build>
<plugins> <plugins>

View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0" <metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<groupId>com.google.code.guice</groupId> <groupId>com.google.code.guice</groupId>

View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0" <metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<groupId>com.google.code.guice</groupId> <groupId>com.google.code.guice</groupId>

View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0" <metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<groupId>com.google.code.guice</groupId> <groupId>com.google.code.guice</groupId>

View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0" <metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<groupId>com.google.code.guice</groupId> <groupId>com.google.code.guice</groupId>

View File

@ -30,6 +30,7 @@ import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import org.jclouds.aws.s3.S3Constants;
import org.testng.annotations.BeforeTest; import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional; import org.testng.annotations.Optional;
import org.testng.annotations.Parameters; import org.testng.annotations.Parameters;
@ -46,7 +47,7 @@ public class AmazonPerformance extends BasePerformance {
@Override @Override
@BeforeTest @BeforeTest
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" }) @Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpClient(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception { protected void setUpClient(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception {
super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey); super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey);
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey, amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey,

View File

@ -34,6 +34,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.jclouds.aws.s3.S3Constants;
import org.jclouds.aws.s3.S3IntegrationTest; import org.jclouds.aws.s3.S3IntegrationTest;
import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Bucket;
import org.testng.annotations.AfterTest; import org.testng.annotations.AfterTest;
@ -79,7 +80,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
@Override @Override
@BeforeTest @BeforeTest
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" }) @Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpClient(@Optional String AWSAccessKeyId, protected void setUpClient(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey) throws Exception { @Optional String AWSSecretAccessKey) throws Exception {
super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey); super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey);

View File

@ -67,6 +67,11 @@
<artifactId>bcprov-jdk15</artifactId> <artifactId>bcprov-jdk15</artifactId>
<version>140</version> <version>140</version>
</dependency> </dependency>
<dependency>
<groupId>eu.medsea.mimeutil</groupId>
<artifactId>mime-util</artifactId>
<version>1.3</version>
</dependency>
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>

View File

@ -23,20 +23,18 @@
*/ */
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Bucket;
import java.io.IOException;
import java.io.Closeable;
import java.util.List; import java.util.List;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
/** /**
* // TODO: Adrian: Document this! * // TODO: Adrian: Document this!
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
public interface S3Connection extends Closeable { public interface S3Connection {
Future<S3Object> getObject(S3Bucket s3Bucket, String key); Future<S3Object> getObject(S3Bucket s3Bucket, String key);
Future<S3Object> headObject(S3Bucket s3Bucket, String key); Future<S3Object> headObject(S3Bucket s3Bucket, String key);
@ -56,6 +54,4 @@ public interface S3Connection extends Closeable {
Future<S3Bucket> getBucket(S3Bucket s3Bucket); Future<S3Bucket> getBucket(S3Bucket s3Bucket);
Future<List<S3Bucket>> getBuckets(); Future<List<S3Bucket>> getBuckets();
public void close() throws IOException;
} }

View File

@ -1,116 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.name.Names;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import java.util.Properties;
/**
* // TODO: Adrian: Document return getConnection!
*
* @author Adrian Cole
*/
public class S3ConnectionFactory {
public static final Properties DEFAULT_PROPERTIES;
static {
DEFAULT_PROPERTIES = new Properties();
DEFAULT_PROPERTIES.setProperty("jclouds.http.address", "s3.amazonaws.com");
DEFAULT_PROPERTIES.setProperty("jclouds.http.port", "443");
DEFAULT_PROPERTIES.setProperty("jclouds.http.secure", "true");
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.max_connection_reuse", "75");
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.max_session_failures", "2");
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.request_invoker_threads", "1");
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.io_worker_threads", "2");
DEFAULT_PROPERTIES.setProperty("jclouds.pool.max_connections", "12");
}
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
}
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
if (!isSecure)
properties.setProperty("jclouds.http.port", "80");
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
}
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure,
String server) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
properties.setProperty("jclouds.http.address", server);
if (!isSecure)
properties.setProperty("jclouds.http.port", "80");
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
}
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure,
String server, int port) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
properties.setProperty("jclouds.http.address", server);
properties.setProperty("jclouds.http.port", port + "");
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
}
/**
* Create a new interface to interact with S3 with the given credential and connection
* parameters
*/
public static synchronized S3Connection getConnection(final Properties properties, Module httpModule) {
return getInjector(properties, httpModule).getInstance(S3Connection.class);
}
/**
* Create a new interface to interact with S3 with the given credential and connection
* parameters
*/
public static synchronized Injector getInjector(final Properties properties, Module httpModule) {
return Guice.createInjector(new AbstractModule() {
@Override
protected void configure() {
Names.bindProperties(binder(), properties);
}
}, httpModule, new S3ConnectionModule());
}
}

View File

@ -23,6 +23,7 @@
*/ */
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import org.jclouds.command.pool.PoolConstants;
import org.jclouds.http.HttpConstants; import org.jclouds.http.HttpConstants;
/** /**
@ -30,6 +31,8 @@ import org.jclouds.http.HttpConstants;
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
public class S3Constants extends HttpConstants { public interface S3Constants extends HttpConstants, PoolConstants {
public static final String AUTH = "Authorization"; public static final String AUTH = "Authorization";
public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey";
public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid";
} }

View File

@ -0,0 +1,49 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import org.jclouds.aws.s3.domain.S3Bucket;
public interface S3Context {
/**
* @return a connection to S3
*/
S3Connection getConnection();
/**
* Creates a <code>Map<String,InputStream</code> view of the specified
* bucket.
*
* @param bucket
* @return
*/
S3ObjectMap createMapView(S3Bucket bucket);
/**
* Closes all connections to S3.
*/
void close();
}

View File

@ -0,0 +1,148 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import org.jclouds.aws.s3.config.S3ContextModule;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.name.Names;
import static org.jclouds.aws.s3.S3Constants.*;
/**
* // TODO: Adrian: Document return getConnection!
*
* @author Adrian Cole
*/
public class S3ContextFactory {
public static final Properties DEFAULT_PROPERTIES;
static {
DEFAULT_PROPERTIES = new Properties();
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_ADDRESS,
"s3.amazonaws.com");
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_PORT, "443");
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_SECURE, "true");
DEFAULT_PROPERTIES
.setProperty(PROPERTY_POOL_MAX_CONNECTION_REUSE, "75");
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_MAX_SESSION_FAILURES, "2");
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_REQUEST_INVOKER_THREADS,
"1");
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_IO_WORKER_THREADS, "2");
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_MAX_CONNECTIONS, "12");
}
public static S3Context createS3Context(String awsAccessKeyId,
String awsSecretAccessKey, Module... modules) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
properties
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
return createS3Context(properties, modules);
}
public static S3Context createS3Context(String awsAccessKeyId,
String awsSecretAccessKey, boolean isSecure, Module... modules) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
properties
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
properties
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
if (!isSecure)
properties.setProperty(PROPERTY_HTTP_PORT, "80");
return createS3Context(properties, modules);
}
public static S3Context createS3Context(String awsAccessKeyId,
String awsSecretAccessKey, boolean isSecure, String server,
Module... modules) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
properties
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
properties
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
properties.setProperty(PROPERTY_HTTP_ADDRESS, server);
if (!isSecure)
properties.setProperty(PROPERTY_HTTP_PORT, "80");
return createS3Context(properties, modules);
}
public static S3Context createS3Context(String awsAccessKeyId,
String awsSecretAccessKey, boolean isSecure, String server,
int port, Module... modules) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
properties
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
properties
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
properties.setProperty(PROPERTY_HTTP_ADDRESS, server);
properties.setProperty(PROPERTY_HTTP_PORT, port + "");
return createS3Context(properties, modules);
}
public static S3Context createS3Context(Properties properties,
Module... modules) {
return getInjector(properties, modules).getInstance(S3Context.class);
}
/**
* Bind the given properties and install the list of modules. If no modules
* are specified, install the default
* {@link JavaUrlHttpFutureCommandClientModule}
*
* @param properties
* - contains constants used by jclouds
* {@link #DEFAULT_PROPERTIES}
* @param httpModules
* - modules that must bind {@link HttpFutureCommandClient} if
* specified
* */
public static Injector getInjector(final Properties properties,
Module... httpModules) {
final List<? extends Module> modules = httpModules.length != 0 ? Arrays
.asList(httpModules) : Collections
.singletonList(new JavaUrlHttpFutureCommandClientModule());
return Guice.createInjector(new AbstractModule() {
@Override
protected void configure() {
Names.bindProperties(binder(), properties);
for (Module module : modules)
install(module);
}
}, new S3ContextModule());
}
}

View File

@ -23,138 +23,32 @@
*/ */
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import com.google.inject.Inject; import java.io.File;
import org.jclouds.Logger; import java.io.InputStream;
import org.jclouds.aws.s3.domain.S3Object; import java.util.Map;
import java.util.Set;
import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import java.util.*; public interface S3ObjectMap extends Map<String, InputStream> {
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
/** InputStream putString(String key, String value);
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public class S3ObjectMap implements ConcurrentMap<String, Object> {
private Logger logger;
private S3Connection connection;
private S3Bucket bucket;
private S3Utils utils;
@Inject InputStream putFile(String key, File value);
public S3ObjectMap(java.util.logging.Logger logger, S3Connection connection, S3Bucket bucket, S3Utils utils) {
this.logger = new Logger(logger);
this.connection = connection;
this.bucket = bucket;
this.utils = utils;
}
InputStream putBytes(String key, byte[] value);
public Object putIfAbsent(String s, Object o) { void putAllStrings(Map<? extends String, ? extends String> map);
return null; // TODO: Adrian: Customise this generated block
}
public boolean remove(Object o, Object o1) { void putAllBytes(Map<? extends String, ? extends byte[]> map);
return false; // TODO: Adrian: Customise this generated block
}
public boolean replace(String s, Object o, Object o1) { void putAllFiles(Map<? extends String, ? extends File> map);
return false; // TODO: Adrian: Customise this generated block
}
public Object replace(String s, Object o) { InputStream put(S3Object object);
return null; // TODO: Adrian: Customise this generated block
}
public int size() { void putAll(Set<S3Object> objects);
try {
bucket = connection.getBucket(bucket).get();
return bucket.getContents().size();
} catch (Exception e) {
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
}
}
public boolean isEmpty() { S3Bucket getBucket();
return false; // TODO: Adrian: Customise this generated block
}
public boolean containsKey(Object o) {
return false; // TODO: Adrian: Customise this generated block
}
public boolean containsValue(Object o) {
return false; // TODO: Adrian: Customise this generated block
}
public Object get(Object o) {
try {
return connection.getObject(bucket, o.toString()).get();
} catch (Exception e) {
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format("Error geting object %1s:%2s", bucket, o), e);
}
}
public Object put(String s, Object o) {
S3Object object = new S3Object();
try {
object.setKey(s);
object.setContent(o);
return connection.addObject(bucket, object).get();
} catch (Exception e) {
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format("Error adding object %1s:%2s", bucket, object), e);
}
}
public Object remove(Object o) {
return null; // TODO: Adrian: Customise this generated block
}
public void putAll(Map<? extends String, ? extends Object> map) {
// TODO: Adrian: Customise this generated block
}
private class S3RuntimeException extends RuntimeException {
public S3RuntimeException(String s) {
super(s); // TODO: Adrian: Customise this generated block
}
public S3RuntimeException(String s, Throwable throwable) {
super(s, throwable); // TODO: Adrian: Customise this generated block
}
}
public void clear() {
try {
bucket = connection.getBucket(bucket).get();
List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
for (S3Object object : bucket.getContents()) {
deletes.add(connection.deleteObject(bucket, object.getKey()));
}
for (Future<Boolean> isdeleted : deletes)
if (!isdeleted.get()) {
throw new S3RuntimeException("failed to delete entry");
}
} catch (Exception e) {
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
}
}
public Set<String> keySet() {
return null; // TODO: Adrian: Customise this generated block
}
public Collection<Object> values() {
return null; // TODO: Adrian: Customise this generated block
}
public Set<Entry<String, Object>> entrySet() {
return null; // TODO: Adrian: Customise this generated block
}
} }

View File

@ -23,6 +23,14 @@
*/ */
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import org.bouncycastle.crypto.digests.MD5Digest;
import org.bouncycastle.crypto.digests.SHA1Digest; import org.bouncycastle.crypto.digests.SHA1Digest;
import org.bouncycastle.crypto.macs.HMac; import org.bouncycastle.crypto.macs.HMac;
import org.bouncycastle.crypto.params.KeyParameter; import org.bouncycastle.crypto.params.KeyParameter;
@ -30,16 +38,29 @@ import org.bouncycastle.util.encoders.Base64;
import org.jclouds.Utils; import org.jclouds.Utils;
import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.aws.s3.domain.S3Object;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
public class S3Utils extends Utils { public class S3Utils extends Utils {
static final byte[] HEX_CHAR_TABLE = { (byte) '0', (byte) '1', (byte) '2',
(byte) '3', (byte) '4', (byte) '5', (byte) '6', (byte) '7',
(byte) '8', (byte) '9', (byte) 'a', (byte) 'b', (byte) 'c',
(byte) 'd', (byte) 'e', (byte) 'f' };
public static String digest(String toEncode, byte[] key) throws NoSuchAlgorithmException, NoSuchProviderException, InvalidKeyException { public static String getHexString(byte[] raw)
throws UnsupportedEncodingException {
byte[] hex = new byte[2 * raw.length];
int index = 0;
for (byte b : raw) {
int v = b & 0xFF;
hex[index++] = HEX_CHAR_TABLE[v >>> 4];
hex[index++] = HEX_CHAR_TABLE[v & 0xF];
}
return new String(hex, "ASCII");
}
public static String hmacSha1Base64(String toEncode, byte[] key)
throws NoSuchAlgorithmException, NoSuchProviderException,
InvalidKeyException {
HMac hmac = new HMac(new SHA1Digest()); HMac hmac = new HMac(new SHA1Digest());
byte[] resBuf = new byte[hmac.getMacSize()]; byte[] resBuf = new byte[hmac.getMacSize()];
byte[] plainBytes = toEncode.getBytes(); byte[] plainBytes = toEncode.getBytes();
@ -50,8 +71,45 @@ public class S3Utils extends Utils {
return new String(Base64.encode(resBuf)); return new String(Base64.encode(resBuf));
} }
public static String md5Hex(byte [] toEncode)
throws NoSuchAlgorithmException, NoSuchProviderException,
InvalidKeyException, UnsupportedEncodingException {
byte[] resBuf = md5(toEncode);
return getHexString(resBuf);
}
public static String getContentAsStringAndClose(S3Object object) throws IOException { public static String md5Base64(byte [] toEncode)
throws NoSuchAlgorithmException, NoSuchProviderException,
InvalidKeyException {
byte[] resBuf = md5(toEncode);
return new String(Base64.encode(resBuf));
}
public static byte[] md5(byte[] plainBytes) {
MD5Digest md5 = new MD5Digest();
byte[] resBuf = new byte[md5.getDigestSize()];
md5.update(plainBytes, 0, plainBytes.length);
md5.doFinal(resBuf, 0);
return resBuf;
}
public static byte[] md5(InputStream toEncode) throws IOException {
MD5Digest md5 = new MD5Digest();
byte[] resBuf = new byte[md5.getDigestSize()];
byte[] buffer = new byte[1024];
int numRead = -1;
do {
numRead = toEncode.read(buffer);
if (numRead > 0) {
md5.update(buffer, 0, numRead);
}
} while (numRead != -1);
md5.doFinal(resBuf, 0);
return resBuf;
}
public static String getContentAsStringAndClose(S3Object object)
throws IOException {
Object o = object.getContent(); Object o = object.getContent();
if (o instanceof InputStream) { if (o instanceof InputStream) {
@ -61,7 +119,8 @@ public class S3Utils extends Utils {
} }
return returnVal; return returnVal;
} else { } else {
throw new IllegalArgumentException("Object type not supported: " + o.getClass().getName()); throw new IllegalArgumentException("Object type not supported: "
+ o.getClass().getName());
} }
} }
} }

View File

@ -67,7 +67,10 @@ public class RetrieveObjectCallable extends
object.setLastModified(dateParser object.setLastModified(dateParser
.dateTimeFromHeaderFormat(getResponse() .dateTimeFromHeaderFormat(getResponse()
.getFirstHeaderOrNull("Last-Modified"))); .getFirstHeaderOrNull("Last-Modified")));
object.setETag(getResponse().getFirstHeaderOrNull("ETag")); String eTag = getResponse().getFirstHeaderOrNull("ETag");
if (eTag != null) {
object.setETag(eTag.replaceAll("\"", ""));
}
object.setContentType(getResponse().getFirstHeaderOrNull( object.setContentType(getResponse().getFirstHeaderOrNull(
"Content-Type")); "Content-Type"));
object.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull( object.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull(

View File

@ -68,7 +68,7 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<S3Bucket> {
} else if (qName.equals("LastModified")) { } else if (qName.equals("LastModified")) {
currentObject.setLastModified(dateParser.dateTimeFromXMLFormat(currentText.toString())); currentObject.setLastModified(dateParser.dateTimeFromXMLFormat(currentText.toString()));
} else if (qName.equals("ETag")) { } else if (qName.equals("ETag")) {
currentObject.setETag(currentText.toString()); currentObject.setETag(currentText.toString().replaceAll("\"", ""));
} else if (qName.equals("Size")) { } else if (qName.equals("Size")) {
currentObject.setSize(Long.parseLong(currentText.toString())); currentObject.setSize(Long.parseLong(currentText.toString()));
} else if (qName.equals("Owner")) { } else if (qName.equals("Owner")) {

View File

@ -21,39 +21,55 @@
* under the License. * under the License.
* ==================================================================== * ====================================================================
*/ */
package org.jclouds.aws.s3; package org.jclouds.aws.s3.config;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.Scopes;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader;
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
import org.jclouds.http.HttpRequestFilter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Context;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader;
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
import org.jclouds.aws.s3.internal.GuiceS3Context;
import org.jclouds.aws.s3.internal.LiveS3Connection;
import org.jclouds.aws.s3.internal.LiveS3ObjectMap;
import org.jclouds.aws.s3.internal.GuiceS3Context.S3ObjectMapFactory;
import org.jclouds.http.HttpRequestFilter;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryProvider;
/** /**
* // TODO: Adrian: Document this! * // TODO: Adrian: Document this!
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
public class S3ConnectionModule extends AbstractModule { public class S3ContextModule extends AbstractModule {
@Override
protected void configure() { protected void configure() {
install(new S3CommandsModule()); install(new S3CommandsModule());
bind(S3Connection.class).to(LiveS3Connection.class).in(Scopes.SINGLETON); bind(S3Connection.class).to(LiveS3Connection.class)
.in(Scopes.SINGLETON);
bind(GuiceS3Context.S3ObjectMapFactory.class).toProvider(
FactoryProvider.newFactory(
GuiceS3Context.S3ObjectMapFactory.class,
LiveS3ObjectMap.class));
bind(S3Context.class).to(GuiceS3Context.class);
} }
@Provides @Provides
@Singleton @Singleton
List<HttpRequestFilter> provideRequestFilters(RemoveTransferEncodingHeader removTransferEncodingHeader, RequestAuthorizeSignature requestAuthorizeSignature) { List<HttpRequestFilter> provideRequestFilters(
RemoveTransferEncodingHeader removTransferEncodingHeader,
RequestAuthorizeSignature requestAuthorizeSignature) {
List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>(); List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>();
filters.add(removTransferEncodingHeader); filters.add(removTransferEncodingHeader);
filters.add(requestAuthorizeSignature); filters.add(requestAuthorizeSignature);
return filters; return filters;
} }
} }

View File

@ -99,8 +99,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
@Inject @Inject
public RequestAuthorizeSignature( public RequestAuthorizeSignature(
@Named("jclouds.aws.accesskeyid") String accessKey, @Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey,
@Named("jclouds.aws.secretaccesskey") String secretKey, @Named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY) String secretKey,
DateService dateService) { DateService dateService) {
this.accessKey = accessKey; this.accessKey = accessKey;
this.secretKey = secretKey; this.secretKey = secretKey;
@ -142,7 +142,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
toSign.append(request.getUri()); toSign.append(request.getUri());
String signature; String signature;
try { try {
signature = S3Utils.digest(toSign.toString(), secretKey.getBytes()); signature = S3Utils.hmacSha1Base64(toSign.toString(), secretKey.getBytes());
} catch (Exception e) { } catch (Exception e) {
throw new HttpException("error signing request", e); throw new HttpException("error signing request", e);
} }

View File

@ -0,0 +1,88 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.internal;
import java.io.IOException;
import org.jclouds.Logger;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Context;
import org.jclouds.aws.s3.S3ObjectMap;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.lifecycle.Closer;
import com.google.inject.Inject;
import com.google.inject.Injector;
/**
* // TODO: Adrian: Document return getConnection!
*
* @author Adrian Cole
*/
public class GuiceS3Context implements S3Context {
public interface S3ObjectMapFactory {
S3ObjectMap createMapView(S3Bucket bucket);
}
private final Logger logger;
private final Injector injector;
private final S3ObjectMapFactory s3ObjectMapFactory;
private final Closer closer;
@Inject
private GuiceS3Context(java.util.logging.Logger logger, Injector injector,
Closer closer, S3ObjectMapFactory s3ObjectMapFactory) {
this.logger = new Logger(logger);
this.injector = injector;
this.s3ObjectMapFactory = s3ObjectMapFactory;
this.closer = closer;
}
/**
* {@inheritDoc}
*/
public S3Connection getConnection() {
return injector.getInstance(S3Connection.class);
}
/**
* {@inheritDoc}
*/
public S3ObjectMap createMapView(S3Bucket bucket) {
getConnection().createBucketIfNotExists(bucket);
return s3ObjectMapFactory.createMapView(bucket);
}
/**
* {@inheritDoc}
*/
public void close() {
try {
closer.close();
} catch (IOException e) {
logger.error(e, "error closing content");
}
}
}

View File

@ -21,12 +21,12 @@
* under the License. * under the License.
* ==================================================================== * ====================================================================
*/ */
package org.jclouds.aws.s3; package org.jclouds.aws.s3.internal;
import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.commands.CopyObject; import org.jclouds.aws.s3.commands.CopyObject;
import org.jclouds.aws.s3.commands.DeleteBucket; import org.jclouds.aws.s3.commands.DeleteBucket;
import org.jclouds.aws.s3.commands.DeleteObject; import org.jclouds.aws.s3.commands.DeleteObject;
@ -127,9 +127,4 @@ public class LiveS3Connection implements S3Connection {
client.submit(listRequest); client.submit(listRequest);
return listRequest; return listRequest;
} }
public void close() throws IOException {
client.close();
}
} }

View File

@ -0,0 +1,433 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.internal;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.apache.commons.io.IOUtils;
import org.jclouds.Utils;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3ObjectMap;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
/**
* Map representation of a live connection to S3.
*
* @author Adrian Cole
*/
public class LiveS3ObjectMap implements S3ObjectMap {
private final S3Connection connection;
private final S3Bucket bucket;
@Inject
public LiveS3ObjectMap(S3Connection connection, @Assisted S3Bucket bucket) {
this.connection = connection;
this.bucket = bucket;
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#size()
*/
public int size() {
try {
return refreshBucket().getContents().size();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#get(java.lang.Object)
*/
public InputStream get(Object o) {
try {
return (InputStream) (connection.getObject(bucket, o.toString())
.get()).getContent();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error geting object %1s:%2s", bucket, o), e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#remove(java.lang.Object)
*/
public InputStream remove(Object o) {
InputStream old = get(o);
try {
connection.deleteObject(bucket, o.toString()).get();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error removing object %1s:%2s", bucket, o), e);
}
return old;
}
public class S3RuntimeException extends RuntimeException {
S3RuntimeException(String s) {
super(s);
}
public S3RuntimeException(String s, Throwable throwable) {
super(s, throwable);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#clear()
*/
public void clear() {
try {
List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
for (String key : keySet()) {
deletes.add(connection.deleteObject(bucket, key));
}
for (Future<Boolean> isdeleted : deletes)
if (!isdeleted.get()) {
throw new S3RuntimeException("failed to delete entry");
}
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
}
}
private S3Bucket refreshBucket() throws InterruptedException,
ExecutionException {
return connection.getBucket(bucket).get();
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#keySet()
*/
public Set<String> keySet() {
try {
Set<String> keys = new HashSet<String>();
for (S3Object object : refreshBucket().getContents())
keys.add(object.getKey());
return keys;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting keys in bucket: "
+ bucket, e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#values()
*/
public Collection<InputStream> values() {
Collection<InputStream> values = new LinkedList<InputStream>();
Set<Future<S3Object>> futureObjects = new HashSet<Future<S3Object>>();
for (String key : keySet()) {
futureObjects.add(connection.getObject(bucket, key));
}
for (Future<S3Object> futureObject : futureObjects) {
S3Object object = null;
try {
object = futureObject.get();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error getting value from bucket %1s:%2s", bucket,
object != null ? object.getKey() : "unknown"), e);
}
System.err.printf("key: %1s, MD5: %2s", object.getKey(), object
.getContentMD5());
values.add((InputStream) object.getContent());
}
return values;
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#entrySet()
*/
public Set<Map.Entry<String, InputStream>> entrySet() {
Set<Map.Entry<String, InputStream>> entrySet = new HashSet<Map.Entry<String, InputStream>>();
for (String key : keySet()) {
Map.Entry<String, InputStream> entry = new Entry(key, get(key));
entrySet.add(entry);
}
return entrySet;
}
public class Entry implements java.util.Map.Entry<String, InputStream> {
private InputStream value;
private String key;
Entry(String key, InputStream value) {
this.key = key;
this.value = value;
}
public String getKey() {
return key;
}
public InputStream getValue() {
return value;
}
public InputStream setValue(InputStream value) {
return put(key, value);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#containsKey(java.lang.Object)
*/
public boolean containsKey(Object key) {
try {
return connection.headObject(bucket, key.toString()).get() != S3Object.NOT_FOUND;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for %1s:%2s", bucket, key), e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#containsValue(java.lang.Object)
*/
public boolean containsValue(Object value) {
try {
byte[] md5;
if (value instanceof InputStream) {
md5 = S3Utils.md5((InputStream) value);
} else if (value instanceof byte[]) {
md5 = S3Utils.md5((byte[]) value);
} else if (value instanceof String) {
md5 = S3Utils.md5(((String) value).getBytes());
} else if (value instanceof File) {
md5 = S3Utils.md5(new FileInputStream((File) value));
} else {
throw new IllegalArgumentException("unsupported value type: "
+ value.getClass());
}
String eTagOfValue = S3Utils.getHexString(md5);
for (S3Object object : refreshBucket().getContents()) {
if (object.getETag().equals(eTagOfValue))
return true;
}
return false;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for ETAG of value: [%2s] in bucket:%1s",
bucket, value), e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#isEmpty()
*/
public boolean isEmpty() {
return keySet().size() == 0;
}
private InputStream putInternal(String s, Object o) {
S3Object object = new S3Object();
try {
InputStream returnVal = containsKey(s) ? get(s) : null;
object.setKey(s);
object.setContent(o);
setSizeIfContentIsInputStream(object);
connection.addObject(bucket, object).get();
return returnVal;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error adding object %1s:%2s", bucket, object), e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#putAll(java.util.Map)
*/
public void putAll(Map<? extends String, ? extends InputStream> map) {
putAllInternal(map);
}
public void putAllBytes(Map<? extends String, ? extends byte[]> map) {
putAllInternal(map);
}
public void putAllFiles(Map<? extends String, ? extends File> map) {
putAllInternal(map);
}
public void putAllStrings(Map<? extends String, ? extends String> map) {
putAllInternal(map);
}
private void putAllInternal(Map<? extends String, ? extends Object> map) {
try {
List<Future<String>> puts = new ArrayList<Future<String>>();
for (String key : map.keySet()) {
S3Object object = new S3Object();
object.setKey(key);
object.setContent(map.get(key));
setSizeIfContentIsInputStream(object);
puts.add(connection.addObject(bucket, object));
}
for (Future<String> put : puts)
put.get();// this will throw an exception if there was a problem
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error putting into bucket" + bucket,
e);
}
}
private void setSizeIfContentIsInputStream(S3Object object)
throws IOException {
if (object.getContent() instanceof InputStream) {
byte[] buffer = IOUtils.toByteArray((InputStream) object
.getContent());
object.setSize(buffer.length);
object.setContent(new ByteArrayInputStream(buffer));
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#putString(java.lang.String,
* java.lang.String)
*/
public InputStream putString(String key, String value) {
return putInternal(key, value);
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#putFile(java.lang.String,
* java.io.File)
*/
public InputStream putFile(String key, File value) {
return putInternal(key, value);
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#putBytes(java.lang.String, byte[])
*/
public InputStream putBytes(String key, byte[] value) {
return putInternal(key, value);
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#put(java.lang.String,
* java.io.InputStream)
*/
public InputStream put(String key, InputStream value) {
return putInternal(key, value);
}
public S3Bucket getBucket() {
return bucket;
}
public InputStream put(S3Object object) {
InputStream returnVal = get(object.getKey());
try {
connection.addObject(bucket, object).get();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error putting object %1s:%2s", bucket, object.getKey()), e);
}
return returnVal;
}
public void putAll(Set<S3Object> objects) {
try {
List<Future<String>> puts = new ArrayList<Future<String>>();
for (S3Object object : objects) {
puts.add(connection.addObject(bucket, object));
}
for (Future<String> put : puts)
put.get();// this will throw an exception if there was a problem
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error putting into bucket" + bucket,
e);
}
}
}

View File

@ -41,6 +41,7 @@ import java.util.logging.Logger;
import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import org.testng.annotations.AfterTest; import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest; import org.testng.annotations.BeforeTest;
@ -48,7 +49,6 @@ import org.testng.annotations.Optional;
import org.testng.annotations.Parameters; import org.testng.annotations.Parameters;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.inject.Injector;
import com.google.inject.Module; import com.google.inject.Module;
@Test(sequential = true) @Test(sequential = true)
@ -90,44 +90,45 @@ public class S3IntegrationTest {
+ "/adriancole.s3.amazons3test.filetestsforadrian/file</StringToSign><AWSAccessKeyId>0101100101001001</AWSAccessKeyId></Error>"; + "/adriancole.s3.amazons3test.filetestsforadrian/file</StringToSign><AWSAccessKeyId>0101100101001001</AWSAccessKeyId></Error>";
String amazonHadAnError = "<Error><Code>InternalError</Code><Message>We encountered an internal error. Please try again.</Message><RequestId>EF6FA7A639CAFF15</RequestId><HostId>tBkX23mIeq2riHsNw2YShupMlZ9+iy3V/uN+lRhqCR4qHTE07ujFeyAUPTowvuH/</HostId></Error>"; String amazonHadAnError = "<Error><Code>InternalError</Code><Message>We encountered an internal error. Please try again.</Message><RequestId>EF6FA7A639CAFF15</RequestId><HostId>tBkX23mIeq2riHsNw2YShupMlZ9+iy3V/uN+lRhqCR4qHTE07ujFeyAUPTowvuH/</HostId></Error>";
protected S3Connection client; protected S3Connection client;
Injector i = null; protected S3Context context = null;
protected String bucketPrefix = System.getProperty("user.name") + "." protected String bucketPrefix = System.getProperty("user.name") + "."
+ this.getClass().getName(); + this.getClass().getName();
private static final String sysAWSAccessKeyId = System private static final String sysAWSAccessKeyId = System
.getProperty("jclouds.aws.accesskeyid"); .getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
private static final String sysAWSSecretAccessKey = System private static final String sysAWSSecretAccessKey = System
.getProperty("jclouds.aws.secretaccesskey"); .getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
@BeforeTest @BeforeTest
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" }) @Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID,
S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpClient(@Optional String AWSAccessKeyId, protected void setUpClient(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey) throws Exception { @Optional String AWSSecretAccessKey) throws Exception {
i = createInject(AWSAccessKeyId != null ? AWSAccessKeyId context = createS3Context(AWSAccessKeyId != null ? AWSAccessKeyId
: sysAWSAccessKeyId, : sysAWSAccessKeyId,
AWSSecretAccessKey != null ? AWSSecretAccessKey AWSSecretAccessKey != null ? AWSSecretAccessKey
: sysAWSSecretAccessKey); : sysAWSSecretAccessKey);
client = i.getInstance(LiveS3Connection.class); client = context.getConnection();
deleteEverything(); deleteEverything();
} }
protected Injector createInject(String AWSAccessKeyId, protected S3Context createS3Context(String AWSAccessKeyId,
String AWSSecretAccessKey) { String AWSSecretAccessKey) {
return S3ConnectionFactory.getInjector(buildS3Properties( return S3ContextFactory.createS3Context(buildS3Properties(
AWSAccessKeyId, AWSSecretAccessKey), createHttpModule()); AWSAccessKeyId, AWSSecretAccessKey), createHttpModule());
} }
protected Properties buildS3Properties(String AWSAccessKeyId, protected Properties buildS3Properties(String AWSAccessKeyId,
String AWSSecretAccessKey) { String AWSSecretAccessKey) {
Properties properties = new Properties( Properties properties = new Properties(
S3ConnectionFactory.DEFAULT_PROPERTIES); S3ContextFactory.DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", checkNotNull( properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID,
AWSAccessKeyId, "AWSAccessKeyId")); checkNotNull(AWSAccessKeyId, "AWSAccessKeyId"));
properties.setProperty("jclouds.aws.secretaccesskey", checkNotNull( properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY,
AWSSecretAccessKey, "AWSSecretAccessKey")); checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey"));
properties.setProperty("jclouds.http.secure", "false"); properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false");
properties.setProperty("jclouds.http.port", "80"); properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80");
return properties; return properties;
} }
@ -163,8 +164,8 @@ public class S3IntegrationTest {
@AfterTest @AfterTest
protected void tearDownClient() throws Exception { protected void tearDownClient() throws Exception {
deleteEverything(); deleteEverything();
client.close(); context.close();
i = null; context = null;
} }
} }

View File

@ -23,59 +23,307 @@
*/ */
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import static org.testng.Assert.assertEquals;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
import org.apache.commons.io.IOUtils;
import org.jclouds.Utils;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.AfterMethod; import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import org.jclouds.aws.s3.domain.S3Bucket;
import java.util.Map; import com.google.common.collect.ImmutableMap;
import java.util.logging.Logger; import com.google.common.collect.ImmutableSet;
/** /**
* // TODO: Adrian: Document this! * Tests to cover @{link LiveS3ObjectMap}
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest") @Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest")
public class S3ObjectMapTest { public class S3ObjectMapTest extends S3IntegrationTest {
private S3Connection connection;
private S3Bucket bucket; private S3Bucket bucket;
private Map map; private S3ObjectMap map;
private Map<String, String> fiveStrings = ImmutableMap.of("one", "apple",
"two", "bear", "three", "candy", "four", "dogma", "five", "emma");
private Map<String, byte[]> fiveBytes = ImmutableMap.of("one", "apple"
.getBytes(), "two", "bear".getBytes(), "three", "candy".getBytes(),
"four", "dogma".getBytes(), "five", "emma".getBytes());
private Map<String, InputStream> fiveInputs;
private Map<String, File> fiveFiles;
String tmpDirectory;
@BeforeMethod @BeforeMethod
public void setupExecutorService() throws Exception { @Parameters( { "basedir" })
connection = new StubS3Connection(); protected void setUpTempDir(String basedir) throws InterruptedException,
ExecutionException, FileNotFoundException, IOException {
tmpDirectory = basedir + File.separator + "target" + File.separator
+ "testFiles" + File.separator + getClass().getSimpleName();
new File(tmpDirectory).mkdirs();
fiveFiles = ImmutableMap.of("one", new File(tmpDirectory, "apple"),
"two", new File(tmpDirectory, "bear"), "three", new File(
tmpDirectory, "candy"), "four", new File(tmpDirectory,
"dogma"), "five", new File(tmpDirectory, "emma"));
for (File file : fiveFiles.values()) {
IOUtils.write(file.getName(), new FileOutputStream(file));
}
fiveInputs = ImmutableMap.of("one", IOUtils.toInputStream("apple"),
"two", IOUtils.toInputStream("bear"), "three", IOUtils
.toInputStream("candy"), "four", IOUtils
.toInputStream("dogma"), "five", IOUtils
.toInputStream("emma"));
bucket = new S3Bucket(); bucket = new S3Bucket();
bucket.setName("mimi"); bucket.setName(bucketPrefix + ".mimi");
connection.createBucketIfNotExists(bucket).get(); client.createBucketIfNotExists(bucket).get();
map = new S3ObjectMap(Logger.getLogger("test"), connection, bucket, new S3Utils()); map = context.createMapView(bucket);
map.clear();
} }
@AfterMethod @AfterMethod
public void teardownExecutorService() { public void tearDown() {
map.clear();
map = null; map = null;
bucket = null; bucket = null;
connection = null;
} }
@Test @Test
public void testClearWhenNothingInMap(){ public void testClear() {
map.clear();
assert map.size() == 0;
map.putString("one", "apple");
map.clear(); map.clear();
assert map.size() == 0; assert map.size() == 0;
} }
// @Test @Test()
// public void testGetReturnsWhatWasPut(){ public void testRemove() throws IOException {
// map.put("hello","goodbye"); map.putString("one", "two");
// assert "goodbye".equals(map.get("hello")); InputStream old = map.remove("one");
// assert map.size() == 0; assertEquals(Utils.toStringAndClose(old), "two");
// } old = map.remove("one");
assert old == null;
old = map.get("one");
assert old == null;
assertEquals(map.keySet().size(), 0);
}
@Test()
public void testKeySet() {
assertEquals(map.keySet().size(), 0);
map.putString("one", "two");
assertEquals(map.keySet(), ImmutableSet.of("one"));
}
@Test()
public void testValues() throws IOException {
map.putAll(this.fiveInputs);
Collection<InputStream> values = map.values();
assertEquals(values.size(), 5);
Set<String> valuesAsString = new HashSet<String>();
for (InputStream stream : values) {
valuesAsString.add(Utils.toStringAndClose(stream));
}
valuesAsString.removeAll(fiveStrings.values());
assert valuesAsString.size() == 0;
}
@Test()
public void testEntrySet() throws IOException {
map.putAllStrings(this.fiveStrings);
Set<Entry<String, InputStream>> entries = map.entrySet();
assertEquals(entries.size(), 5);
for (Entry<String, InputStream> entry : entries) {
assertEquals(IOUtils.toString(entry.getValue()), fiveStrings
.get(entry.getKey()));
entry.setValue(IOUtils.toInputStream(""));
}
assertEquals(map.size(), 5);
for (InputStream value : map.values()) {
assertEquals(IOUtils.toString(value), "");
}
}
@Test()
public void testContainsKey() {
assert !map.containsKey("one");
map.putString("one", "apple");
assert map.containsKey("one");
}
@Test()
public void testContainsStringValue() {
map.putString("one", "apple");
assert map.containsValue(fiveStrings.get("one"));
}
@Test()
public void testContainsFileValue() {
map.putString("one", "apple");
assert map.containsValue(fiveFiles.get("one"));
}
@Test()
public void testContainsInputStreamValue() {
map.putString("one", "apple");
assert map.containsValue(this.fiveInputs.get("one"));
}
@Test()
public void testContainsBytesValue() {
map.putString("one", "apple");
assert map.containsValue(this.fiveBytes.get("one"));
}
@Test()
public void testIsEmpty() {
assert map.isEmpty();
map.putString("one", "apple");
assert !map.isEmpty();
}
@Test()
public void testPutAll() {
map.putAll(this.fiveInputs);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveInputs.keySet()));
fourLeftRemovingOne();
}
@Test()
public void testPutAllBytes() {
map.putAllBytes(this.fiveBytes);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveBytes.keySet()));
fourLeftRemovingOne();
}
@Test @Test
public void testClearWhenSomethingInMap(){ public void testPutAllFiles() {
map.put("hello","goodbye"); map.putAllFiles(this.fiveFiles);
map.clear(); assertEquals(map.size(), 5);
assert map.size() == 0; assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveFiles.keySet()));
fourLeftRemovingOne();
}
void fourLeftRemovingOne() {
map.remove("one");
assertEquals(map.size(), 4);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
ImmutableSet.of("two", "three", "four", "five")));
}
@Test()
public void testPutAllStrings() {
map.putAllStrings(this.fiveStrings);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveStrings.keySet()));
fourLeftRemovingOne();
}
@Test()
public void testPutString() throws IOException {
InputStream old = map.putString("one", "apple");
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putString("one", "bear");
getOneReturnsBearAndOldValueIsApple(apple);
}
private void getOneReturnsAppleAndOldValueIsNull(InputStream old)
throws IOException {
assert old == null;
assertEquals(Utils.toStringAndClose(map.get("one")), "apple");
assert map.size() == 1;
}
private void getOneReturnsBearAndOldValueIsApple(InputStream oldValue)
throws IOException {
assertEquals(Utils.toStringAndClose(map.get("one")), "bear");
assertEquals(Utils.toStringAndClose(oldValue), "apple");
assert map.size() == 1;
}
@Test()
public void testPutFile() throws IOException {
InputStream old = map.putFile("one", fiveFiles.get("one"));
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putFile("one", fiveFiles.get("two"));
getOneReturnsBearAndOldValueIsApple(apple);
}
@Test()
public void testPutBytes() throws IOException {
InputStream old = map.putBytes("one", "apple".getBytes());
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putBytes("one", "bear".getBytes());
getOneReturnsBearAndOldValueIsApple(apple);
}
@Test()
public void testPut() throws IOException {
InputStream old = map.put("one", IOUtils.toInputStream("apple"));
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.put("one", IOUtils.toInputStream("bear"));
getOneReturnsBearAndOldValueIsApple(apple);
}
@Test()
public void testGetBucket() {
assertEquals(map.getBucket().getName(), bucket.getName());
}
@Test
void testPutS3Object() throws IOException {
S3Object object = new S3Object();
object.setKey("one");
object.setContent(IOUtils.toInputStream("apple"));
object.setSize("apple".getBytes().length);
InputStream old = map.put(object);
getOneReturnsAppleAndOldValueIsNull(old);
object.setContent(IOUtils.toInputStream("bear"));
object.setSize("bear".getBytes().length);
InputStream apple = map.put(object);
getOneReturnsBearAndOldValueIsApple(apple);
}
@Test
void testPutAllS3Objects() {
Set<S3Object> set = new HashSet<S3Object>();
for (String key : fiveInputs.keySet()) {
S3Object object = new S3Object();
object.setKey(key);
object.setContent(fiveInputs.get(key));
object.setSize(fiveBytes.get(key).length);
set.add(object);
}
map.putAll(set);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveInputs.keySet()));
fourLeftRemovingOne();
} }
} }

View File

@ -28,6 +28,7 @@ import org.jclouds.aws.PerformanceTest;
import org.testng.annotations.DataProvider; import org.testng.annotations.DataProvider;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException; import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException; import java.security.NoSuchProviderException;
@ -51,7 +52,7 @@ public class S3UtilsTest extends PerformanceTest {
String base64Digest) throws NoSuchProviderException, String base64Digest) throws NoSuchProviderException,
NoSuchAlgorithmException, InvalidKeyException { NoSuchAlgorithmException, InvalidKeyException {
for (int i = 0; i < 10000; i++) for (int i = 0; i < 10000; i++)
testBouncyCastleDigest(key, message, base64Digest); testBouncyCastleHmacSha1Base64(key, message, base64Digest);
} }
@Test(dataProvider = "hmacsha1") @Test(dataProvider = "hmacsha1")
@ -64,7 +65,7 @@ public class S3UtilsTest extends PerformanceTest {
for (int i = 0; i < 10000; i++) for (int i = 0; i < 10000; i++)
completer.submit(new Callable<Boolean>() { completer.submit(new Callable<Boolean>() {
public Boolean call() throws Exception { public Boolean call() throws Exception {
testBouncyCastleDigest(key, message, base64Digest); testBouncyCastleHmacSha1Base64(key, message, base64Digest);
return true; return true;
} }
}); });
@ -72,6 +73,17 @@ public class S3UtilsTest extends PerformanceTest {
assert completer.take().get(); assert completer.take().get();
} }
@DataProvider(name = "md5")
public Object[][] createMD5Data() {
return base64MD5MessageDigest;
}
public final static Object[][] base64MD5MessageDigest = {
{ "apple", "1f3870be274f6c49b3e31a0c6728957f" },
{ "bear", "893b56e3cfe153fb770a120b83bac20c" },
{ "candy", "c48ba993d35c3abe0380f91738fe2a34" },
{ "dogma", "95eb470e4faee302e9cd3063b1923dab" },
{ "emma", "00a809937eddc44521da9521269e75c6" } };
public final static Object[][] base64KeyMessageDigest = { public final static Object[][] base64KeyMessageDigest = {
{ Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There", { Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There",
"thcxhlUFcmTii8C2+zeMjvFGvgA=" }, "thcxhlUFcmTii8C2+zeMjvFGvgA=" },
@ -96,11 +108,19 @@ public class S3UtilsTest extends PerformanceTest {
} }
@Test(dataProvider = "hmacsha1") @Test(dataProvider = "hmacsha1")
public void testBouncyCastleDigest(byte[] key, String message, public void testBouncyCastleHmacSha1Base64(byte[] key, String message,
String base64Digest) throws NoSuchProviderException, String base64Digest) throws NoSuchProviderException,
NoSuchAlgorithmException, InvalidKeyException { NoSuchAlgorithmException, InvalidKeyException {
String b64 = S3Utils.digest(message, key); String b64 = S3Utils.hmacSha1Base64(message, key);
assertEquals(b64, base64Digest); assertEquals(b64, base64Digest);
} }
@Test(dataProvider = "md5")
public void testBouncyCastleMD5Digest(String message,
String base64Digest) throws NoSuchProviderException,
NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException {
String b64 = S3Utils.md5Hex(message.getBytes());
assertEquals(base64Digest,b64);
}
} }

View File

@ -23,12 +23,19 @@
*/ */
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import org.jclouds.aws.s3.domain.S3Object; import java.util.ArrayList;
import org.jclouds.aws.s3.domain.S3Bucket; import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.*; import org.jclouds.aws.s3.domain.S3Bucket;
import java.util.concurrent.*; import org.jclouds.aws.s3.domain.S3Object;
import java.io.IOException;
/** /**
* // TODO: Adrian: Document this! * // TODO: Adrian: Document this!
@ -38,13 +45,14 @@ import java.io.IOException;
public class StubS3Connection implements S3Connection { public class StubS3Connection implements S3Connection {
private static Map<S3Bucket, Map<String, Object>> bucketToContents = new ConcurrentHashMap<S3Bucket, Map<String, Object>>(); private static Map<S3Bucket, Map<String, Object>> bucketToContents = new ConcurrentHashMap<S3Bucket, Map<String, Object>>();
public Future<S3Object> getObject(final S3Bucket s3Bucket, final String key) { public Future<S3Object> getObject(final S3Bucket s3Bucket, final String key) {
return new FutureBase<S3Object>() { return new FutureBase<S3Object>() {
public S3Object get() throws InterruptedException, ExecutionException { public S3Object get() throws InterruptedException,
ExecutionException {
if (!bucketToContents.containsKey(s3Bucket)) if (!bucketToContents.containsKey(s3Bucket))
return S3Object.NOT_FOUND; return S3Object.NOT_FOUND;
Map<String, Object> realContents = bucketToContents.get(s3Bucket); Map<String, Object> realContents = bucketToContents
.get(s3Bucket);
if (!realContents.containsKey(key)) if (!realContents.containsKey(key))
return S3Object.NOT_FOUND; return S3Object.NOT_FOUND;
S3Object object = new S3Object(); S3Object object = new S3Object();
@ -59,9 +67,11 @@ public class StubS3Connection implements S3Connection {
return getObject(s3Bucket, key); return getObject(s3Bucket, key);
} }
public Future<Boolean> deleteObject(final S3Bucket s3Bucket, final String key) { public Future<Boolean> deleteObject(final S3Bucket s3Bucket,
final String key) {
return new FutureBase<Boolean>() { return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException { public Boolean get() throws InterruptedException,
ExecutionException {
if (bucketToContents.containsKey(s3Bucket)) { if (bucketToContents.containsKey(s3Bucket)) {
bucketToContents.get(s3Bucket).remove(key); bucketToContents.get(s3Bucket).remove(key);
} }
@ -70,13 +80,16 @@ public class StubS3Connection implements S3Connection {
}; };
} }
public Future<String> addObject(final S3Bucket s3Bucket, final S3Object object) { public Future<String> addObject(final S3Bucket s3Bucket,
final S3Object object) {
return new FutureBase<String>() { return new FutureBase<String>() {
public String get() throws InterruptedException, ExecutionException { public String get() throws InterruptedException, ExecutionException {
if (!bucketToContents.containsKey(s3Bucket)) { if (!bucketToContents.containsKey(s3Bucket)) {
throw new ExecutionException(new RuntimeException("bucket not found: " + s3Bucket.getName())); throw new ExecutionException(new RuntimeException(
"bucket not found: " + s3Bucket.getName()));
} }
bucketToContents.get(s3Bucket).put(object.getKey(), object.getClass()); bucketToContents.get(s3Bucket).put(object.getKey(),
object.getClass());
return object.getKey(); return object.getKey();
} }
}; };
@ -84,9 +97,11 @@ public class StubS3Connection implements S3Connection {
public Future<Boolean> createBucketIfNotExists(final S3Bucket s3Bucket) { public Future<Boolean> createBucketIfNotExists(final S3Bucket s3Bucket) {
return new FutureBase<Boolean>() { return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException { public Boolean get() throws InterruptedException,
ExecutionException {
if (!bucketToContents.containsKey(s3Bucket)) { if (!bucketToContents.containsKey(s3Bucket)) {
bucketToContents.put(s3Bucket, new ConcurrentHashMap<String, Object>()); bucketToContents.put(s3Bucket,
new ConcurrentHashMap<String, Object>());
} }
return bucketToContents.containsKey(s3Bucket); return bucketToContents.containsKey(s3Bucket);
} }
@ -95,7 +110,8 @@ public class StubS3Connection implements S3Connection {
public Future<Boolean> deleteBucket(final S3Bucket s3Bucket) { public Future<Boolean> deleteBucket(final S3Bucket s3Bucket) {
return new FutureBase<Boolean>() { return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException { public Boolean get() throws InterruptedException,
ExecutionException {
if (bucketToContents.containsKey(s3Bucket)) { if (bucketToContents.containsKey(s3Bucket)) {
if (bucketToContents.get(s3Bucket).size() == 0) if (bucketToContents.get(s3Bucket).size() == 0)
return true; return true;
@ -105,13 +121,18 @@ public class StubS3Connection implements S3Connection {
}; };
} }
public Future<Boolean> copyObject(final S3Bucket sourceBucket, final S3Object sourceObject, final S3Bucket destinationBucket, final S3Object destinationObject) { public Future<Boolean> copyObject(final S3Bucket sourceBucket,
final S3Object sourceObject, final S3Bucket destinationBucket,
final S3Object destinationObject) {
return new FutureBase<Boolean>() { return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException { public Boolean get() throws InterruptedException,
ExecutionException {
Map<String, Object> source = bucketToContents.get(sourceBucket); Map<String, Object> source = bucketToContents.get(sourceBucket);
Map<String, Object> dest = bucketToContents.get(destinationBucket); Map<String, Object> dest = bucketToContents
.get(destinationBucket);
if (source.containsKey(sourceObject.getKey())) { if (source.containsKey(sourceObject.getKey())) {
dest.put(destinationObject.getKey(), source.get(sourceObject.getKey())); dest.put(destinationObject.getKey(), source
.get(sourceObject.getKey()));
return true; return true;
} }
return false; return false;
@ -121,7 +142,8 @@ public class StubS3Connection implements S3Connection {
public Future<Boolean> bucketExists(final S3Bucket s3Bucket) { public Future<Boolean> bucketExists(final S3Bucket s3Bucket) {
return new FutureBase<Boolean>() { return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException { public Boolean get() throws InterruptedException,
ExecutionException {
return bucketToContents.containsKey(s3Bucket); return bucketToContents.containsKey(s3Bucket);
} }
}; };
@ -129,9 +151,11 @@ public class StubS3Connection implements S3Connection {
public Future<S3Bucket> getBucket(final S3Bucket s3Bucket) { public Future<S3Bucket> getBucket(final S3Bucket s3Bucket) {
return new FutureBase<S3Bucket>() { return new FutureBase<S3Bucket>() {
public S3Bucket get() throws InterruptedException, ExecutionException { public S3Bucket get() throws InterruptedException,
ExecutionException {
Set<S3Object> contents = new HashSet<S3Object>(); Set<S3Object> contents = new HashSet<S3Object>();
Map<String, Object> realContents = bucketToContents.get(s3Bucket); Map<String, Object> realContents = bucketToContents
.get(s3Bucket);
if (realContents != null) { if (realContents != null) {
for (String key : realContents.keySet()) { for (String key : realContents.keySet()) {
S3Object object = new S3Object(); S3Object object = new S3Object();
@ -150,16 +174,13 @@ public class StubS3Connection implements S3Connection {
public Future<List<S3Bucket>> getBuckets() { public Future<List<S3Bucket>> getBuckets() {
return new FutureBase<List<S3Bucket>>() { return new FutureBase<List<S3Bucket>>() {
public List<S3Bucket> get() throws InterruptedException, ExecutionException { public List<S3Bucket> get() throws InterruptedException,
ExecutionException {
return new ArrayList<S3Bucket>(bucketToContents.keySet()); return new ArrayList<S3Bucket>(bucketToContents.keySet());
} }
}; };
} }
public void close() throws IOException {
// nothing to close
}
private abstract class FutureBase<V> implements Future<V> { private abstract class FutureBase<V> implements Future<V> {
public boolean cancel(boolean b) { public boolean cancel(boolean b) {
return false; return false;
@ -173,7 +194,8 @@ public class StubS3Connection implements S3Connection {
return true; return true;
} }
public V get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { public V get(long l, TimeUnit timeUnit) throws InterruptedException,
ExecutionException, TimeoutException {
return get(); return get();
} }
} }

View File

@ -23,6 +23,8 @@
*/ */
package org.jclouds.aws.s3.commands; package org.jclouds.aws.s3.commands;
import static org.testng.Assert.assertEquals;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
@ -148,7 +150,7 @@ public class S3ParserTest extends PerformanceTest {
assert object.getLastModified().equals(expected) : String assert object.getLastModified().equals(expected) : String
.format("expected %1s, but got %1s", expected, object .format("expected %1s, but got %1s", expected, object
.getLastModified()); .getLastModified());
assert object.getETag().equals("\"9d7bb64e8e18ee34eec06dd2cf37b766\""); assertEquals(object.getETag(), "9d7bb64e8e18ee34eec06dd2cf37b766");
assert object.getSize() == 136; assert object.getSize() == 136;
S3Owner owner = new S3Owner(); S3Owner owner = new S3Owner();
owner owner

View File

@ -27,6 +27,7 @@ import com.google.inject.AbstractModule;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.name.Names; import com.google.inject.name.Names;
import org.jclouds.aws.s3.DateService; import org.jclouds.aws.s3.DateService;
import org.jclouds.aws.s3.S3Constants;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -40,8 +41,8 @@ public class RequestAuthorizeSignatureTest {
filter = Guice.createInjector(new AbstractModule() { filter = Guice.createInjector(new AbstractModule() {
protected void configure() { protected void configure() {
bindConstant().annotatedWith(Names.named("jclouds.aws.accesskeyid")).to("foo"); bindConstant().annotatedWith(Names.named(S3Constants.PROPERTY_AWS_ACCESSKEYID)).to("foo");
bindConstant().annotatedWith(Names.named("jclouds.aws.secretaccesskey")).to("bar"); bindConstant().annotatedWith(Names.named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY)).to("bar");
bind(DateService.class); bind(DateService.class);
} }

View File

@ -31,6 +31,7 @@ import java.net.URL;
import java.util.Properties; import java.util.Properties;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3Constants;
import org.testng.annotations.BeforeTest; import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional; import org.testng.annotations.Optional;
import org.testng.annotations.Parameters; import org.testng.annotations.Parameters;
@ -47,13 +48,14 @@ import org.testng.annotations.Test;
public class GoogleAppEngineTest extends BaseGoogleAppEngineTest { public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
private static final String sysAWSAccessKeyId = System private static final String sysAWSAccessKeyId = System
.getProperty("jclouds.aws.accesskeyid"); .getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
private static final String sysAWSSecretAccessKey = System private static final String sysAWSSecretAccessKey = System
.getProperty("jclouds.aws.secretaccesskey"); .getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
@BeforeTest @BeforeTest
@Parameters( { "warfile", "devappserver.address", "devappserver.port", @Parameters( { "warfile", "devappserver.address", "devappserver.port",
"jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" }) S3Constants.PROPERTY_AWS_ACCESSKEYID,
S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
public void startDevAppServer(final String warfile, final String address, public void startDevAppServer(final String warfile, final String address,
final String port, @Optional String AWSAccessKeyId, final String port, @Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey) throws Exception { @Optional String AWSSecretAccessKey) throws Exception {
@ -66,8 +68,8 @@ public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey"); checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey");
Properties props = new Properties(); Properties props = new Properties();
props.put("jclouds.aws.accesskeyid", AWSAccessKeyId); props.put(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId);
props.put("jclouds.aws.secretaccesskey", AWSSecretAccessKey); props.put(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey);
writePropertiesAndStartServer(address, port, warfile, props); writePropertiesAndStartServer(address, port, warfile, props);
} }

View File

@ -31,8 +31,8 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextEvent;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3ConnectionFactory; import org.jclouds.aws.s3.S3ContextFactory;
import org.jclouds.aws.s3.S3ConnectionModule; import org.jclouds.aws.s3.config.S3ContextModule;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import org.jclouds.lifecycle.Closer; import org.jclouds.lifecycle.Closer;
import org.jclouds.samples.googleappengine.JCloudsServlet; import org.jclouds.samples.googleappengine.JCloudsServlet;
@ -84,11 +84,11 @@ public class GuiceServletConfig extends GuiceServletContextListener {
} finally { } finally {
IOUtils.closeQuietly(input); IOUtils.closeQuietly(input);
} }
props.putAll(S3ConnectionFactory.DEFAULT_PROPERTIES); props.putAll(S3ContextFactory.DEFAULT_PROPERTIES);
Names.bindProperties(binder(), props); Names.bindProperties(binder(), props);
} }
}, new JavaUrlHttpFutureCommandClientModule(), }, new JavaUrlHttpFutureCommandClientModule(),
new S3ConnectionModule(), new ServletModule() { new S3ContextModule(), new ServletModule() {
@Override @Override
protected void configureServlets() { protected void configureServlets() {
serve("*.s3").with(JCloudsServlet.class); serve("*.s3").with(JCloudsServlet.class);