Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

removed google from default module list 

git-svn-id: http://jclouds.googlecode.com/svn/trunk@72 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-04-30 13:28:24 +00:00
parent 9dfac47f05
commit ca8f278580
46 changed files with 1924 additions and 753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/main/java/org/jclouds/command/FutureCommandClient.java
View File

@ -31,5 +31,4 @@ package org.jclouds.command;
public interface FutureCommandClient {
@SuppressWarnings("unchecked")
<O extends FutureCommand> void submit(O operation);
void close();
}

60
core/src/main/java/org/jclouds/command/pool/FutureCommandConnectionPoolClient.java
View File

@ -23,46 +23,50 @@
*/
package org.jclouds.command.pool;
import com.google.inject.Inject;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.Logger;
import org.jclouds.Utils;
import org.jclouds.command.FutureCommand;
import org.jclouds.command.FutureCommandClient;
import org.jclouds.lifecycle.BaseLifeCycle;
import org.jclouds.lifecycle.Closer;
import java.io.IOException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.inject.Inject;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implements FutureCommandClient {
@Inject private Closer closer;
public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle
implements FutureCommandClient {
private final FutureCommandConnectionPool<C> futureCommandConnectionPool;
private final BlockingQueue<FutureCommand> commandQueue;
@Inject
public FutureCommandConnectionPoolClient(java.util.logging.Logger logger, ExecutorService executor, FutureCommandConnectionPool<C> futureCommandConnectionPool, BlockingQueue<FutureCommand> commandQueue) {
public FutureCommandConnectionPoolClient(java.util.logging.Logger logger,
ExecutorService executor,
FutureCommandConnectionPool<C> futureCommandConnectionPool,
BlockingQueue<FutureCommand> commandQueue) {
super(new Logger(logger), executor, futureCommandConnectionPool);
this.futureCommandConnectionPool = futureCommandConnectionPool;
this.commandQueue = commandQueue;
}
@Override
protected boolean shouldDoWork() {
return super.shouldDoWork() && futureCommandConnectionPool.getStatus().equals(Status.ACTIVE);
return super.shouldDoWork()
&& futureCommandConnectionPool.getStatus()
.equals(Status.ACTIVE);
}
@Override
protected void doShutdown() {
if (exception == null && futureCommandConnectionPool.getException() != null)
if (exception == null
&& futureCommandConnectionPool.getException() != null)
exception = futureCommandConnectionPool.getException();
while (!commandQueue.isEmpty()) {
FutureCommand command = commandQueue.remove();
@ -75,6 +79,7 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
}
}
@Override
protected void doWork() throws InterruptedException {
FutureCommand command = commandQueue.poll(1, TimeUnit.SECONDS);
if (command != null) {
@ -87,7 +92,6 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
}
}
public <O extends FutureCommand> void submit(O operation) {
exceptionIfNotActive();
commandQueue.add(operation);
@ -99,17 +103,25 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
try {
connectionHandle = futureCommandConnectionPool.getHandle(operation);
} catch (InterruptedException e) {
logger.warn(e, "Interrupted getting a connection for operation %1s; retrying", operation);
logger
.warn(
e,
"Interrupted getting a connection for operation %1s; retrying",
operation);
commandQueue.add(operation);
return;
} catch (TimeoutException e) {
logger.warn(e, "Timeout getting a connection for operation %1s; retrying", operation);
logger.warn(e,
"Timeout getting a connection for operation %1s; retrying",
operation);
commandQueue.add(operation);
return;
}
if (connectionHandle == null) {
logger.error("Failed to obtain connection for operation %1s; retrying", operation);
logger.error(
"Failed to obtain connection for operation %1s; retrying",
operation);
commandQueue.add(operation);
return;
}
@ -121,18 +133,12 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
final StringBuilder sb = new StringBuilder();
sb.append("FutureCommandConnectionPoolClient");
sb.append("{status=").append(status);
sb.append(", commandQueue=").append((commandQueue != null) ? commandQueue.size() : 0);
sb.append(", futureCommandConnectionPool=").append(futureCommandConnectionPool);
sb.append(", commandQueue=").append(
(commandQueue != null) ? commandQueue.size() : 0);
sb.append(", futureCommandConnectionPool=").append(
futureCommandConnectionPool);
sb.append('}');
return sb.toString();
}
public void close(){
try {
closer.close();
} catch (IOException e) {
e.printStackTrace(); // TODO: Adrian: Customise this generated block
}
}
}

37
core/src/main/java/org/jclouds/command/pool/PoolConstants.java Normal file
View File

@ -0,0 +1,37 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.command.pool;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public interface PoolConstants {
public static final String PROPERTY_POOL_MAX_CONNECTIONS = "jclouds.pool.max_connections";
public static final String PROPERTY_POOL_IO_WORKER_THREADS = "jclouds.http.pool.io_worker_threads";
public static final String PROPERTY_POOL_REQUEST_INVOKER_THREADS = "jclouds.http.pool.request_invoker_threads";
public static final String PROPERTY_POOL_MAX_SESSION_FAILURES = "jclouds.http.pool.max_session_failures";
public static final String PROPERTY_POOL_MAX_CONNECTION_REUSE = "jclouds.http.pool.max_connection_reuse";
}

6
core/src/main/java/org/jclouds/command/pool/config/FutureCommandConnectionPoolClientModule.java
View File

@ -27,6 +27,7 @@ import com.google.inject.*;
import com.google.inject.name.Named;
import org.jclouds.command.FutureCommand;
import org.jclouds.lifecycle.config.LifeCycleModule;
import org.jclouds.command.pool.PoolConstants;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@ -50,8 +51,7 @@ public abstract class FutureCommandConnectionPoolClientModule<C> extends Abstrac
@Provides
@Singleton
public abstract BlockingQueue<C> provideAvailablePool(@Named("jclouds.pool.max_connections") int max) throws Exception;
public abstract BlockingQueue<C> provideAvailablePool(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception;
/**
* controls production and destruction of real connections.
* <p/>
@ -64,7 +64,7 @@ public abstract class FutureCommandConnectionPoolClientModule<C> extends Abstrac
*/
@Provides
@Singleton
public Semaphore provideTotalConnectionSemaphore(@Named("jclouds.pool.max_connections") int max) throws Exception {
public Semaphore provideTotalConnectionSemaphore(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception {
return new Semaphore(max, true);
}
}

5
core/src/main/java/org/jclouds/http/HttpConstants.java
View File

@ -28,7 +28,7 @@ package org.jclouds.http;
*
* @author Adrian Cole
*/
public class HttpConstants {
public interface HttpConstants {
public static final String CONTENT_LENGTH = "Content-Length";
public static final String CONTENT_TYPE = "Content-Type";
public static final String HOST = "Host";
@ -36,4 +36,7 @@ public class HttpConstants {
public static final String BINARY = "application/octet-stream";
public static final String PLAIN = "text/plain";
public static final String TRANSFER_ENCODING = "Transfer-Encoding";
public static final String PROPERTY_HTTP_SECURE = "jclouds.http.secure";
public static final String PROPERTY_HTTP_PORT = "jclouds.http.port";
public static final String PROPERTY_HTTP_ADDRESS = "jclouds.http.address";
}

7
core/src/main/java/org/jclouds/http/HttpFutureCommandClient.java
View File

@ -23,11 +23,12 @@
*/
package org.jclouds.http;
import com.google.inject.Inject;
import java.util.List;
import org.jclouds.command.FutureCommand;
import org.jclouds.command.FutureCommandClient;
import java.util.List;
import com.google.inject.Inject;
/**
* // TODO: Adrian: Document this!
@ -41,6 +42,4 @@ public interface HttpFutureCommandClient extends FutureCommandClient {
void setRequestFilters(List<HttpRequestFilter> requestFilters);
<O extends FutureCommand> void submit(O operation);
void close();
}

34
core/src/main/java/org/jclouds/http/JavaUrlHttpFutureCommandClient.java
View File

@ -25,6 +25,7 @@ package org.jclouds.http;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -82,35 +83,40 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
HttpResponse response = getResponse(connection);
logger.trace("%1s - received response %2s", target, response);
operation.getResponseFuture().setResponse(response);
operation.getResponseFuture().run();
} catch (FileNotFoundException e) {
HttpResponse response = new HttpResponse();
response.setStatusCode(404);
operation.getResponseFuture().setResponse(response);
operation.getResponseFuture().run();
} catch (Exception e) {
if (connection != null) {
InputStream errorStream = connection.getErrorStream();
if (errorStream != null) {
StringBuilder errors = new StringBuilder();
try {
String errorMessage = Utils.toStringAndClose(connection
.getErrorStream());
for (InputStream in : new InputStream[] {
connection.getErrorStream(),
connection.getInputStream() }) {
if (in != null) {
errors.append(Utils.toStringAndClose(in)).append(
"\n");
}
}
logger.error(e,
"error encountered during the exception: %1s",
errorMessage);
} catch (IOException e1) {
errors.toString());
} catch (IOException e2) {
}
}
}
operation.setException(e);
} finally {
// DO NOT disconnect, as it will also close the unconsumed
// outputStream from above.
// connection.disconnect();
if (request.getMethod().equals("HEAD"))
connection.disconnect();
}
}
public void close() {
// Nothing to stop;
}
private HttpResponse getResponse(HttpURLConnection connection)
throws IOException {
HttpResponse response = new HttpResponse();
@ -121,9 +127,11 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
}
response.setMessage(connection.getResponseMessage());
if (!connection.getRequestMethod().equals("HEAD")) {
response.setContent(connection.getInputStream());
response.setContentType(connection
.getHeaderField(HttpConstants.CONTENT_TYPE));
}
return response;
}

29
core/src/main/java/org/jclouds/http/config/JavaUrlHttpFutureCommandClientModule.java
View File

@ -23,19 +23,20 @@
*/
package org.jclouds.http.config;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.JavaUrlHttpFutureCommandClient;
import java.net.MalformedURLException;
import java.net.URL;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.JavaUrlHttpFutureCommandClient;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
/**
* // TODO: Adrian: Document this!
* Configures {@link JavaUrlHttpFutureCommandClient}.
*
* @author Adrian Cole
*/
@ -44,15 +45,19 @@ public class JavaUrlHttpFutureCommandClientModule extends AbstractModule {
@Override
protected void configure() {
// note this is not threadsafe, so it cannot be singleton
bind(HttpFutureCommandClient.class).to(JavaUrlHttpFutureCommandClient.class);
bind(HttpFutureCommandClient.class).to(
JavaUrlHttpFutureCommandClient.class);
}
@Singleton
@Provides
protected URL provideAddress(@Named("jclouds.http.address") String address, @Named("jclouds.http.port") int port, @Named("jclouds.http.secure") boolean isSecure) throws MalformedURLException {
protected URL provideAddress(
@Named(HttpConstants.PROPERTY_HTTP_ADDRESS) String address,
@Named(HttpConstants.PROPERTY_HTTP_PORT) int port,
@Named(HttpConstants.PROPERTY_HTTP_SECURE) boolean isSecure)
throws MalformedURLException {
return new URL(isSecure ? "https" : "http", address, port, "/");
}
}

11
core/src/test/java/org/jclouds/http/BaseHttpFutureCommandClientTest.java
View File

@ -40,6 +40,7 @@ import org.jclouds.http.commands.GetString;
import org.jclouds.http.commands.Head;
import org.jclouds.http.commands.callables.xml.ParseSax;
import org.jclouds.http.commands.config.HttpCommandsModule;
import org.jclouds.lifecycle.Closer;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Request;
import org.mortbay.jetty.Server;
@ -69,6 +70,7 @@ public abstract class BaseHttpFutureCommandClientTest {
protected CommandFactory factory;
protected HttpFutureCommandClient client;
protected Injector injector;
private Closer closer;
@BeforeClass
@Parameters( { "test-jetty-port" })
@ -95,9 +97,9 @@ public abstract class BaseHttpFutureCommandClientTest {
server.setHandler(handler);
server.start();
final Properties properties = new Properties();
properties.put("jclouds.http.address", "localhost");
properties.put("jclouds.http.port", testPort + "");
properties.put("jclouds.http.secure", "false");
properties.put(HttpConstants.PROPERTY_HTTP_ADDRESS, "localhost");
properties.put(HttpConstants.PROPERTY_HTTP_PORT, testPort + "");
properties.put(HttpConstants.PROPERTY_HTTP_SECURE, "false");
addConnectionProperties(properties);
final List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>(
1);
@ -123,6 +125,7 @@ public abstract class BaseHttpFutureCommandClientTest {
});
factory = injector.getInstance(CommandFactory.class);
client = injector.getInstance(HttpFutureCommandClient.class);
closer = injector.getInstance(Closer.class);
assert client != null;
}
@ -132,7 +135,7 @@ public abstract class BaseHttpFutureCommandClientTest {
@AfterClass
public void tearDownJetty() throws Exception {
client.close();
closer.close();
server.stop();
}

2
extensions/httpnio/src/main/java/org/jclouds/http/httpnio/HttpNioUtils.java
View File

@ -61,7 +61,7 @@ public class HttpNioUtils {
String contentType, long length) {
if (content instanceof InputStream) {
InputStream inputStream = (InputStream) content;
if (length <= 0)
if (length == -1)
throw new IllegalArgumentException(
"you must specify size when content is an InputStream");
InputStreamEntity entity = new InputStreamEntity(inputStream,

29
extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/HttpNioConnectionPoolClientModule.java
View File

@ -23,48 +23,51 @@
*/
package org.jclouds.http.httpnio.config;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
import java.net.InetSocketAddress;
import org.apache.http.nio.NHttpConnection;
import org.jclouds.command.pool.FutureCommandConnectionRetry;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.httpnio.config.internal.NonSSLHttpNioConnectionPoolClientModule;
import org.jclouds.http.httpnio.config.internal.SSLHttpNioConnectionPoolClientModule;
import org.jclouds.http.httpnio.pool.HttpNioConnectionPoolClient;
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionRetry;
import java.net.InetSocketAddress;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
/**
* // TODO: Adrian: Document this!
* Configures {@link HttpNioConnectionPoolClient}
*
* @author Adrian Cole
*/
public class HttpNioConnectionPoolClientModule extends AbstractModule {
@Named("jclouds.http.secure")
@Named(HttpConstants.PROPERTY_HTTP_SECURE)
boolean isSecure;
@Override
protected void configure() {
requestInjection(this);
//TODO test...
if (isSecure)
install(new SSLHttpNioConnectionPoolClientModule());
else
install(new NonSSLHttpNioConnectionPoolClientModule());
bind(new TypeLiteral<FutureCommandConnectionRetry<NHttpConnection>>(){}).to(HttpNioFutureCommandConnectionRetry.class);
bind(HttpFutureCommandClient.class).to(HttpNioConnectionPoolClient.class);
bind(new TypeLiteral<FutureCommandConnectionRetry<NHttpConnection>>() {
}).to(HttpNioFutureCommandConnectionRetry.class);
bind(HttpFutureCommandClient.class).to(
HttpNioConnectionPoolClient.class);
}
@Singleton
@Provides
protected InetSocketAddress provideAddress(@Named("jclouds.http.address") String address, @Named("jclouds.http.port") int port) {
protected InetSocketAddress provideAddress(
@Named(HttpConstants.PROPERTY_HTTP_ADDRESS) String address,
@Named(HttpConstants.PROPERTY_HTTP_PORT) int port) {
return new InetSocketAddress(address, port);
}

5
extensions/httpnio/src/main/java/org/jclouds/http/httpnio/config/internal/BaseHttpNioConnectionPoolClientModule.java
View File

@ -45,6 +45,7 @@ import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.*;
import org.jclouds.command.pool.FutureCommandConnectionRetry;
import org.jclouds.command.pool.PoolConstants;
import org.jclouds.command.pool.config.FutureCommandConnectionPoolClientModule;
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionHandle;
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionPool;
@ -114,7 +115,7 @@ public abstract class BaseHttpNioConnectionPoolClientModule extends FutureComman
}
@Override
public BlockingQueue<NHttpConnection> provideAvailablePool(@Named("jclouds.pool.max_connections") int max) throws Exception {
public BlockingQueue<NHttpConnection> provideAvailablePool(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception {
return new ArrayBlockingQueue<NHttpConnection>(max, true);
}
@ -125,7 +126,7 @@ public abstract class BaseHttpNioConnectionPoolClientModule extends FutureComman
@Provides
@Singleton
public DefaultConnectingIOReactor provideDefaultConnectingIOReactor(@Named("jclouds.http.pool.io_worker_threads") int ioWorkerThreads, HttpParams params) throws IOReactorException {
public DefaultConnectingIOReactor provideDefaultConnectingIOReactor(@Named(PoolConstants.PROPERTY_POOL_IO_WORKER_THREADS) int ioWorkerThreads, HttpParams params) throws IOReactorException {
return new DefaultConnectingIOReactor(ioWorkerThreads, params);
}

95
extensions/httpnio/src/main/java/org/jclouds/http/httpnio/pool/HttpNioFutureCommandConnectionPool.java
View File

@ -23,8 +23,13 @@
*/
package org.jclouds.http.httpnio.pool;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.apache.http.HttpException;
import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
import org.apache.http.nio.NHttpConnection;
@ -38,21 +43,18 @@ import org.jclouds.Logger;
import org.jclouds.command.FutureCommand;
import org.jclouds.command.pool.FutureCommandConnectionPool;
import org.jclouds.command.pool.FutureCommandConnectionRetry;
import org.jclouds.command.pool.PoolConstants;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import com.google.inject.Inject;
import com.google.inject.name.Named;
/**
* // TODO: Adrian: Document this!
* Connection Pool for HTTP requests that utilizes Apache HTTPNio
*
* @author Adrian Cole
*/
public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionPool<NHttpConnection> implements EventListener {
public class HttpNioFutureCommandConnectionPool extends
FutureCommandConnectionPool<NHttpConnection> implements EventListener {
private final NHttpClientConnectionPoolSessionRequestCallback sessionCallback;
private final DefaultConnectingIOReactor ioReactor;
@ -61,8 +63,22 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
private final int maxSessionFailures;
@Inject
public HttpNioFutureCommandConnectionPool(java.util.logging.Logger logger, ExecutorService executor, Semaphore allConnections, BlockingQueue<NHttpConnection> available, AsyncNHttpClientHandler clientHandler, DefaultConnectingIOReactor ioReactor, IOEventDispatch dispatch, FutureCommandConnectionHandleFactory requestHandleFactory, InetSocketAddress target, FutureCommandConnectionRetry<NHttpConnection> futureCommandConnectionRetry, @Named("jclouds.http.pool.max_connection_reuse") int maxConnectionReuse, @Named("jclouds.http.pool.max_session_failures") int maxSessionFailures) {
super(new Logger(logger), executor, futureCommandConnectionRetry, allConnections, requestHandleFactory, maxConnectionReuse, available);
public HttpNioFutureCommandConnectionPool(
java.util.logging.Logger logger,
ExecutorService executor,
Semaphore allConnections,
BlockingQueue<NHttpConnection> available,
AsyncNHttpClientHandler clientHandler,
DefaultConnectingIOReactor ioReactor,
IOEventDispatch dispatch,
FutureCommandConnectionHandleFactory requestHandleFactory,
InetSocketAddress target,
FutureCommandConnectionRetry<NHttpConnection> futureCommandConnectionRetry,
@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTION_REUSE) int maxConnectionReuse,
@Named(PoolConstants.PROPERTY_POOL_MAX_SESSION_FAILURES) int maxSessionFailures) {
super(new Logger(logger), executor, futureCommandConnectionRetry,
allConnections, requestHandleFactory, maxConnectionReuse,
available);
this.ioReactor = ioReactor;
this.dispatch = dispatch;
this.target = target;
@ -99,9 +115,9 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
}
}
public boolean connectionValid(NHttpConnection conn) {
return conn.isOpen() && !conn.isStale() && conn.getMetrics().getRequestCount() < maxConnectionReuse;
return conn.isOpen() && !conn.isStale()
&& conn.getMetrics().getRequestCount() < maxConnectionReuse;
}
protected void doWork() throws Exception {
@ -128,31 +144,38 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
@Override
protected boolean shouldDoWork() {
return super.shouldDoWork() && ioReactor.getStatus().equals(IOReactorStatus.ACTIVE);
return super.shouldDoWork()
&& ioReactor.getStatus().equals(IOReactorStatus.ACTIVE);
}
class NHttpClientConnectionPoolSessionRequestCallback implements SessionRequestCallback {
class NHttpClientConnectionPoolSessionRequestCallback implements
SessionRequestCallback {
public void completed(SessionRequest request) {
logger.trace("%1s - %2s - operation complete", request, request.getAttachment());
logger.trace("%1s - %2s - operation complete", request, request
.getAttachment());
}
public void cancelled(SessionRequest request) {
logger.info("%1s - %2s - operation cancelled", request, request.getAttachment());
logger.info("%1s - %2s - operation cancelled", request, request
.getAttachment());
releaseConnectionAndCancelResponse(request);
}
private void releaseConnectionAndCancelResponse(SessionRequest request) {
allConnections.release();
FutureCommand frequest = (FutureCommand) request.getAttachment();
FutureCommand<?, ?, ?> frequest = (FutureCommand<?, ?, ?>) request
.getAttachment();
if (frequest != null) {
frequest.cancel(true);
}
}
private void releaseConnectionAndSetResponseException(SessionRequest request, Exception e) {
private void releaseConnectionAndSetResponseException(
SessionRequest request, Exception e) {
allConnections.release();
FutureCommand frequest = (FutureCommand) request.getAttachment();
FutureCommand<?, ?, ?> frequest = (FutureCommand<?, ?, ?>) request
.getAttachment();
if (frequest != null) {
frequest.setException(e);
}
@ -160,8 +183,11 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
public void failed(SessionRequest request) {
int count = currentSessionFailures.getAndIncrement();
logger.error(request.getException(), "%1s - %2s - operation failed", request, request.getAttachment());
releaseConnectionAndSetResponseException(request, request.getException());
logger.error(request.getException(),
"%1s - %2s - operation failed", request, request
.getAttachment());
releaseConnectionAndSetResponseException(request, request
.getException());
if (count >= maxSessionFailures) {
exception = request.getException();
}
@ -169,22 +195,22 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
}
public void timeout(SessionRequest request) {
logger.warn("%1s - %2s - operation timed out", request, request.getAttachment());
logger.warn("%1s - %2s - operation timed out", request, request
.getAttachment());
releaseConnectionAndCancelResponse(request);
}
}
public void connectionOpen(NHttpConnection conn) {
conn.setSocketTimeout(0);
available.offer(conn);
logger.trace("%1s - %2d - open", conn, conn.hashCode());
}
public void connectionTimeout(NHttpConnection conn) {
logger.warn("%1s - %2d - timeout %2d", conn, conn.hashCode(), conn.getSocketTimeout());
logger.warn("%1s - %2d - timeout %2d", conn, conn.hashCode(), conn
.getSocketTimeout());
allConnections.release();
futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn);
}
@ -196,18 +222,23 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
public void fatalIOException(IOException ex, NHttpConnection conn) {
exception = ex;
logger.error(ex, "%1s - %2d - %3s - pool error", conn, conn.hashCode(), target);
logger.error(ex, "%1s - %2d - %3s - pool error", conn, conn.hashCode(),
target);
futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn);
}
public void fatalProtocolException(HttpException ex, NHttpConnection conn) {
exception = ex;
logger.error(ex, "%1s - %2d - %3s - http error", conn, conn.hashCode(), target);
logger.error(ex, "%1s - %2d - %3s - http error", conn, conn.hashCode(),
target);
fatalException(ex, conn);
}
public static interface FutureCommandConnectionHandleFactory extends FutureCommandConnectionPool.FutureCommandConnectionHandleFactory<NHttpConnection> {
HttpNioFutureCommandConnectionHandle create(FutureCommand command, NHttpConnection conn);
public static interface FutureCommandConnectionHandleFactory
extends
FutureCommandConnectionPool.FutureCommandConnectionHandleFactory<NHttpConnection> {
HttpNioFutureCommandConnectionHandle create(FutureCommand command,
NHttpConnection conn);
}
}

20
extensions/httpnio/src/test/java/org/jclouds/http/httpnio/pool/HttpNioConnectionPoolFutureCommandClientTest.java
View File

@ -24,6 +24,8 @@
package org.jclouds.http.httpnio.pool;
import com.google.inject.Module;
import org.jclouds.command.pool.PoolConstants;
import org.jclouds.http.BaseHttpFutureCommandClientTest;
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test;
@ -36,14 +38,20 @@ import java.util.Properties;
* @author Adrian Cole
*/
@Test
public class HttpNioConnectionPoolFutureCommandClientTest extends BaseHttpFutureCommandClientTest {
public class HttpNioConnectionPoolFutureCommandClientTest extends
BaseHttpFutureCommandClientTest {
protected void addConnectionProperties(Properties properties) {
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
properties.setProperty("jclouds.http.pool.max_session_failures", "2");
properties.setProperty("jclouds.http.pool.request_invoker_threads", "1");
properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
properties.setProperty("jclouds.pool.max_connections", "12");
properties.setProperty(
PoolConstants.PROPERTY_POOL_MAX_CONNECTION_REUSE, "75");
properties.setProperty(
PoolConstants.PROPERTY_POOL_MAX_SESSION_FAILURES, "2");
properties.setProperty(
PoolConstants.PROPERTY_POOL_REQUEST_INVOKER_THREADS, "1");
properties.setProperty(PoolConstants.PROPERTY_POOL_IO_WORKER_THREADS,
"2");
properties.setProperty(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS,
"12");
}
protected Module createClientModule() {

37
extensions/s3nio/pom.xml
View File

@ -46,11 +46,48 @@
<url>http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</url>
</scm>
<properties>
<jclouds.aws.accesskeyid></jclouds.aws.accesskeyid>
<jclouds.aws.secretaccesskey></jclouds.aws.secretaccesskey>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-httpnio</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemProperties>
<property>
<name>jclouds.aws.accesskeyid</name>
<value>${jclouds.aws.accesskeyid}</value>
</property>
<property>
<name>jclouds.aws.secretaccesskey</name>
<value>${jclouds.aws.secretaccesskey}</value>
</property>
</systemProperties>
</configuration>
</plugin>
</plugins>
</build>
</project>

61
extensions/s3nio/src/test/java/org/jclouds/aws/s3/nio/NioS3ObjectMapTest.java Normal file
View File

@ -0,0 +1,61 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.nio;
import java.util.Properties;
import org.jclouds.aws.s3.S3ObjectMapTest;
import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test;
import com.google.inject.Module;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest")
public class NioS3ObjectMapTest extends S3ObjectMapTest {
@Override
protected Properties buildS3Properties(String AWSAccessKeyId,
String AWSSecretAccessKey) {
Properties properties = super.buildS3Properties(AWSAccessKeyId,
AWSSecretAccessKey);
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
properties.setProperty("jclouds.http.pool.max_session_failures", "2");
properties
.setProperty("jclouds.http.pool.request_invoker_threads", "1");
properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
properties.setProperty("jclouds.pool.max_connections", "12");
return properties;
}
@Override
protected Module createHttpModule() {
return new S3HttpNioConnectionPoolClientModule();
}
}

1
pom.xml
View File

@ -44,7 +44,6 @@
<module>extensions/s3nio</module>
<module>s3</module>
<module>s3/perftest</module>
<module>samples/googleappengine</module>
</modules>
<build>
<plugins>

25
repo/com/google/code/guice/guice-assistedinject/maven-metadata-local.xml
View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<groupId>com.google.code.guice</groupId>

25
repo/com/google/code/guice/guice-multibindings/maven-metadata-local.xml
View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<groupId>com.google.code.guice</groupId>

25
repo/com/google/code/guice/guice-servlet/maven-metadata-local.xml
View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<groupId>com.google.code.guice</groupId>

25
repo/com/google/code/guice/guice/maven-metadata-local.xml
View File

@ -1,4 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
====================================================================
-->
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<groupId>com.google.code.guice</groupId>

3
s3/perftest/src/test/java/com/amazon/s3/AmazonPerformance.java
View File

@ -30,6 +30,7 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import org.jclouds.aws.s3.S3Constants;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
@ -46,7 +47,7 @@ public class AmazonPerformance extends BasePerformance {
@Override
@BeforeTest
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" })
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpClient(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception {
super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey);
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey,

3
s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java
View File

@ -34,6 +34,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import org.jclouds.aws.s3.S3Constants;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.testng.annotations.AfterTest;
@ -79,7 +80,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
@Override
@BeforeTest
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" })
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpClient(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey) throws Exception {
super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey);

5
s3/pom.xml
View File

@ -67,6 +67,11 @@
<artifactId>bcprov-jdk15</artifactId>
<version>140</version>
</dependency>
<dependency>
<groupId>eu.medsea.mimeutil</groupId>
<artifactId>mime-util</artifactId>
<version>1.3</version>
</dependency>
</dependencies>
<build>
<plugins>

12
s3/src/main/java/org/jclouds/aws/s3/S3Connection.java
View File

@ -23,20 +23,18 @@
*/
package org.jclouds.aws.s3;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Bucket;
import java.io.IOException;
import java.io.Closeable;
import java.util.List;
import java.util.concurrent.Future;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public interface S3Connection extends Closeable {
public interface S3Connection {
Future<S3Object> getObject(S3Bucket s3Bucket, String key);
Future<S3Object> headObject(S3Bucket s3Bucket, String key);
@ -56,6 +54,4 @@ public interface S3Connection extends Closeable {
Future<S3Bucket> getBucket(S3Bucket s3Bucket);
Future<List<S3Bucket>> getBuckets();
public void close() throws IOException;
}

116
s3/src/main/java/org/jclouds/aws/s3/S3ConnectionFactory.java
View File

@ -1,116 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.name.Names;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import java.util.Properties;
/**
* // TODO: Adrian: Document return getConnection!
*
* @author Adrian Cole
*/
public class S3ConnectionFactory {
public static final Properties DEFAULT_PROPERTIES;
static {
DEFAULT_PROPERTIES = new Properties();
DEFAULT_PROPERTIES.setProperty("jclouds.http.address", "s3.amazonaws.com");
DEFAULT_PROPERTIES.setProperty("jclouds.http.port", "443");
DEFAULT_PROPERTIES.setProperty("jclouds.http.secure", "true");
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.max_connection_reuse", "75");
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.max_session_failures", "2");
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.request_invoker_threads", "1");
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.io_worker_threads", "2");
DEFAULT_PROPERTIES.setProperty("jclouds.pool.max_connections", "12");
}
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
}
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
if (!isSecure)
properties.setProperty("jclouds.http.port", "80");
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
}
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure,
String server) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
properties.setProperty("jclouds.http.address", server);
if (!isSecure)
properties.setProperty("jclouds.http.port", "80");
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
}
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure,
String server, int port) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
properties.setProperty("jclouds.http.address", server);
properties.setProperty("jclouds.http.port", port + "");
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
}
/**
* Create a new interface to interact with S3 with the given credential and connection
* parameters
*/
public static synchronized S3Connection getConnection(final Properties properties, Module httpModule) {
return getInjector(properties, httpModule).getInstance(S3Connection.class);
}
/**
* Create a new interface to interact with S3 with the given credential and connection
* parameters
*/
public static synchronized Injector getInjector(final Properties properties, Module httpModule) {
return Guice.createInjector(new AbstractModule() {
@Override
protected void configure() {
Names.bindProperties(binder(), properties);
}
}, httpModule, new S3ConnectionModule());
}
}

5
s3/src/main/java/org/jclouds/aws/s3/S3Constants.java
View File

@ -23,6 +23,7 @@
*/
package org.jclouds.aws.s3;
import org.jclouds.command.pool.PoolConstants;
import org.jclouds.http.HttpConstants;
/**
@ -30,6 +31,8 @@ import org.jclouds.http.HttpConstants;
*
* @author Adrian Cole
*/
public class S3Constants extends HttpConstants {
public interface S3Constants extends HttpConstants, PoolConstants {
public static final String AUTH = "Authorization";
public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey";
public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid";
}

49
s3/src/main/java/org/jclouds/aws/s3/S3Context.java Normal file
View File

@ -0,0 +1,49 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import org.jclouds.aws.s3.domain.S3Bucket;
public interface S3Context {
/**
* @return a connection to S3
*/
S3Connection getConnection();
/**
* Creates a <code>Map<String,InputStream</code> view of the specified
* bucket.
*
* @param bucket
* @return
*/
S3ObjectMap createMapView(S3Bucket bucket);
/**
* Closes all connections to S3.
*/
void close();
}

148
s3/src/main/java/org/jclouds/aws/s3/S3ContextFactory.java Normal file
View File

@ -0,0 +1,148 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import org.jclouds.aws.s3.config.S3ContextModule;
import org.jclouds.http.HttpFutureCommandClient;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.name.Names;
import static org.jclouds.aws.s3.S3Constants.*;
/**
* // TODO: Adrian: Document return getConnection!
*
* @author Adrian Cole
*/
public class S3ContextFactory {
public static final Properties DEFAULT_PROPERTIES;
static {
DEFAULT_PROPERTIES = new Properties();
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_ADDRESS,
"s3.amazonaws.com");
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_PORT, "443");
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_SECURE, "true");
DEFAULT_PROPERTIES
.setProperty(PROPERTY_POOL_MAX_CONNECTION_REUSE, "75");
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_MAX_SESSION_FAILURES, "2");
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_REQUEST_INVOKER_THREADS,
"1");
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_IO_WORKER_THREADS, "2");
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_MAX_CONNECTIONS, "12");
}
public static S3Context createS3Context(String awsAccessKeyId,
String awsSecretAccessKey, Module... modules) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
properties
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
return createS3Context(properties, modules);
}
public static S3Context createS3Context(String awsAccessKeyId,
String awsSecretAccessKey, boolean isSecure, Module... modules) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
properties
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
properties
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
if (!isSecure)
properties.setProperty(PROPERTY_HTTP_PORT, "80");
return createS3Context(properties, modules);
}
public static S3Context createS3Context(String awsAccessKeyId,
String awsSecretAccessKey, boolean isSecure, String server,
Module... modules) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
properties
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
properties
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
properties.setProperty(PROPERTY_HTTP_ADDRESS, server);
if (!isSecure)
properties.setProperty(PROPERTY_HTTP_PORT, "80");
return createS3Context(properties, modules);
}
public static S3Context createS3Context(String awsAccessKeyId,
String awsSecretAccessKey, boolean isSecure, String server,
int port, Module... modules) {
Properties properties = new Properties(DEFAULT_PROPERTIES);
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
properties
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
properties
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
properties.setProperty(PROPERTY_HTTP_ADDRESS, server);
properties.setProperty(PROPERTY_HTTP_PORT, port + "");
return createS3Context(properties, modules);
}
public static S3Context createS3Context(Properties properties,
Module... modules) {
return getInjector(properties, modules).getInstance(S3Context.class);
}
/**
* Bind the given properties and install the list of modules. If no modules
* are specified, install the default
* {@link JavaUrlHttpFutureCommandClientModule}
*
* @param properties
* - contains constants used by jclouds
* {@link #DEFAULT_PROPERTIES}
* @param httpModules
* - modules that must bind {@link HttpFutureCommandClient} if
* specified
* */
public static Injector getInjector(final Properties properties,
Module... httpModules) {
final List<? extends Module> modules = httpModules.length != 0 ? Arrays
.asList(httpModules) : Collections
.singletonList(new JavaUrlHttpFutureCommandClientModule());
return Guice.createInjector(new AbstractModule() {
@Override
protected void configure() {
Names.bindProperties(binder(), properties);
for (Module module : modules)
install(module);
}
}, new S3ContextModule());
}
}

138
s3/src/main/java/org/jclouds/aws/s3/S3ObjectMap.java
View File

@ -23,138 +23,32 @@
*/
package org.jclouds.aws.s3;
import com.google.inject.Inject;
import org.jclouds.Logger;
import org.jclouds.aws.s3.domain.S3Object;
import java.io.File;
import java.io.InputStream;
import java.util.Map;
import java.util.Set;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import java.util.*;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
public interface S3ObjectMap extends Map<String, InputStream> {
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public class S3ObjectMap implements ConcurrentMap<String, Object> {
private Logger logger;
private S3Connection connection;
private S3Bucket bucket;
private S3Utils utils;
InputStream putString(String key, String value);
@Inject
public S3ObjectMap(java.util.logging.Logger logger, S3Connection connection, S3Bucket bucket, S3Utils utils) {
this.logger = new Logger(logger);
this.connection = connection;
this.bucket = bucket;
this.utils = utils;
}
InputStream putFile(String key, File value);
InputStream putBytes(String key, byte[] value);
public Object putIfAbsent(String s, Object o) {
return null; // TODO: Adrian: Customise this generated block
}
void putAllStrings(Map<? extends String, ? extends String> map);
public boolean remove(Object o, Object o1) {
return false; // TODO: Adrian: Customise this generated block
}
void putAllBytes(Map<? extends String, ? extends byte[]> map);
public boolean replace(String s, Object o, Object o1) {
return false; // TODO: Adrian: Customise this generated block
}
void putAllFiles(Map<? extends String, ? extends File> map);
public Object replace(String s, Object o) {
return null; // TODO: Adrian: Customise this generated block
}
InputStream put(S3Object object);
public int size() {
try {
bucket = connection.getBucket(bucket).get();
return bucket.getContents().size();
} catch (Exception e) {
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
}
}
void putAll(Set<S3Object> objects);
public boolean isEmpty() {
return false; // TODO: Adrian: Customise this generated block
}
S3Bucket getBucket();
public boolean containsKey(Object o) {
return false; // TODO: Adrian: Customise this generated block
}
public boolean containsValue(Object o) {
return false; // TODO: Adrian: Customise this generated block
}
public Object get(Object o) {
try {
return connection.getObject(bucket, o.toString()).get();
} catch (Exception e) {
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format("Error geting object %1s:%2s", bucket, o), e);
}
}
public Object put(String s, Object o) {
S3Object object = new S3Object();
try {
object.setKey(s);
object.setContent(o);
return connection.addObject(bucket, object).get();
} catch (Exception e) {
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format("Error adding object %1s:%2s", bucket, object), e);
}
}
public Object remove(Object o) {
return null; // TODO: Adrian: Customise this generated block
}
public void putAll(Map<? extends String, ? extends Object> map) {
// TODO: Adrian: Customise this generated block
}
private class S3RuntimeException extends RuntimeException {
public S3RuntimeException(String s) {
super(s); // TODO: Adrian: Customise this generated block
}
public S3RuntimeException(String s, Throwable throwable) {
super(s, throwable); // TODO: Adrian: Customise this generated block
}
}
public void clear() {
try {
bucket = connection.getBucket(bucket).get();
List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
for (S3Object object : bucket.getContents()) {
deletes.add(connection.deleteObject(bucket, object.getKey()));
}
for (Future<Boolean> isdeleted : deletes)
if (!isdeleted.get()) {
throw new S3RuntimeException("failed to delete entry");
}
} catch (Exception e) {
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
}
}
public Set<String> keySet() {
return null; // TODO: Adrian: Customise this generated block
}
public Collection<Object> values() {
return null; // TODO: Adrian: Customise this generated block
}
public Set<Entry<String, Object>> entrySet() {
return null; // TODO: Adrian: Customise this generated block
}
}

77
s3/src/main/java/org/jclouds/aws/s3/S3Utils.java
View File

@ -23,6 +23,14 @@
*/
package org.jclouds.aws.s3;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import org.bouncycastle.crypto.digests.MD5Digest;
import org.bouncycastle.crypto.digests.SHA1Digest;
import org.bouncycastle.crypto.macs.HMac;
import org.bouncycastle.crypto.params.KeyParameter;
@ -30,16 +38,29 @@ import org.bouncycastle.util.encoders.Base64;
import org.jclouds.Utils;
import org.jclouds.aws.s3.domain.S3Object;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
public class S3Utils extends Utils {
static final byte[] HEX_CHAR_TABLE = { (byte) '0', (byte) '1', (byte) '2',
(byte) '3', (byte) '4', (byte) '5', (byte) '6', (byte) '7',
(byte) '8', (byte) '9', (byte) 'a', (byte) 'b', (byte) 'c',
(byte) 'd', (byte) 'e', (byte) 'f' };
public static String digest(String toEncode, byte[] key) throws NoSuchAlgorithmException, NoSuchProviderException, InvalidKeyException {
public static String getHexString(byte[] raw)
throws UnsupportedEncodingException {
byte[] hex = new byte[2 * raw.length];
int index = 0;
for (byte b : raw) {
int v = b & 0xFF;
hex[index++] = HEX_CHAR_TABLE[v >>> 4];
hex[index++] = HEX_CHAR_TABLE[v & 0xF];
}
return new String(hex, "ASCII");
}
public static String hmacSha1Base64(String toEncode, byte[] key)
throws NoSuchAlgorithmException, NoSuchProviderException,
InvalidKeyException {
HMac hmac = new HMac(new SHA1Digest());
byte[] resBuf = new byte[hmac.getMacSize()];
byte[] plainBytes = toEncode.getBytes();
@ -50,8 +71,45 @@ public class S3Utils extends Utils {
return new String(Base64.encode(resBuf));
}
public static String md5Hex(byte [] toEncode)
throws NoSuchAlgorithmException, NoSuchProviderException,
InvalidKeyException, UnsupportedEncodingException {
byte[] resBuf = md5(toEncode);
return getHexString(resBuf);
}
public static String getContentAsStringAndClose(S3Object object) throws IOException {
public static String md5Base64(byte [] toEncode)
throws NoSuchAlgorithmException, NoSuchProviderException,
InvalidKeyException {
byte[] resBuf = md5(toEncode);
return new String(Base64.encode(resBuf));
}
public static byte[] md5(byte[] plainBytes) {
MD5Digest md5 = new MD5Digest();
byte[] resBuf = new byte[md5.getDigestSize()];
md5.update(plainBytes, 0, plainBytes.length);
md5.doFinal(resBuf, 0);
return resBuf;
}
public static byte[] md5(InputStream toEncode) throws IOException {
MD5Digest md5 = new MD5Digest();
byte[] resBuf = new byte[md5.getDigestSize()];
byte[] buffer = new byte[1024];
int numRead = -1;
do {
numRead = toEncode.read(buffer);
if (numRead > 0) {
md5.update(buffer, 0, numRead);
}
} while (numRead != -1);
md5.doFinal(resBuf, 0);
return resBuf;
}
public static String getContentAsStringAndClose(S3Object object)
throws IOException {
Object o = object.getContent();
if (o instanceof InputStream) {
@ -61,7 +119,8 @@ public class S3Utils extends Utils {
}
return returnVal;
} else {
throw new IllegalArgumentException("Object type not supported: " + o.getClass().getName());
throw new IllegalArgumentException("Object type not supported: "
+ o.getClass().getName());
}
}
}

5
s3/src/main/java/org/jclouds/aws/s3/commands/callables/RetrieveObjectCallable.java
View File

@ -67,7 +67,10 @@ public class RetrieveObjectCallable extends
object.setLastModified(dateParser
.dateTimeFromHeaderFormat(getResponse()
.getFirstHeaderOrNull("Last-Modified")));
object.setETag(getResponse().getFirstHeaderOrNull("ETag"));
String eTag = getResponse().getFirstHeaderOrNull("ETag");
if (eTag != null) {
object.setETag(eTag.replaceAll("\"", ""));
}
object.setContentType(getResponse().getFirstHeaderOrNull(
"Content-Type"));
object.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull(

2
s3/src/main/java/org/jclouds/aws/s3/commands/callables/xml/ListBucketHandler.java
View File

@ -68,7 +68,7 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<S3Bucket> {
} else if (qName.equals("LastModified")) {
currentObject.setLastModified(dateParser.dateTimeFromXMLFormat(currentText.toString()));
} else if (qName.equals("ETag")) {
currentObject.setETag(currentText.toString());
currentObject.setETag(currentText.toString().replaceAll("\"", ""));
} else if (qName.equals("Size")) {
currentObject.setSize(Long.parseLong(currentText.toString()));
} else if (qName.equals("Owner")) {

44
s3/src/main/java/org/jclouds/aws/s3/S3ConnectionModule.java → s3/src/main/java/org/jclouds/aws/s3/config/S3ContextModule.java
View File

@ -21,39 +21,55 @@
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.Scopes;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader;
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
import org.jclouds.http.HttpRequestFilter;
package org.jclouds.aws.s3.config;
import java.util.ArrayList;
import java.util.List;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Context;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader;
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
import org.jclouds.aws.s3.internal.GuiceS3Context;
import org.jclouds.aws.s3.internal.LiveS3Connection;
import org.jclouds.aws.s3.internal.LiveS3ObjectMap;
import org.jclouds.aws.s3.internal.GuiceS3Context.S3ObjectMapFactory;
import org.jclouds.http.HttpRequestFilter;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryProvider;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public class S3ConnectionModule extends AbstractModule {
public class S3ContextModule extends AbstractModule {
@Override
protected void configure() {
install(new S3CommandsModule());
bind(S3Connection.class).to(LiveS3Connection.class).in(Scopes.SINGLETON);
bind(S3Connection.class).to(LiveS3Connection.class)
.in(Scopes.SINGLETON);
bind(GuiceS3Context.S3ObjectMapFactory.class).toProvider(
FactoryProvider.newFactory(
GuiceS3Context.S3ObjectMapFactory.class,
LiveS3ObjectMap.class));
bind(S3Context.class).to(GuiceS3Context.class);
}
@Provides
@Singleton
List<HttpRequestFilter> provideRequestFilters(RemoveTransferEncodingHeader removTransferEncodingHeader, RequestAuthorizeSignature requestAuthorizeSignature) {
List<HttpRequestFilter> provideRequestFilters(
RemoveTransferEncodingHeader removTransferEncodingHeader,
RequestAuthorizeSignature requestAuthorizeSignature) {
List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>();
filters.add(removTransferEncodingHeader);
filters.add(requestAuthorizeSignature);
return filters;
}
}

6
s3/src/main/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignature.java
View File

@ -99,8 +99,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
@Inject
public RequestAuthorizeSignature(
@Named("jclouds.aws.accesskeyid") String accessKey,
@Named("jclouds.aws.secretaccesskey") String secretKey,
@Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey,
@Named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY) String secretKey,
DateService dateService) {
this.accessKey = accessKey;
this.secretKey = secretKey;
@ -142,7 +142,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
toSign.append(request.getUri());
String signature;
try {
signature = S3Utils.digest(toSign.toString(), secretKey.getBytes());
signature = S3Utils.hmacSha1Base64(toSign.toString(), secretKey.getBytes());
} catch (Exception e) {
throw new HttpException("error signing request", e);
}

88
s3/src/main/java/org/jclouds/aws/s3/internal/GuiceS3Context.java Normal file
View File

@ -0,0 +1,88 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.internal;
import java.io.IOException;
import org.jclouds.Logger;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Context;
import org.jclouds.aws.s3.S3ObjectMap;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.lifecycle.Closer;
import com.google.inject.Inject;
import com.google.inject.Injector;
/**
* // TODO: Adrian: Document return getConnection!
*
* @author Adrian Cole
*/
public class GuiceS3Context implements S3Context {
public interface S3ObjectMapFactory {
S3ObjectMap createMapView(S3Bucket bucket);
}
private final Logger logger;
private final Injector injector;
private final S3ObjectMapFactory s3ObjectMapFactory;
private final Closer closer;
@Inject
private GuiceS3Context(java.util.logging.Logger logger, Injector injector,
Closer closer, S3ObjectMapFactory s3ObjectMapFactory) {
this.logger = new Logger(logger);
this.injector = injector;
this.s3ObjectMapFactory = s3ObjectMapFactory;
this.closer = closer;
}
/**
* {@inheritDoc}
*/
public S3Connection getConnection() {
return injector.getInstance(S3Connection.class);
}
/**
* {@inheritDoc}
*/
public S3ObjectMap createMapView(S3Bucket bucket) {
getConnection().createBucketIfNotExists(bucket);
return s3ObjectMapFactory.createMapView(bucket);
}
/**
* {@inheritDoc}
*/
public void close() {
try {
closer.close();
} catch (IOException e) {
logger.error(e, "error closing content");
}
}
}

9
s3/src/main/java/org/jclouds/aws/s3/LiveS3Connection.java → s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3Connection.java
View File

@ -21,12 +21,12 @@
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
package org.jclouds.aws.s3.internal;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.Future;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.commands.CopyObject;
import org.jclouds.aws.s3.commands.DeleteBucket;
import org.jclouds.aws.s3.commands.DeleteObject;
@ -127,9 +127,4 @@ public class LiveS3Connection implements S3Connection {
client.submit(listRequest);
return listRequest;
}
public void close() throws IOException {
client.close();
}
}

433
s3/src/main/java/org/jclouds/aws/s3/internal/LiveS3ObjectMap.java Normal file
View File

@ -0,0 +1,433 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.internal;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.apache.commons.io.IOUtils;
import org.jclouds.Utils;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3ObjectMap;
import org.jclouds.aws.s3.S3Utils;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
/**
* Map representation of a live connection to S3.
*
* @author Adrian Cole
*/
public class LiveS3ObjectMap implements S3ObjectMap {
private final S3Connection connection;
private final S3Bucket bucket;
@Inject
public LiveS3ObjectMap(S3Connection connection, @Assisted S3Bucket bucket) {
this.connection = connection;
this.bucket = bucket;
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#size()
*/
public int size() {
try {
return refreshBucket().getContents().size();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#get(java.lang.Object)
*/
public InputStream get(Object o) {
try {
return (InputStream) (connection.getObject(bucket, o.toString())
.get()).getContent();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error geting object %1s:%2s", bucket, o), e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#remove(java.lang.Object)
*/
public InputStream remove(Object o) {
InputStream old = get(o);
try {
connection.deleteObject(bucket, o.toString()).get();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error removing object %1s:%2s", bucket, o), e);
}
return old;
}
public class S3RuntimeException extends RuntimeException {
S3RuntimeException(String s) {
super(s);
}
public S3RuntimeException(String s, Throwable throwable) {
super(s, throwable);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#clear()
*/
public void clear() {
try {
List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
for (String key : keySet()) {
deletes.add(connection.deleteObject(bucket, key));
}
for (Future<Boolean> isdeleted : deletes)
if (!isdeleted.get()) {
throw new S3RuntimeException("failed to delete entry");
}
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
}
}
private S3Bucket refreshBucket() throws InterruptedException,
ExecutionException {
return connection.getBucket(bucket).get();
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#keySet()
*/
public Set<String> keySet() {
try {
Set<String> keys = new HashSet<String>();
for (S3Object object : refreshBucket().getContents())
keys.add(object.getKey());
return keys;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting keys in bucket: "
+ bucket, e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#values()
*/
public Collection<InputStream> values() {
Collection<InputStream> values = new LinkedList<InputStream>();
Set<Future<S3Object>> futureObjects = new HashSet<Future<S3Object>>();
for (String key : keySet()) {
futureObjects.add(connection.getObject(bucket, key));
}
for (Future<S3Object> futureObject : futureObjects) {
S3Object object = null;
try {
object = futureObject.get();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error getting value from bucket %1s:%2s", bucket,
object != null ? object.getKey() : "unknown"), e);
}
System.err.printf("key: %1s, MD5: %2s", object.getKey(), object
.getContentMD5());
values.add((InputStream) object.getContent());
}
return values;
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#entrySet()
*/
public Set<Map.Entry<String, InputStream>> entrySet() {
Set<Map.Entry<String, InputStream>> entrySet = new HashSet<Map.Entry<String, InputStream>>();
for (String key : keySet()) {
Map.Entry<String, InputStream> entry = new Entry(key, get(key));
entrySet.add(entry);
}
return entrySet;
}
public class Entry implements java.util.Map.Entry<String, InputStream> {
private InputStream value;
private String key;
Entry(String key, InputStream value) {
this.key = key;
this.value = value;
}
public String getKey() {
return key;
}
public InputStream getValue() {
return value;
}
public InputStream setValue(InputStream value) {
return put(key, value);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#containsKey(java.lang.Object)
*/
public boolean containsKey(Object key) {
try {
return connection.headObject(bucket, key.toString()).get() != S3Object.NOT_FOUND;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for %1s:%2s", bucket, key), e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#containsValue(java.lang.Object)
*/
public boolean containsValue(Object value) {
try {
byte[] md5;
if (value instanceof InputStream) {
md5 = S3Utils.md5((InputStream) value);
} else if (value instanceof byte[]) {
md5 = S3Utils.md5((byte[]) value);
} else if (value instanceof String) {
md5 = S3Utils.md5(((String) value).getBytes());
} else if (value instanceof File) {
md5 = S3Utils.md5(new FileInputStream((File) value));
} else {
throw new IllegalArgumentException("unsupported value type: "
+ value.getClass());
}
String eTagOfValue = S3Utils.getHexString(md5);
for (S3Object object : refreshBucket().getContents()) {
if (object.getETag().equals(eTagOfValue))
return true;
}
return false;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for ETAG of value: [%2s] in bucket:%1s",
bucket, value), e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMapi#isEmpty()
*/
public boolean isEmpty() {
return keySet().size() == 0;
}
private InputStream putInternal(String s, Object o) {
S3Object object = new S3Object();
try {
InputStream returnVal = containsKey(s) ? get(s) : null;
object.setKey(s);
object.setContent(o);
setSizeIfContentIsInputStream(object);
connection.addObject(bucket, object).get();
return returnVal;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error adding object %1s:%2s", bucket, object), e);
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#putAll(java.util.Map)
*/
public void putAll(Map<? extends String, ? extends InputStream> map) {
putAllInternal(map);
}
public void putAllBytes(Map<? extends String, ? extends byte[]> map) {
putAllInternal(map);
}
public void putAllFiles(Map<? extends String, ? extends File> map) {
putAllInternal(map);
}
public void putAllStrings(Map<? extends String, ? extends String> map) {
putAllInternal(map);
}
private void putAllInternal(Map<? extends String, ? extends Object> map) {
try {
List<Future<String>> puts = new ArrayList<Future<String>>();
for (String key : map.keySet()) {
S3Object object = new S3Object();
object.setKey(key);
object.setContent(map.get(key));
setSizeIfContentIsInputStream(object);
puts.add(connection.addObject(bucket, object));
}
for (Future<String> put : puts)
put.get();// this will throw an exception if there was a problem
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error putting into bucket" + bucket,
e);
}
}
private void setSizeIfContentIsInputStream(S3Object object)
throws IOException {
if (object.getContent() instanceof InputStream) {
byte[] buffer = IOUtils.toByteArray((InputStream) object
.getContent());
object.setSize(buffer.length);
object.setContent(new ByteArrayInputStream(buffer));
}
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#putString(java.lang.String,
* java.lang.String)
*/
public InputStream putString(String key, String value) {
return putInternal(key, value);
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#putFile(java.lang.String,
* java.io.File)
*/
public InputStream putFile(String key, File value) {
return putInternal(key, value);
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#putBytes(java.lang.String, byte[])
*/
public InputStream putBytes(String key, byte[] value) {
return putInternal(key, value);
}
/*
* (non-Javadoc)
*
* @see org.jclouds.aws.s3.S3ObjectMap#put(java.lang.String,
* java.io.InputStream)
*/
public InputStream put(String key, InputStream value) {
return putInternal(key, value);
}
public S3Bucket getBucket() {
return bucket;
}
public InputStream put(S3Object object) {
InputStream returnVal = get(object.getKey());
try {
connection.addObject(bucket, object).get();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error putting object %1s:%2s", bucket, object.getKey()), e);
}
return returnVal;
}
public void putAll(Set<S3Object> objects) {
try {
List<Future<String>> puts = new ArrayList<Future<String>>();
for (S3Object object : objects) {
puts.add(connection.addObject(bucket, object));
}
for (Future<String> put : puts)
put.get();// this will throw an exception if there was a problem
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error putting into bucket" + bucket,
e);
}
}
}

37
s3/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java
View File

@ -41,6 +41,7 @@ import java.util.logging.Logger;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
@ -48,7 +49,6 @@ import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import com.google.inject.Injector;
import com.google.inject.Module;
@Test(sequential = true)
@ -90,44 +90,45 @@ public class S3IntegrationTest {
+ "/adriancole.s3.amazons3test.filetestsforadrian/file</StringToSign><AWSAccessKeyId>0101100101001001</AWSAccessKeyId></Error>";
String amazonHadAnError = "<Error><Code>InternalError</Code><Message>We encountered an internal error. Please try again.</Message><RequestId>EF6FA7A639CAFF15</RequestId><HostId>tBkX23mIeq2riHsNw2YShupMlZ9+iy3V/uN+lRhqCR4qHTE07ujFeyAUPTowvuH/</HostId></Error>";
protected S3Connection client;
Injector i = null;
protected S3Context context = null;
protected String bucketPrefix = System.getProperty("user.name") + "."
+ this.getClass().getName();
private static final String sysAWSAccessKeyId = System
.getProperty("jclouds.aws.accesskeyid");
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
private static final String sysAWSSecretAccessKey = System
.getProperty("jclouds.aws.secretaccesskey");
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
@BeforeTest
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" })
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID,
S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpClient(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey) throws Exception {
i = createInject(AWSAccessKeyId != null ? AWSAccessKeyId
context = createS3Context(AWSAccessKeyId != null ? AWSAccessKeyId
: sysAWSAccessKeyId,
AWSSecretAccessKey != null ? AWSSecretAccessKey
: sysAWSSecretAccessKey);
client = i.getInstance(LiveS3Connection.class);
client = context.getConnection();
deleteEverything();
}
protected Injector createInject(String AWSAccessKeyId,
protected S3Context createS3Context(String AWSAccessKeyId,
String AWSSecretAccessKey) {
return S3ConnectionFactory.getInjector(buildS3Properties(
return S3ContextFactory.createS3Context(buildS3Properties(
AWSAccessKeyId, AWSSecretAccessKey), createHttpModule());
}
protected Properties buildS3Properties(String AWSAccessKeyId,
String AWSSecretAccessKey) {
Properties properties = new Properties(
S3ConnectionFactory.DEFAULT_PROPERTIES);
properties.setProperty("jclouds.aws.accesskeyid", checkNotNull(
AWSAccessKeyId, "AWSAccessKeyId"));
properties.setProperty("jclouds.aws.secretaccesskey", checkNotNull(
AWSSecretAccessKey, "AWSSecretAccessKey"));
properties.setProperty("jclouds.http.secure", "false");
properties.setProperty("jclouds.http.port", "80");
S3ContextFactory.DEFAULT_PROPERTIES);
properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID,
checkNotNull(AWSAccessKeyId, "AWSAccessKeyId"));
properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY,
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey"));
properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false");
properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80");
return properties;
}
@ -163,8 +164,8 @@ public class S3IntegrationTest {
@AfterTest
protected void tearDownClient() throws Exception {
deleteEverything();
client.close();
i = null;
context.close();
context = null;
}
}

298
s3/src/test/java/org/jclouds/aws/s3/S3ObjectMapTest.java
View File

@ -23,59 +23,307 @@
*/
package org.jclouds.aws.s3;
import static org.testng.Assert.assertEquals;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
import org.apache.commons.io.IOUtils;
import org.jclouds.Utils;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import org.jclouds.aws.s3.domain.S3Bucket;
import java.util.Map;
import java.util.logging.Logger;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
/**
* // TODO: Adrian: Document this!
* Tests to cover @{link LiveS3ObjectMap}
*
* @author Adrian Cole
*/
@Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest")
public class S3ObjectMapTest {
private S3Connection connection;
public class S3ObjectMapTest extends S3IntegrationTest {
private S3Bucket bucket;
private Map map;
private S3ObjectMap map;
private Map<String, String> fiveStrings = ImmutableMap.of("one", "apple",
"two", "bear", "three", "candy", "four", "dogma", "five", "emma");
private Map<String, byte[]> fiveBytes = ImmutableMap.of("one", "apple"
.getBytes(), "two", "bear".getBytes(), "three", "candy".getBytes(),
"four", "dogma".getBytes(), "five", "emma".getBytes());
private Map<String, InputStream> fiveInputs;
private Map<String, File> fiveFiles;
String tmpDirectory;
@BeforeMethod
public void setupExecutorService() throws Exception {
connection = new StubS3Connection();
@Parameters( { "basedir" })
protected void setUpTempDir(String basedir) throws InterruptedException,
ExecutionException, FileNotFoundException, IOException {
tmpDirectory = basedir + File.separator + "target" + File.separator
+ "testFiles" + File.separator + getClass().getSimpleName();
new File(tmpDirectory).mkdirs();
fiveFiles = ImmutableMap.of("one", new File(tmpDirectory, "apple"),
"two", new File(tmpDirectory, "bear"), "three", new File(
tmpDirectory, "candy"), "four", new File(tmpDirectory,
"dogma"), "five", new File(tmpDirectory, "emma"));
for (File file : fiveFiles.values()) {
IOUtils.write(file.getName(), new FileOutputStream(file));
}
fiveInputs = ImmutableMap.of("one", IOUtils.toInputStream("apple"),
"two", IOUtils.toInputStream("bear"), "three", IOUtils
.toInputStream("candy"), "four", IOUtils
.toInputStream("dogma"), "five", IOUtils
.toInputStream("emma"));
bucket = new S3Bucket();
bucket.setName("mimi");
connection.createBucketIfNotExists(bucket).get();
map = new S3ObjectMap(Logger.getLogger("test"), connection, bucket, new S3Utils());
bucket.setName(bucketPrefix + ".mimi");
client.createBucketIfNotExists(bucket).get();
map = context.createMapView(bucket);
map.clear();
}
@AfterMethod
public void teardownExecutorService() {
public void tearDown() {
map.clear();
map = null;
bucket = null;
connection = null;
}
@Test
public void testClearWhenNothingInMap(){
public void testClear() {
map.clear();
assert map.size() == 0;
map.putString("one", "apple");
map.clear();
assert map.size() == 0;
}
// @Test
// public void testGetReturnsWhatWasPut(){
// map.put("hello","goodbye");
// assert "goodbye".equals(map.get("hello"));
// assert map.size() == 0;
// }
@Test()
public void testRemove() throws IOException {
map.putString("one", "two");
InputStream old = map.remove("one");
assertEquals(Utils.toStringAndClose(old), "two");
old = map.remove("one");
assert old == null;
old = map.get("one");
assert old == null;
assertEquals(map.keySet().size(), 0);
}
@Test()
public void testKeySet() {
assertEquals(map.keySet().size(), 0);
map.putString("one", "two");
assertEquals(map.keySet(), ImmutableSet.of("one"));
}
@Test()
public void testValues() throws IOException {
map.putAll(this.fiveInputs);
Collection<InputStream> values = map.values();
assertEquals(values.size(), 5);
Set<String> valuesAsString = new HashSet<String>();
for (InputStream stream : values) {
valuesAsString.add(Utils.toStringAndClose(stream));
}
valuesAsString.removeAll(fiveStrings.values());
assert valuesAsString.size() == 0;
}
@Test()
public void testEntrySet() throws IOException {
map.putAllStrings(this.fiveStrings);
Set<Entry<String, InputStream>> entries = map.entrySet();
assertEquals(entries.size(), 5);
for (Entry<String, InputStream> entry : entries) {
assertEquals(IOUtils.toString(entry.getValue()), fiveStrings
.get(entry.getKey()));
entry.setValue(IOUtils.toInputStream(""));
}
assertEquals(map.size(), 5);
for (InputStream value : map.values()) {
assertEquals(IOUtils.toString(value), "");
}
}
@Test()
public void testContainsKey() {
assert !map.containsKey("one");
map.putString("one", "apple");
assert map.containsKey("one");
}
@Test()
public void testContainsStringValue() {
map.putString("one", "apple");
assert map.containsValue(fiveStrings.get("one"));
}
@Test()
public void testContainsFileValue() {
map.putString("one", "apple");
assert map.containsValue(fiveFiles.get("one"));
}
@Test()
public void testContainsInputStreamValue() {
map.putString("one", "apple");
assert map.containsValue(this.fiveInputs.get("one"));
}
@Test()
public void testContainsBytesValue() {
map.putString("one", "apple");
assert map.containsValue(this.fiveBytes.get("one"));
}
@Test()
public void testIsEmpty() {
assert map.isEmpty();
map.putString("one", "apple");
assert !map.isEmpty();
}
@Test()
public void testPutAll() {
map.putAll(this.fiveInputs);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveInputs.keySet()));
fourLeftRemovingOne();
}
@Test()
public void testPutAllBytes() {
map.putAllBytes(this.fiveBytes);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveBytes.keySet()));
fourLeftRemovingOne();
}
@Test
public void testClearWhenSomethingInMap(){
map.put("hello","goodbye");
map.clear();
assert map.size() == 0;
public void testPutAllFiles() {
map.putAllFiles(this.fiveFiles);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveFiles.keySet()));
fourLeftRemovingOne();
}
void fourLeftRemovingOne() {
map.remove("one");
assertEquals(map.size(), 4);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
ImmutableSet.of("two", "three", "four", "five")));
}
@Test()
public void testPutAllStrings() {
map.putAllStrings(this.fiveStrings);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveStrings.keySet()));
fourLeftRemovingOne();
}
@Test()
public void testPutString() throws IOException {
InputStream old = map.putString("one", "apple");
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putString("one", "bear");
getOneReturnsBearAndOldValueIsApple(apple);
}
private void getOneReturnsAppleAndOldValueIsNull(InputStream old)
throws IOException {
assert old == null;
assertEquals(Utils.toStringAndClose(map.get("one")), "apple");
assert map.size() == 1;
}
private void getOneReturnsBearAndOldValueIsApple(InputStream oldValue)
throws IOException {
assertEquals(Utils.toStringAndClose(map.get("one")), "bear");
assertEquals(Utils.toStringAndClose(oldValue), "apple");
assert map.size() == 1;
}
@Test()
public void testPutFile() throws IOException {
InputStream old = map.putFile("one", fiveFiles.get("one"));
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putFile("one", fiveFiles.get("two"));
getOneReturnsBearAndOldValueIsApple(apple);
}
@Test()
public void testPutBytes() throws IOException {
InputStream old = map.putBytes("one", "apple".getBytes());
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putBytes("one", "bear".getBytes());
getOneReturnsBearAndOldValueIsApple(apple);
}
@Test()
public void testPut() throws IOException {
InputStream old = map.put("one", IOUtils.toInputStream("apple"));
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.put("one", IOUtils.toInputStream("bear"));
getOneReturnsBearAndOldValueIsApple(apple);
}
@Test()
public void testGetBucket() {
assertEquals(map.getBucket().getName(), bucket.getName());
}
@Test
void testPutS3Object() throws IOException {
S3Object object = new S3Object();
object.setKey("one");
object.setContent(IOUtils.toInputStream("apple"));
object.setSize("apple".getBytes().length);
InputStream old = map.put(object);
getOneReturnsAppleAndOldValueIsNull(old);
object.setContent(IOUtils.toInputStream("bear"));
object.setSize("bear".getBytes().length);
InputStream apple = map.put(object);
getOneReturnsBearAndOldValueIsApple(apple);
}
@Test
void testPutAllS3Objects() {
Set<S3Object> set = new HashSet<S3Object>();
for (String key : fiveInputs.keySet()) {
S3Object object = new S3Object();
object.setKey(key);
object.setContent(fiveInputs.get(key));
object.setSize(fiveBytes.get(key).length);
set.add(object);
}
map.putAll(set);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveInputs.keySet()));
fourLeftRemovingOne();
}
}

28
s3/src/test/java/org/jclouds/aws/s3/S3UtilsTest.java
View File

@ -28,6 +28,7 @@ import org.jclouds.aws.PerformanceTest;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
@ -51,7 +52,7 @@ public class S3UtilsTest extends PerformanceTest {
String base64Digest) throws NoSuchProviderException,
NoSuchAlgorithmException, InvalidKeyException {
for (int i = 0; i < 10000; i++)
testBouncyCastleDigest(key, message, base64Digest);
testBouncyCastleHmacSha1Base64(key, message, base64Digest);
}
@Test(dataProvider = "hmacsha1")
@ -64,7 +65,7 @@ public class S3UtilsTest extends PerformanceTest {
for (int i = 0; i < 10000; i++)
completer.submit(new Callable<Boolean>() {
public Boolean call() throws Exception {
testBouncyCastleDigest(key, message, base64Digest);
testBouncyCastleHmacSha1Base64(key, message, base64Digest);
return true;
}
});
@ -72,6 +73,17 @@ public class S3UtilsTest extends PerformanceTest {
assert completer.take().get();
}
@DataProvider(name = "md5")
public Object[][] createMD5Data() {
return base64MD5MessageDigest;
}
public final static Object[][] base64MD5MessageDigest = {
{ "apple", "1f3870be274f6c49b3e31a0c6728957f" },
{ "bear", "893b56e3cfe153fb770a120b83bac20c" },
{ "candy", "c48ba993d35c3abe0380f91738fe2a34" },
{ "dogma", "95eb470e4faee302e9cd3063b1923dab" },
{ "emma", "00a809937eddc44521da9521269e75c6" } };
public final static Object[][] base64KeyMessageDigest = {
{ Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There",
"thcxhlUFcmTii8C2+zeMjvFGvgA=" },
@ -96,11 +108,19 @@ public class S3UtilsTest extends PerformanceTest {
}
@Test(dataProvider = "hmacsha1")
public void testBouncyCastleDigest(byte[] key, String message,
public void testBouncyCastleHmacSha1Base64(byte[] key, String message,
String base64Digest) throws NoSuchProviderException,
NoSuchAlgorithmException, InvalidKeyException {
String b64 = S3Utils.digest(message, key);
String b64 = S3Utils.hmacSha1Base64(message, key);
assertEquals(b64, base64Digest);
}
@Test(dataProvider = "md5")
public void testBouncyCastleMD5Digest(String message,
String base64Digest) throws NoSuchProviderException,
NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException {
String b64 = S3Utils.md5Hex(message.getBytes());
assertEquals(base64Digest,b64);
}
}

80
s3/src/test/java/org/jclouds/aws/s3/StubS3Connection.java
View File

@ -23,12 +23,19 @@
*/
package org.jclouds.aws.s3;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Bucket;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.*;
import java.util.concurrent.*;
import java.io.IOException;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
/**
* // TODO: Adrian: Document this!
@ -38,13 +45,14 @@ import java.io.IOException;
public class StubS3Connection implements S3Connection {
private static Map<S3Bucket, Map<String, Object>> bucketToContents = new ConcurrentHashMap<S3Bucket, Map<String, Object>>();
public Future<S3Object> getObject(final S3Bucket s3Bucket, final String key) {
return new FutureBase<S3Object>() {
public S3Object get() throws InterruptedException, ExecutionException {
public S3Object get() throws InterruptedException,
ExecutionException {
if (!bucketToContents.containsKey(s3Bucket))
return S3Object.NOT_FOUND;
Map<String, Object> realContents = bucketToContents.get(s3Bucket);
Map<String, Object> realContents = bucketToContents
.get(s3Bucket);
if (!realContents.containsKey(key))
return S3Object.NOT_FOUND;
S3Object object = new S3Object();
@ -59,9 +67,11 @@ public class StubS3Connection implements S3Connection {
return getObject(s3Bucket, key);
}
public Future<Boolean> deleteObject(final S3Bucket s3Bucket, final String key) {
public Future<Boolean> deleteObject(final S3Bucket s3Bucket,
final String key) {
return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException {
public Boolean get() throws InterruptedException,
ExecutionException {
if (bucketToContents.containsKey(s3Bucket)) {
bucketToContents.get(s3Bucket).remove(key);
}
@ -70,13 +80,16 @@ public class StubS3Connection implements S3Connection {
};
}
public Future<String> addObject(final S3Bucket s3Bucket, final S3Object object) {
public Future<String> addObject(final S3Bucket s3Bucket,
final S3Object object) {
return new FutureBase<String>() {
public String get() throws InterruptedException, ExecutionException {
if (!bucketToContents.containsKey(s3Bucket)) {
throw new ExecutionException(new RuntimeException("bucket not found: " + s3Bucket.getName()));
throw new ExecutionException(new RuntimeException(
"bucket not found: " + s3Bucket.getName()));
}
bucketToContents.get(s3Bucket).put(object.getKey(), object.getClass());
bucketToContents.get(s3Bucket).put(object.getKey(),
object.getClass());
return object.getKey();
}
};
@ -84,9 +97,11 @@ public class StubS3Connection implements S3Connection {
public Future<Boolean> createBucketIfNotExists(final S3Bucket s3Bucket) {
return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException {
public Boolean get() throws InterruptedException,
ExecutionException {
if (!bucketToContents.containsKey(s3Bucket)) {
bucketToContents.put(s3Bucket, new ConcurrentHashMap<String, Object>());
bucketToContents.put(s3Bucket,
new ConcurrentHashMap<String, Object>());
}
return bucketToContents.containsKey(s3Bucket);
}
@ -95,7 +110,8 @@ public class StubS3Connection implements S3Connection {
public Future<Boolean> deleteBucket(final S3Bucket s3Bucket) {
return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException {
public Boolean get() throws InterruptedException,
ExecutionException {
if (bucketToContents.containsKey(s3Bucket)) {
if (bucketToContents.get(s3Bucket).size() == 0)
return true;
@ -105,13 +121,18 @@ public class StubS3Connection implements S3Connection {
};
}
public Future<Boolean> copyObject(final S3Bucket sourceBucket, final S3Object sourceObject, final S3Bucket destinationBucket, final S3Object destinationObject) {
public Future<Boolean> copyObject(final S3Bucket sourceBucket,
final S3Object sourceObject, final S3Bucket destinationBucket,
final S3Object destinationObject) {
return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException {
public Boolean get() throws InterruptedException,
ExecutionException {
Map<String, Object> source = bucketToContents.get(sourceBucket);
Map<String, Object> dest = bucketToContents.get(destinationBucket);
Map<String, Object> dest = bucketToContents
.get(destinationBucket);
if (source.containsKey(sourceObject.getKey())) {
dest.put(destinationObject.getKey(), source.get(sourceObject.getKey()));
dest.put(destinationObject.getKey(), source
.get(sourceObject.getKey()));
return true;
}
return false;
@ -121,7 +142,8 @@ public class StubS3Connection implements S3Connection {
public Future<Boolean> bucketExists(final S3Bucket s3Bucket) {
return new FutureBase<Boolean>() {
public Boolean get() throws InterruptedException, ExecutionException {
public Boolean get() throws InterruptedException,
ExecutionException {
return bucketToContents.containsKey(s3Bucket);
}
};
@ -129,9 +151,11 @@ public class StubS3Connection implements S3Connection {
public Future<S3Bucket> getBucket(final S3Bucket s3Bucket) {
return new FutureBase<S3Bucket>() {
public S3Bucket get() throws InterruptedException, ExecutionException {
public S3Bucket get() throws InterruptedException,
ExecutionException {
Set<S3Object> contents = new HashSet<S3Object>();
Map<String, Object> realContents = bucketToContents.get(s3Bucket);
Map<String, Object> realContents = bucketToContents
.get(s3Bucket);
if (realContents != null) {
for (String key : realContents.keySet()) {
S3Object object = new S3Object();
@ -150,16 +174,13 @@ public class StubS3Connection implements S3Connection {
public Future<List<S3Bucket>> getBuckets() {
return new FutureBase<List<S3Bucket>>() {
public List<S3Bucket> get() throws InterruptedException, ExecutionException {
public List<S3Bucket> get() throws InterruptedException,
ExecutionException {
return new ArrayList<S3Bucket>(bucketToContents.keySet());
}
};
}
public void close() throws IOException {
// nothing to close
}
private abstract class FutureBase<V> implements Future<V> {
public boolean cancel(boolean b) {
return false;
@ -173,7 +194,8 @@ public class StubS3Connection implements S3Connection {
return true;
}
public V get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException {
public V get(long l, TimeUnit timeUnit) throws InterruptedException,
ExecutionException, TimeoutException {
return get();
}
}

4
s3/src/test/java/org/jclouds/aws/s3/commands/S3ParserTest.java
View File

@ -23,6 +23,8 @@
*/
package org.jclouds.aws.s3.commands;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.Callable;
@ -148,7 +150,7 @@ public class S3ParserTest extends PerformanceTest {
assert object.getLastModified().equals(expected) : String
.format("expected %1s, but got %1s", expected, object
.getLastModified());
assert object.getETag().equals("\"9d7bb64e8e18ee34eec06dd2cf37b766\"");
assertEquals(object.getETag(), "9d7bb64e8e18ee34eec06dd2cf37b766");
assert object.getSize() == 136;
S3Owner owner = new S3Owner();
owner

5
s3/src/test/java/org/jclouds/aws/s3/filters/RequestAuthorizeSignatureTest.java
View File

@ -27,6 +27,7 @@ import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.name.Names;
import org.jclouds.aws.s3.DateService;
import org.jclouds.aws.s3.S3Constants;
import org.testng.annotations.Test;
@ -40,8 +41,8 @@ public class RequestAuthorizeSignatureTest {
filter = Guice.createInjector(new AbstractModule() {
protected void configure() {
bindConstant().annotatedWith(Names.named("jclouds.aws.accesskeyid")).to("foo");
bindConstant().annotatedWith(Names.named("jclouds.aws.secretaccesskey")).to("bar");
bindConstant().annotatedWith(Names.named(S3Constants.PROPERTY_AWS_ACCESSKEYID)).to("foo");
bindConstant().annotatedWith(Names.named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY)).to("bar");
bind(DateService.class);
}

12
samples/googleappengine/src/it/java/org/jclouds/samples/googleappengine/functest/GoogleAppEngineTest.java
View File

@ -31,6 +31,7 @@ import java.net.URL;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3Constants;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
@ -47,13 +48,14 @@ import org.testng.annotations.Test;
public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
private static final String sysAWSAccessKeyId = System
.getProperty("jclouds.aws.accesskeyid");
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
private static final String sysAWSSecretAccessKey = System
.getProperty("jclouds.aws.secretaccesskey");
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
@BeforeTest
@Parameters( { "warfile", "devappserver.address", "devappserver.port",
"jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" })
S3Constants.PROPERTY_AWS_ACCESSKEYID,
S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
public void startDevAppServer(final String warfile, final String address,
final String port, @Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey) throws Exception {
@ -66,8 +68,8 @@ public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey");
Properties props = new Properties();
props.put("jclouds.aws.accesskeyid", AWSAccessKeyId);
props.put("jclouds.aws.secretaccesskey", AWSSecretAccessKey);
props.put(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId);
props.put(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey);
writePropertiesAndStartServer(address, port, warfile, props);
}

8
samples/googleappengine/src/main/java/org/jclouds/samples/googleappengine/config/GuiceServletConfig.java
View File

@ -31,8 +31,8 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3ConnectionFactory;
import org.jclouds.aws.s3.S3ConnectionModule;
import org.jclouds.aws.s3.S3ContextFactory;
import org.jclouds.aws.s3.config.S3ContextModule;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import org.jclouds.lifecycle.Closer;
import org.jclouds.samples.googleappengine.JCloudsServlet;
@ -84,11 +84,11 @@ public class GuiceServletConfig extends GuiceServletContextListener {
} finally {
IOUtils.closeQuietly(input);
}
props.putAll(S3ConnectionFactory.DEFAULT_PROPERTIES);
props.putAll(S3ContextFactory.DEFAULT_PROPERTIES);
Names.bindProperties(binder(), props);
}
}, new JavaUrlHttpFutureCommandClientModule(),
new S3ConnectionModule(), new ServletModule() {
new S3ContextModule(), new ServletModule() {
@Override
protected void configureServlets() {
serve("*.s3").with(JCloudsServlet.class);