mirror of https://github.com/apache/jclouds.git
removed google from default module list
git-svn-id: http://jclouds.googlecode.com/svn/trunk@72 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
parent
9dfac47f05
commit
ca8f278580
|
@ -31,5 +31,4 @@ package org.jclouds.command;
|
||||||
public interface FutureCommandClient {
|
public interface FutureCommandClient {
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
<O extends FutureCommand> void submit(O operation);
|
<O extends FutureCommand> void submit(O operation);
|
||||||
void close();
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,46 +23,50 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.command.pool;
|
package org.jclouds.command.pool;
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import java.util.concurrent.BlockingQueue;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.TimeoutException;
|
||||||
|
|
||||||
import org.jclouds.Logger;
|
import org.jclouds.Logger;
|
||||||
import org.jclouds.Utils;
|
import org.jclouds.Utils;
|
||||||
import org.jclouds.command.FutureCommand;
|
import org.jclouds.command.FutureCommand;
|
||||||
import org.jclouds.command.FutureCommandClient;
|
import org.jclouds.command.FutureCommandClient;
|
||||||
import org.jclouds.lifecycle.BaseLifeCycle;
|
import org.jclouds.lifecycle.BaseLifeCycle;
|
||||||
import org.jclouds.lifecycle.Closer;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import com.google.inject.Inject;
|
||||||
import java.util.concurrent.BlockingQueue;
|
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
import java.util.concurrent.TimeoutException;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* // TODO: Adrian: Document this!
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implements FutureCommandClient {
|
public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle
|
||||||
@Inject private Closer closer;
|
implements FutureCommandClient {
|
||||||
private final FutureCommandConnectionPool<C> futureCommandConnectionPool;
|
private final FutureCommandConnectionPool<C> futureCommandConnectionPool;
|
||||||
private final BlockingQueue<FutureCommand> commandQueue;
|
private final BlockingQueue<FutureCommand> commandQueue;
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
public FutureCommandConnectionPoolClient(java.util.logging.Logger logger, ExecutorService executor, FutureCommandConnectionPool<C> futureCommandConnectionPool, BlockingQueue<FutureCommand> commandQueue) {
|
public FutureCommandConnectionPoolClient(java.util.logging.Logger logger,
|
||||||
|
ExecutorService executor,
|
||||||
|
FutureCommandConnectionPool<C> futureCommandConnectionPool,
|
||||||
|
BlockingQueue<FutureCommand> commandQueue) {
|
||||||
super(new Logger(logger), executor, futureCommandConnectionPool);
|
super(new Logger(logger), executor, futureCommandConnectionPool);
|
||||||
this.futureCommandConnectionPool = futureCommandConnectionPool;
|
this.futureCommandConnectionPool = futureCommandConnectionPool;
|
||||||
this.commandQueue = commandQueue;
|
this.commandQueue = commandQueue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean shouldDoWork() {
|
protected boolean shouldDoWork() {
|
||||||
return super.shouldDoWork() && futureCommandConnectionPool.getStatus().equals(Status.ACTIVE);
|
return super.shouldDoWork()
|
||||||
|
&& futureCommandConnectionPool.getStatus()
|
||||||
|
.equals(Status.ACTIVE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void doShutdown() {
|
protected void doShutdown() {
|
||||||
if (exception == null && futureCommandConnectionPool.getException() != null)
|
if (exception == null
|
||||||
|
&& futureCommandConnectionPool.getException() != null)
|
||||||
exception = futureCommandConnectionPool.getException();
|
exception = futureCommandConnectionPool.getException();
|
||||||
while (!commandQueue.isEmpty()) {
|
while (!commandQueue.isEmpty()) {
|
||||||
FutureCommand command = commandQueue.remove();
|
FutureCommand command = commandQueue.remove();
|
||||||
|
@ -75,6 +79,7 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
protected void doWork() throws InterruptedException {
|
protected void doWork() throws InterruptedException {
|
||||||
FutureCommand command = commandQueue.poll(1, TimeUnit.SECONDS);
|
FutureCommand command = commandQueue.poll(1, TimeUnit.SECONDS);
|
||||||
if (command != null) {
|
if (command != null) {
|
||||||
|
@ -87,7 +92,6 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public <O extends FutureCommand> void submit(O operation) {
|
public <O extends FutureCommand> void submit(O operation) {
|
||||||
exceptionIfNotActive();
|
exceptionIfNotActive();
|
||||||
commandQueue.add(operation);
|
commandQueue.add(operation);
|
||||||
|
@ -99,17 +103,25 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
|
||||||
try {
|
try {
|
||||||
connectionHandle = futureCommandConnectionPool.getHandle(operation);
|
connectionHandle = futureCommandConnectionPool.getHandle(operation);
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
logger.warn(e, "Interrupted getting a connection for operation %1s; retrying", operation);
|
logger
|
||||||
|
.warn(
|
||||||
|
e,
|
||||||
|
"Interrupted getting a connection for operation %1s; retrying",
|
||||||
|
operation);
|
||||||
commandQueue.add(operation);
|
commandQueue.add(operation);
|
||||||
return;
|
return;
|
||||||
} catch (TimeoutException e) {
|
} catch (TimeoutException e) {
|
||||||
logger.warn(e, "Timeout getting a connection for operation %1s; retrying", operation);
|
logger.warn(e,
|
||||||
|
"Timeout getting a connection for operation %1s; retrying",
|
||||||
|
operation);
|
||||||
commandQueue.add(operation);
|
commandQueue.add(operation);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (connectionHandle == null) {
|
if (connectionHandle == null) {
|
||||||
logger.error("Failed to obtain connection for operation %1s; retrying", operation);
|
logger.error(
|
||||||
|
"Failed to obtain connection for operation %1s; retrying",
|
||||||
|
operation);
|
||||||
commandQueue.add(operation);
|
commandQueue.add(operation);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -121,18 +133,12 @@ public class FutureCommandConnectionPoolClient<C> extends BaseLifeCycle implemen
|
||||||
final StringBuilder sb = new StringBuilder();
|
final StringBuilder sb = new StringBuilder();
|
||||||
sb.append("FutureCommandConnectionPoolClient");
|
sb.append("FutureCommandConnectionPoolClient");
|
||||||
sb.append("{status=").append(status);
|
sb.append("{status=").append(status);
|
||||||
sb.append(", commandQueue=").append((commandQueue != null) ? commandQueue.size() : 0);
|
sb.append(", commandQueue=").append(
|
||||||
sb.append(", futureCommandConnectionPool=").append(futureCommandConnectionPool);
|
(commandQueue != null) ? commandQueue.size() : 0);
|
||||||
|
sb.append(", futureCommandConnectionPool=").append(
|
||||||
|
futureCommandConnectionPool);
|
||||||
sb.append('}');
|
sb.append('}');
|
||||||
return sb.toString();
|
return sb.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close(){
|
|
||||||
try {
|
|
||||||
closer.close();
|
|
||||||
} catch (IOException e) {
|
|
||||||
e.printStackTrace(); // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,37 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
*
|
||||||
|
* ====================================================================
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
* ====================================================================
|
||||||
|
*/
|
||||||
|
package org.jclouds.command.pool;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* // TODO: Adrian: Document this!
|
||||||
|
*
|
||||||
|
* @author Adrian Cole
|
||||||
|
*/
|
||||||
|
public interface PoolConstants {
|
||||||
|
public static final String PROPERTY_POOL_MAX_CONNECTIONS = "jclouds.pool.max_connections";
|
||||||
|
public static final String PROPERTY_POOL_IO_WORKER_THREADS = "jclouds.http.pool.io_worker_threads";
|
||||||
|
public static final String PROPERTY_POOL_REQUEST_INVOKER_THREADS = "jclouds.http.pool.request_invoker_threads";
|
||||||
|
public static final String PROPERTY_POOL_MAX_SESSION_FAILURES = "jclouds.http.pool.max_session_failures";
|
||||||
|
public static final String PROPERTY_POOL_MAX_CONNECTION_REUSE = "jclouds.http.pool.max_connection_reuse";
|
||||||
|
}
|
|
@ -27,6 +27,7 @@ import com.google.inject.*;
|
||||||
import com.google.inject.name.Named;
|
import com.google.inject.name.Named;
|
||||||
import org.jclouds.command.FutureCommand;
|
import org.jclouds.command.FutureCommand;
|
||||||
import org.jclouds.lifecycle.config.LifeCycleModule;
|
import org.jclouds.lifecycle.config.LifeCycleModule;
|
||||||
|
import org.jclouds.command.pool.PoolConstants;
|
||||||
|
|
||||||
import java.util.concurrent.BlockingQueue;
|
import java.util.concurrent.BlockingQueue;
|
||||||
import java.util.concurrent.LinkedBlockingQueue;
|
import java.util.concurrent.LinkedBlockingQueue;
|
||||||
|
@ -50,8 +51,7 @@ public abstract class FutureCommandConnectionPoolClientModule<C> extends Abstrac
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Singleton
|
@Singleton
|
||||||
public abstract BlockingQueue<C> provideAvailablePool(@Named("jclouds.pool.max_connections") int max) throws Exception;
|
public abstract BlockingQueue<C> provideAvailablePool(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* controls production and destruction of real connections.
|
* controls production and destruction of real connections.
|
||||||
* <p/>
|
* <p/>
|
||||||
|
@ -64,7 +64,7 @@ public abstract class FutureCommandConnectionPoolClientModule<C> extends Abstrac
|
||||||
*/
|
*/
|
||||||
@Provides
|
@Provides
|
||||||
@Singleton
|
@Singleton
|
||||||
public Semaphore provideTotalConnectionSemaphore(@Named("jclouds.pool.max_connections") int max) throws Exception {
|
public Semaphore provideTotalConnectionSemaphore(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception {
|
||||||
return new Semaphore(max, true);
|
return new Semaphore(max, true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,7 +28,7 @@ package org.jclouds.http;
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
public class HttpConstants {
|
public interface HttpConstants {
|
||||||
public static final String CONTENT_LENGTH = "Content-Length";
|
public static final String CONTENT_LENGTH = "Content-Length";
|
||||||
public static final String CONTENT_TYPE = "Content-Type";
|
public static final String CONTENT_TYPE = "Content-Type";
|
||||||
public static final String HOST = "Host";
|
public static final String HOST = "Host";
|
||||||
|
@ -36,4 +36,7 @@ public class HttpConstants {
|
||||||
public static final String BINARY = "application/octet-stream";
|
public static final String BINARY = "application/octet-stream";
|
||||||
public static final String PLAIN = "text/plain";
|
public static final String PLAIN = "text/plain";
|
||||||
public static final String TRANSFER_ENCODING = "Transfer-Encoding";
|
public static final String TRANSFER_ENCODING = "Transfer-Encoding";
|
||||||
|
public static final String PROPERTY_HTTP_SECURE = "jclouds.http.secure";
|
||||||
|
public static final String PROPERTY_HTTP_PORT = "jclouds.http.port";
|
||||||
|
public static final String PROPERTY_HTTP_ADDRESS = "jclouds.http.address";
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,11 +23,12 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.http;
|
package org.jclouds.http;
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import java.util.List;
|
||||||
|
|
||||||
import org.jclouds.command.FutureCommand;
|
import org.jclouds.command.FutureCommand;
|
||||||
import org.jclouds.command.FutureCommandClient;
|
import org.jclouds.command.FutureCommandClient;
|
||||||
|
|
||||||
import java.util.List;
|
import com.google.inject.Inject;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* // TODO: Adrian: Document this!
|
||||||
|
@ -41,6 +42,4 @@ public interface HttpFutureCommandClient extends FutureCommandClient {
|
||||||
void setRequestFilters(List<HttpRequestFilter> requestFilters);
|
void setRequestFilters(List<HttpRequestFilter> requestFilters);
|
||||||
|
|
||||||
<O extends FutureCommand> void submit(O operation);
|
<O extends FutureCommand> void submit(O operation);
|
||||||
|
|
||||||
void close();
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,7 @@ package org.jclouds.http;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
|
@ -82,35 +83,40 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
|
||||||
HttpResponse response = getResponse(connection);
|
HttpResponse response = getResponse(connection);
|
||||||
logger.trace("%1s - received response %2s", target, response);
|
logger.trace("%1s - received response %2s", target, response);
|
||||||
|
|
||||||
|
operation.getResponseFuture().setResponse(response);
|
||||||
|
operation.getResponseFuture().run();
|
||||||
|
} catch (FileNotFoundException e) {
|
||||||
|
HttpResponse response = new HttpResponse();
|
||||||
|
response.setStatusCode(404);
|
||||||
operation.getResponseFuture().setResponse(response);
|
operation.getResponseFuture().setResponse(response);
|
||||||
operation.getResponseFuture().run();
|
operation.getResponseFuture().run();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
if (connection != null) {
|
if (connection != null) {
|
||||||
InputStream errorStream = connection.getErrorStream();
|
StringBuilder errors = new StringBuilder();
|
||||||
if (errorStream != null) {
|
|
||||||
try {
|
try {
|
||||||
String errorMessage = Utils.toStringAndClose(connection
|
for (InputStream in : new InputStream[] {
|
||||||
.getErrorStream());
|
connection.getErrorStream(),
|
||||||
|
connection.getInputStream() }) {
|
||||||
|
if (in != null) {
|
||||||
|
errors.append(Utils.toStringAndClose(in)).append(
|
||||||
|
"\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
logger.error(e,
|
logger.error(e,
|
||||||
"error encountered during the exception: %1s",
|
"error encountered during the exception: %1s",
|
||||||
errorMessage);
|
errors.toString());
|
||||||
} catch (IOException e1) {
|
} catch (IOException e2) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
|
||||||
operation.setException(e);
|
operation.setException(e);
|
||||||
} finally {
|
} finally {
|
||||||
// DO NOT disconnect, as it will also close the unconsumed
|
// DO NOT disconnect, as it will also close the unconsumed
|
||||||
// outputStream from above.
|
// outputStream from above.
|
||||||
// connection.disconnect();
|
if (request.getMethod().equals("HEAD"))
|
||||||
|
connection.disconnect();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close() {
|
|
||||||
// Nothing to stop;
|
|
||||||
}
|
|
||||||
|
|
||||||
private HttpResponse getResponse(HttpURLConnection connection)
|
private HttpResponse getResponse(HttpURLConnection connection)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
HttpResponse response = new HttpResponse();
|
HttpResponse response = new HttpResponse();
|
||||||
|
@ -121,9 +127,11 @@ public class JavaUrlHttpFutureCommandClient implements HttpFutureCommandClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
response.setMessage(connection.getResponseMessage());
|
response.setMessage(connection.getResponseMessage());
|
||||||
|
if (!connection.getRequestMethod().equals("HEAD")) {
|
||||||
response.setContent(connection.getInputStream());
|
response.setContent(connection.getInputStream());
|
||||||
response.setContentType(connection
|
response.setContentType(connection
|
||||||
.getHeaderField(HttpConstants.CONTENT_TYPE));
|
.getHeaderField(HttpConstants.CONTENT_TYPE));
|
||||||
|
}
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -23,19 +23,20 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.http.config;
|
package org.jclouds.http.config;
|
||||||
|
|
||||||
import com.google.inject.AbstractModule;
|
|
||||||
import com.google.inject.Provides;
|
|
||||||
import com.google.inject.Scopes;
|
|
||||||
import com.google.inject.Singleton;
|
|
||||||
import com.google.inject.name.Named;
|
|
||||||
import org.jclouds.http.HttpFutureCommandClient;
|
|
||||||
import org.jclouds.http.JavaUrlHttpFutureCommandClient;
|
|
||||||
|
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
|
|
||||||
|
import org.jclouds.http.HttpConstants;
|
||||||
|
import org.jclouds.http.HttpFutureCommandClient;
|
||||||
|
import org.jclouds.http.JavaUrlHttpFutureCommandClient;
|
||||||
|
|
||||||
|
import com.google.inject.AbstractModule;
|
||||||
|
import com.google.inject.Provides;
|
||||||
|
import com.google.inject.Singleton;
|
||||||
|
import com.google.inject.name.Named;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* Configures {@link JavaUrlHttpFutureCommandClient}.
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
|
@ -44,15 +45,19 @@ public class JavaUrlHttpFutureCommandClientModule extends AbstractModule {
|
||||||
@Override
|
@Override
|
||||||
protected void configure() {
|
protected void configure() {
|
||||||
// note this is not threadsafe, so it cannot be singleton
|
// note this is not threadsafe, so it cannot be singleton
|
||||||
bind(HttpFutureCommandClient.class).to(JavaUrlHttpFutureCommandClient.class);
|
bind(HttpFutureCommandClient.class).to(
|
||||||
|
JavaUrlHttpFutureCommandClient.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
@Provides
|
@Provides
|
||||||
protected URL provideAddress(@Named("jclouds.http.address") String address, @Named("jclouds.http.port") int port, @Named("jclouds.http.secure") boolean isSecure) throws MalformedURLException {
|
protected URL provideAddress(
|
||||||
|
@Named(HttpConstants.PROPERTY_HTTP_ADDRESS) String address,
|
||||||
|
@Named(HttpConstants.PROPERTY_HTTP_PORT) int port,
|
||||||
|
@Named(HttpConstants.PROPERTY_HTTP_SECURE) boolean isSecure)
|
||||||
|
throws MalformedURLException {
|
||||||
|
|
||||||
return new URL(isSecure ? "https" : "http", address, port, "/");
|
return new URL(isSecure ? "https" : "http", address, port, "/");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
|
@ -40,6 +40,7 @@ import org.jclouds.http.commands.GetString;
|
||||||
import org.jclouds.http.commands.Head;
|
import org.jclouds.http.commands.Head;
|
||||||
import org.jclouds.http.commands.callables.xml.ParseSax;
|
import org.jclouds.http.commands.callables.xml.ParseSax;
|
||||||
import org.jclouds.http.commands.config.HttpCommandsModule;
|
import org.jclouds.http.commands.config.HttpCommandsModule;
|
||||||
|
import org.jclouds.lifecycle.Closer;
|
||||||
import org.mortbay.jetty.Handler;
|
import org.mortbay.jetty.Handler;
|
||||||
import org.mortbay.jetty.Request;
|
import org.mortbay.jetty.Request;
|
||||||
import org.mortbay.jetty.Server;
|
import org.mortbay.jetty.Server;
|
||||||
|
@ -69,6 +70,7 @@ public abstract class BaseHttpFutureCommandClientTest {
|
||||||
protected CommandFactory factory;
|
protected CommandFactory factory;
|
||||||
protected HttpFutureCommandClient client;
|
protected HttpFutureCommandClient client;
|
||||||
protected Injector injector;
|
protected Injector injector;
|
||||||
|
private Closer closer;
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
@Parameters( { "test-jetty-port" })
|
@Parameters( { "test-jetty-port" })
|
||||||
|
@ -95,9 +97,9 @@ public abstract class BaseHttpFutureCommandClientTest {
|
||||||
server.setHandler(handler);
|
server.setHandler(handler);
|
||||||
server.start();
|
server.start();
|
||||||
final Properties properties = new Properties();
|
final Properties properties = new Properties();
|
||||||
properties.put("jclouds.http.address", "localhost");
|
properties.put(HttpConstants.PROPERTY_HTTP_ADDRESS, "localhost");
|
||||||
properties.put("jclouds.http.port", testPort + "");
|
properties.put(HttpConstants.PROPERTY_HTTP_PORT, testPort + "");
|
||||||
properties.put("jclouds.http.secure", "false");
|
properties.put(HttpConstants.PROPERTY_HTTP_SECURE, "false");
|
||||||
addConnectionProperties(properties);
|
addConnectionProperties(properties);
|
||||||
final List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>(
|
final List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>(
|
||||||
1);
|
1);
|
||||||
|
@ -123,6 +125,7 @@ public abstract class BaseHttpFutureCommandClientTest {
|
||||||
});
|
});
|
||||||
factory = injector.getInstance(CommandFactory.class);
|
factory = injector.getInstance(CommandFactory.class);
|
||||||
client = injector.getInstance(HttpFutureCommandClient.class);
|
client = injector.getInstance(HttpFutureCommandClient.class);
|
||||||
|
closer = injector.getInstance(Closer.class);
|
||||||
assert client != null;
|
assert client != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -132,7 +135,7 @@ public abstract class BaseHttpFutureCommandClientTest {
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public void tearDownJetty() throws Exception {
|
public void tearDownJetty() throws Exception {
|
||||||
client.close();
|
closer.close();
|
||||||
server.stop();
|
server.stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -61,7 +61,7 @@ public class HttpNioUtils {
|
||||||
String contentType, long length) {
|
String contentType, long length) {
|
||||||
if (content instanceof InputStream) {
|
if (content instanceof InputStream) {
|
||||||
InputStream inputStream = (InputStream) content;
|
InputStream inputStream = (InputStream) content;
|
||||||
if (length <= 0)
|
if (length == -1)
|
||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException(
|
||||||
"you must specify size when content is an InputStream");
|
"you must specify size when content is an InputStream");
|
||||||
InputStreamEntity entity = new InputStreamEntity(inputStream,
|
InputStreamEntity entity = new InputStreamEntity(inputStream,
|
||||||
|
|
|
@ -23,48 +23,51 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.http.httpnio.config;
|
package org.jclouds.http.httpnio.config;
|
||||||
|
|
||||||
import com.google.inject.AbstractModule;
|
import java.net.InetSocketAddress;
|
||||||
import com.google.inject.Provides;
|
|
||||||
import com.google.inject.Scopes;
|
|
||||||
import com.google.inject.Singleton;
|
|
||||||
import com.google.inject.TypeLiteral;
|
|
||||||
import com.google.inject.name.Named;
|
|
||||||
|
|
||||||
import org.apache.http.nio.NHttpConnection;
|
import org.apache.http.nio.NHttpConnection;
|
||||||
import org.jclouds.command.pool.FutureCommandConnectionRetry;
|
import org.jclouds.command.pool.FutureCommandConnectionRetry;
|
||||||
|
import org.jclouds.http.HttpConstants;
|
||||||
import org.jclouds.http.HttpFutureCommandClient;
|
import org.jclouds.http.HttpFutureCommandClient;
|
||||||
import org.jclouds.http.httpnio.config.internal.NonSSLHttpNioConnectionPoolClientModule;
|
import org.jclouds.http.httpnio.config.internal.NonSSLHttpNioConnectionPoolClientModule;
|
||||||
import org.jclouds.http.httpnio.config.internal.SSLHttpNioConnectionPoolClientModule;
|
import org.jclouds.http.httpnio.config.internal.SSLHttpNioConnectionPoolClientModule;
|
||||||
import org.jclouds.http.httpnio.pool.HttpNioConnectionPoolClient;
|
import org.jclouds.http.httpnio.pool.HttpNioConnectionPoolClient;
|
||||||
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionRetry;
|
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionRetry;
|
||||||
|
|
||||||
import java.net.InetSocketAddress;
|
import com.google.inject.AbstractModule;
|
||||||
|
import com.google.inject.Provides;
|
||||||
|
import com.google.inject.Singleton;
|
||||||
|
import com.google.inject.TypeLiteral;
|
||||||
|
import com.google.inject.name.Named;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* Configures {@link HttpNioConnectionPoolClient}
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
public class HttpNioConnectionPoolClientModule extends AbstractModule {
|
public class HttpNioConnectionPoolClientModule extends AbstractModule {
|
||||||
|
|
||||||
@Named("jclouds.http.secure")
|
@Named(HttpConstants.PROPERTY_HTTP_SECURE)
|
||||||
boolean isSecure;
|
boolean isSecure;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void configure() {
|
protected void configure() {
|
||||||
requestInjection(this);
|
requestInjection(this);
|
||||||
//TODO test...
|
|
||||||
if (isSecure)
|
if (isSecure)
|
||||||
install(new SSLHttpNioConnectionPoolClientModule());
|
install(new SSLHttpNioConnectionPoolClientModule());
|
||||||
else
|
else
|
||||||
install(new NonSSLHttpNioConnectionPoolClientModule());
|
install(new NonSSLHttpNioConnectionPoolClientModule());
|
||||||
bind(new TypeLiteral<FutureCommandConnectionRetry<NHttpConnection>>(){}).to(HttpNioFutureCommandConnectionRetry.class);
|
bind(new TypeLiteral<FutureCommandConnectionRetry<NHttpConnection>>() {
|
||||||
bind(HttpFutureCommandClient.class).to(HttpNioConnectionPoolClient.class);
|
}).to(HttpNioFutureCommandConnectionRetry.class);
|
||||||
|
bind(HttpFutureCommandClient.class).to(
|
||||||
|
HttpNioConnectionPoolClient.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
@Provides
|
@Provides
|
||||||
protected InetSocketAddress provideAddress(@Named("jclouds.http.address") String address, @Named("jclouds.http.port") int port) {
|
protected InetSocketAddress provideAddress(
|
||||||
|
@Named(HttpConstants.PROPERTY_HTTP_ADDRESS) String address,
|
||||||
|
@Named(HttpConstants.PROPERTY_HTTP_PORT) int port) {
|
||||||
return new InetSocketAddress(address, port);
|
return new InetSocketAddress(address, port);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -45,6 +45,7 @@ import org.apache.http.params.CoreProtocolPNames;
|
||||||
import org.apache.http.params.HttpParams;
|
import org.apache.http.params.HttpParams;
|
||||||
import org.apache.http.protocol.*;
|
import org.apache.http.protocol.*;
|
||||||
import org.jclouds.command.pool.FutureCommandConnectionRetry;
|
import org.jclouds.command.pool.FutureCommandConnectionRetry;
|
||||||
|
import org.jclouds.command.pool.PoolConstants;
|
||||||
import org.jclouds.command.pool.config.FutureCommandConnectionPoolClientModule;
|
import org.jclouds.command.pool.config.FutureCommandConnectionPoolClientModule;
|
||||||
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionHandle;
|
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionHandle;
|
||||||
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionPool;
|
import org.jclouds.http.httpnio.pool.HttpNioFutureCommandConnectionPool;
|
||||||
|
@ -114,7 +115,7 @@ public abstract class BaseHttpNioConnectionPoolClientModule extends FutureComman
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public BlockingQueue<NHttpConnection> provideAvailablePool(@Named("jclouds.pool.max_connections") int max) throws Exception {
|
public BlockingQueue<NHttpConnection> provideAvailablePool(@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS) int max) throws Exception {
|
||||||
return new ArrayBlockingQueue<NHttpConnection>(max, true);
|
return new ArrayBlockingQueue<NHttpConnection>(max, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -125,7 +126,7 @@ public abstract class BaseHttpNioConnectionPoolClientModule extends FutureComman
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Singleton
|
@Singleton
|
||||||
public DefaultConnectingIOReactor provideDefaultConnectingIOReactor(@Named("jclouds.http.pool.io_worker_threads") int ioWorkerThreads, HttpParams params) throws IOReactorException {
|
public DefaultConnectingIOReactor provideDefaultConnectingIOReactor(@Named(PoolConstants.PROPERTY_POOL_IO_WORKER_THREADS) int ioWorkerThreads, HttpParams params) throws IOReactorException {
|
||||||
return new DefaultConnectingIOReactor(ioWorkerThreads, params);
|
return new DefaultConnectingIOReactor(ioWorkerThreads, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -23,8 +23,13 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.http.httpnio.pool;
|
package org.jclouds.http.httpnio.pool;
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import java.io.IOException;
|
||||||
import com.google.inject.name.Named;
|
import java.net.InetSocketAddress;
|
||||||
|
import java.util.concurrent.BlockingQueue;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Semaphore;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.apache.http.HttpException;
|
import org.apache.http.HttpException;
|
||||||
import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
|
import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
|
||||||
import org.apache.http.nio.NHttpConnection;
|
import org.apache.http.nio.NHttpConnection;
|
||||||
|
@ -38,21 +43,18 @@ import org.jclouds.Logger;
|
||||||
import org.jclouds.command.FutureCommand;
|
import org.jclouds.command.FutureCommand;
|
||||||
import org.jclouds.command.pool.FutureCommandConnectionPool;
|
import org.jclouds.command.pool.FutureCommandConnectionPool;
|
||||||
import org.jclouds.command.pool.FutureCommandConnectionRetry;
|
import org.jclouds.command.pool.FutureCommandConnectionRetry;
|
||||||
|
import org.jclouds.command.pool.PoolConstants;
|
||||||
|
|
||||||
import java.io.IOException;
|
import com.google.inject.Inject;
|
||||||
import java.net.InetSocketAddress;
|
import com.google.inject.name.Named;
|
||||||
import java.util.concurrent.BlockingQueue;
|
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.Semaphore;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* Connection Pool for HTTP requests that utilizes Apache HTTPNio
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionPool<NHttpConnection> implements EventListener {
|
public class HttpNioFutureCommandConnectionPool extends
|
||||||
|
FutureCommandConnectionPool<NHttpConnection> implements EventListener {
|
||||||
|
|
||||||
private final NHttpClientConnectionPoolSessionRequestCallback sessionCallback;
|
private final NHttpClientConnectionPoolSessionRequestCallback sessionCallback;
|
||||||
private final DefaultConnectingIOReactor ioReactor;
|
private final DefaultConnectingIOReactor ioReactor;
|
||||||
|
@ -61,8 +63,22 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
|
||||||
private final int maxSessionFailures;
|
private final int maxSessionFailures;
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
public HttpNioFutureCommandConnectionPool(java.util.logging.Logger logger, ExecutorService executor, Semaphore allConnections, BlockingQueue<NHttpConnection> available, AsyncNHttpClientHandler clientHandler, DefaultConnectingIOReactor ioReactor, IOEventDispatch dispatch, FutureCommandConnectionHandleFactory requestHandleFactory, InetSocketAddress target, FutureCommandConnectionRetry<NHttpConnection> futureCommandConnectionRetry, @Named("jclouds.http.pool.max_connection_reuse") int maxConnectionReuse, @Named("jclouds.http.pool.max_session_failures") int maxSessionFailures) {
|
public HttpNioFutureCommandConnectionPool(
|
||||||
super(new Logger(logger), executor, futureCommandConnectionRetry, allConnections, requestHandleFactory, maxConnectionReuse, available);
|
java.util.logging.Logger logger,
|
||||||
|
ExecutorService executor,
|
||||||
|
Semaphore allConnections,
|
||||||
|
BlockingQueue<NHttpConnection> available,
|
||||||
|
AsyncNHttpClientHandler clientHandler,
|
||||||
|
DefaultConnectingIOReactor ioReactor,
|
||||||
|
IOEventDispatch dispatch,
|
||||||
|
FutureCommandConnectionHandleFactory requestHandleFactory,
|
||||||
|
InetSocketAddress target,
|
||||||
|
FutureCommandConnectionRetry<NHttpConnection> futureCommandConnectionRetry,
|
||||||
|
@Named(PoolConstants.PROPERTY_POOL_MAX_CONNECTION_REUSE) int maxConnectionReuse,
|
||||||
|
@Named(PoolConstants.PROPERTY_POOL_MAX_SESSION_FAILURES) int maxSessionFailures) {
|
||||||
|
super(new Logger(logger), executor, futureCommandConnectionRetry,
|
||||||
|
allConnections, requestHandleFactory, maxConnectionReuse,
|
||||||
|
available);
|
||||||
this.ioReactor = ioReactor;
|
this.ioReactor = ioReactor;
|
||||||
this.dispatch = dispatch;
|
this.dispatch = dispatch;
|
||||||
this.target = target;
|
this.target = target;
|
||||||
|
@ -99,9 +115,9 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public boolean connectionValid(NHttpConnection conn) {
|
public boolean connectionValid(NHttpConnection conn) {
|
||||||
return conn.isOpen() && !conn.isStale() && conn.getMetrics().getRequestCount() < maxConnectionReuse;
|
return conn.isOpen() && !conn.isStale()
|
||||||
|
&& conn.getMetrics().getRequestCount() < maxConnectionReuse;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void doWork() throws Exception {
|
protected void doWork() throws Exception {
|
||||||
|
@ -128,31 +144,38 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean shouldDoWork() {
|
protected boolean shouldDoWork() {
|
||||||
return super.shouldDoWork() && ioReactor.getStatus().equals(IOReactorStatus.ACTIVE);
|
return super.shouldDoWork()
|
||||||
|
&& ioReactor.getStatus().equals(IOReactorStatus.ACTIVE);
|
||||||
}
|
}
|
||||||
|
|
||||||
class NHttpClientConnectionPoolSessionRequestCallback implements SessionRequestCallback {
|
class NHttpClientConnectionPoolSessionRequestCallback implements
|
||||||
|
SessionRequestCallback {
|
||||||
|
|
||||||
public void completed(SessionRequest request) {
|
public void completed(SessionRequest request) {
|
||||||
logger.trace("%1s - %2s - operation complete", request, request.getAttachment());
|
logger.trace("%1s - %2s - operation complete", request, request
|
||||||
|
.getAttachment());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void cancelled(SessionRequest request) {
|
public void cancelled(SessionRequest request) {
|
||||||
logger.info("%1s - %2s - operation cancelled", request, request.getAttachment());
|
logger.info("%1s - %2s - operation cancelled", request, request
|
||||||
|
.getAttachment());
|
||||||
releaseConnectionAndCancelResponse(request);
|
releaseConnectionAndCancelResponse(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void releaseConnectionAndCancelResponse(SessionRequest request) {
|
private void releaseConnectionAndCancelResponse(SessionRequest request) {
|
||||||
allConnections.release();
|
allConnections.release();
|
||||||
FutureCommand frequest = (FutureCommand) request.getAttachment();
|
FutureCommand<?, ?, ?> frequest = (FutureCommand<?, ?, ?>) request
|
||||||
|
.getAttachment();
|
||||||
if (frequest != null) {
|
if (frequest != null) {
|
||||||
frequest.cancel(true);
|
frequest.cancel(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void releaseConnectionAndSetResponseException(SessionRequest request, Exception e) {
|
private void releaseConnectionAndSetResponseException(
|
||||||
|
SessionRequest request, Exception e) {
|
||||||
allConnections.release();
|
allConnections.release();
|
||||||
FutureCommand frequest = (FutureCommand) request.getAttachment();
|
FutureCommand<?, ?, ?> frequest = (FutureCommand<?, ?, ?>) request
|
||||||
|
.getAttachment();
|
||||||
if (frequest != null) {
|
if (frequest != null) {
|
||||||
frequest.setException(e);
|
frequest.setException(e);
|
||||||
}
|
}
|
||||||
|
@ -160,8 +183,11 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
|
||||||
|
|
||||||
public void failed(SessionRequest request) {
|
public void failed(SessionRequest request) {
|
||||||
int count = currentSessionFailures.getAndIncrement();
|
int count = currentSessionFailures.getAndIncrement();
|
||||||
logger.error(request.getException(), "%1s - %2s - operation failed", request, request.getAttachment());
|
logger.error(request.getException(),
|
||||||
releaseConnectionAndSetResponseException(request, request.getException());
|
"%1s - %2s - operation failed", request, request
|
||||||
|
.getAttachment());
|
||||||
|
releaseConnectionAndSetResponseException(request, request
|
||||||
|
.getException());
|
||||||
if (count >= maxSessionFailures) {
|
if (count >= maxSessionFailures) {
|
||||||
exception = request.getException();
|
exception = request.getException();
|
||||||
}
|
}
|
||||||
|
@ -169,22 +195,22 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
|
||||||
}
|
}
|
||||||
|
|
||||||
public void timeout(SessionRequest request) {
|
public void timeout(SessionRequest request) {
|
||||||
logger.warn("%1s - %2s - operation timed out", request, request.getAttachment());
|
logger.warn("%1s - %2s - operation timed out", request, request
|
||||||
|
.getAttachment());
|
||||||
releaseConnectionAndCancelResponse(request);
|
releaseConnectionAndCancelResponse(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void connectionOpen(NHttpConnection conn) {
|
public void connectionOpen(NHttpConnection conn) {
|
||||||
conn.setSocketTimeout(0);
|
conn.setSocketTimeout(0);
|
||||||
available.offer(conn);
|
available.offer(conn);
|
||||||
logger.trace("%1s - %2d - open", conn, conn.hashCode());
|
logger.trace("%1s - %2d - open", conn, conn.hashCode());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void connectionTimeout(NHttpConnection conn) {
|
public void connectionTimeout(NHttpConnection conn) {
|
||||||
logger.warn("%1s - %2d - timeout %2d", conn, conn.hashCode(), conn.getSocketTimeout());
|
logger.warn("%1s - %2d - timeout %2d", conn, conn.hashCode(), conn
|
||||||
|
.getSocketTimeout());
|
||||||
allConnections.release();
|
allConnections.release();
|
||||||
futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn);
|
futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn);
|
||||||
}
|
}
|
||||||
|
@ -196,18 +222,23 @@ public class HttpNioFutureCommandConnectionPool extends FutureCommandConnectionP
|
||||||
|
|
||||||
public void fatalIOException(IOException ex, NHttpConnection conn) {
|
public void fatalIOException(IOException ex, NHttpConnection conn) {
|
||||||
exception = ex;
|
exception = ex;
|
||||||
logger.error(ex, "%1s - %2d - %3s - pool error", conn, conn.hashCode(), target);
|
logger.error(ex, "%1s - %2d - %3s - pool error", conn, conn.hashCode(),
|
||||||
|
target);
|
||||||
futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn);
|
futureCommandConnectionRetry.shutdownConnectionAndRetryOperation(conn);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void fatalProtocolException(HttpException ex, NHttpConnection conn) {
|
public void fatalProtocolException(HttpException ex, NHttpConnection conn) {
|
||||||
exception = ex;
|
exception = ex;
|
||||||
logger.error(ex, "%1s - %2d - %3s - http error", conn, conn.hashCode(), target);
|
logger.error(ex, "%1s - %2d - %3s - http error", conn, conn.hashCode(),
|
||||||
|
target);
|
||||||
fatalException(ex, conn);
|
fatalException(ex, conn);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static interface FutureCommandConnectionHandleFactory extends FutureCommandConnectionPool.FutureCommandConnectionHandleFactory<NHttpConnection> {
|
public static interface FutureCommandConnectionHandleFactory
|
||||||
HttpNioFutureCommandConnectionHandle create(FutureCommand command, NHttpConnection conn);
|
extends
|
||||||
|
FutureCommandConnectionPool.FutureCommandConnectionHandleFactory<NHttpConnection> {
|
||||||
|
HttpNioFutureCommandConnectionHandle create(FutureCommand command,
|
||||||
|
NHttpConnection conn);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -24,6 +24,8 @@
|
||||||
package org.jclouds.http.httpnio.pool;
|
package org.jclouds.http.httpnio.pool;
|
||||||
|
|
||||||
import com.google.inject.Module;
|
import com.google.inject.Module;
|
||||||
|
|
||||||
|
import org.jclouds.command.pool.PoolConstants;
|
||||||
import org.jclouds.http.BaseHttpFutureCommandClientTest;
|
import org.jclouds.http.BaseHttpFutureCommandClientTest;
|
||||||
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
|
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
|
@ -36,14 +38,20 @@ import java.util.Properties;
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public class HttpNioConnectionPoolFutureCommandClientTest extends BaseHttpFutureCommandClientTest {
|
public class HttpNioConnectionPoolFutureCommandClientTest extends
|
||||||
|
BaseHttpFutureCommandClientTest {
|
||||||
|
|
||||||
protected void addConnectionProperties(Properties properties) {
|
protected void addConnectionProperties(Properties properties) {
|
||||||
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
|
properties.setProperty(
|
||||||
properties.setProperty("jclouds.http.pool.max_session_failures", "2");
|
PoolConstants.PROPERTY_POOL_MAX_CONNECTION_REUSE, "75");
|
||||||
properties.setProperty("jclouds.http.pool.request_invoker_threads", "1");
|
properties.setProperty(
|
||||||
properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
|
PoolConstants.PROPERTY_POOL_MAX_SESSION_FAILURES, "2");
|
||||||
properties.setProperty("jclouds.pool.max_connections", "12");
|
properties.setProperty(
|
||||||
|
PoolConstants.PROPERTY_POOL_REQUEST_INVOKER_THREADS, "1");
|
||||||
|
properties.setProperty(PoolConstants.PROPERTY_POOL_IO_WORKER_THREADS,
|
||||||
|
"2");
|
||||||
|
properties.setProperty(PoolConstants.PROPERTY_POOL_MAX_CONNECTIONS,
|
||||||
|
"12");
|
||||||
}
|
}
|
||||||
|
|
||||||
protected Module createClientModule() {
|
protected Module createClientModule() {
|
||||||
|
|
|
@ -46,11 +46,48 @@
|
||||||
<url>http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</url>
|
<url>http://jclouds.googlecode.com/svn/trunk/extensions/s3nio</url>
|
||||||
</scm>
|
</scm>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<jclouds.aws.accesskeyid></jclouds.aws.accesskeyid>
|
||||||
|
<jclouds.aws.secretaccesskey></jclouds.aws.secretaccesskey>
|
||||||
|
</properties>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>${project.groupId}</groupId>
|
<groupId>${project.groupId}</groupId>
|
||||||
<artifactId>jclouds-httpnio</artifactId>
|
<artifactId>jclouds-httpnio</artifactId>
|
||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>${project.groupId}</groupId>
|
||||||
|
<artifactId>jclouds-s3</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>${project.groupId}</groupId>
|
||||||
|
<artifactId>jclouds-s3</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<type>test-jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-surefire-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<systemProperties>
|
||||||
|
<property>
|
||||||
|
<name>jclouds.aws.accesskeyid</name>
|
||||||
|
<value>${jclouds.aws.accesskeyid}</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>jclouds.aws.secretaccesskey</name>
|
||||||
|
<value>${jclouds.aws.secretaccesskey}</value>
|
||||||
|
</property>
|
||||||
|
</systemProperties>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -0,0 +1,61 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
*
|
||||||
|
* ====================================================================
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
* ====================================================================
|
||||||
|
*/
|
||||||
|
package org.jclouds.aws.s3.nio;
|
||||||
|
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.jclouds.aws.s3.S3ObjectMapTest;
|
||||||
|
import org.jclouds.aws.s3.nio.config.S3HttpNioConnectionPoolClientModule;
|
||||||
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
|
import com.google.inject.Module;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* // TODO: Adrian: Document this!
|
||||||
|
*
|
||||||
|
* @author Adrian Cole
|
||||||
|
*/
|
||||||
|
@Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest")
|
||||||
|
public class NioS3ObjectMapTest extends S3ObjectMapTest {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Properties buildS3Properties(String AWSAccessKeyId,
|
||||||
|
String AWSSecretAccessKey) {
|
||||||
|
Properties properties = super.buildS3Properties(AWSAccessKeyId,
|
||||||
|
AWSSecretAccessKey);
|
||||||
|
properties.setProperty("jclouds.http.pool.max_connection_reuse", "75");
|
||||||
|
properties.setProperty("jclouds.http.pool.max_session_failures", "2");
|
||||||
|
properties
|
||||||
|
.setProperty("jclouds.http.pool.request_invoker_threads", "1");
|
||||||
|
properties.setProperty("jclouds.http.pool.io_worker_threads", "2");
|
||||||
|
properties.setProperty("jclouds.pool.max_connections", "12");
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Module createHttpModule() {
|
||||||
|
return new S3HttpNioConnectionPoolClientModule();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
1
pom.xml
1
pom.xml
|
@ -44,7 +44,6 @@
|
||||||
<module>extensions/s3nio</module>
|
<module>extensions/s3nio</module>
|
||||||
<module>s3</module>
|
<module>s3</module>
|
||||||
<module>s3/perftest</module>
|
<module>s3/perftest</module>
|
||||||
<module>samples/googleappengine</module>
|
|
||||||
</modules>
|
</modules>
|
||||||
<build>
|
<build>
|
||||||
<plugins>
|
<plugins>
|
||||||
|
|
|
@ -1,4 +1,29 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
|
||||||
|
|
||||||
|
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
|
||||||
|
====================================================================
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
or more contributor license agreements. See the NOTICE file
|
||||||
|
distributed with this work for additional information
|
||||||
|
regarding copyright ownership. The ASF licenses this file
|
||||||
|
to you under the Apache License, Version 2.0 (the
|
||||||
|
"License"); you may not use this file except in compliance
|
||||||
|
with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing,
|
||||||
|
software distributed under the License is distributed on an
|
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations
|
||||||
|
under the License.
|
||||||
|
====================================================================
|
||||||
|
|
||||||
|
-->
|
||||||
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
|
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||||
<groupId>com.google.code.guice</groupId>
|
<groupId>com.google.code.guice</groupId>
|
||||||
|
|
|
@ -1,4 +1,29 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
|
||||||
|
|
||||||
|
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
|
||||||
|
====================================================================
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
or more contributor license agreements. See the NOTICE file
|
||||||
|
distributed with this work for additional information
|
||||||
|
regarding copyright ownership. The ASF licenses this file
|
||||||
|
to you under the Apache License, Version 2.0 (the
|
||||||
|
"License"); you may not use this file except in compliance
|
||||||
|
with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing,
|
||||||
|
software distributed under the License is distributed on an
|
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations
|
||||||
|
under the License.
|
||||||
|
====================================================================
|
||||||
|
|
||||||
|
-->
|
||||||
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
|
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||||
<groupId>com.google.code.guice</groupId>
|
<groupId>com.google.code.guice</groupId>
|
||||||
|
|
|
@ -1,4 +1,29 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
|
||||||
|
|
||||||
|
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
|
||||||
|
====================================================================
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
or more contributor license agreements. See the NOTICE file
|
||||||
|
distributed with this work for additional information
|
||||||
|
regarding copyright ownership. The ASF licenses this file
|
||||||
|
to you under the Apache License, Version 2.0 (the
|
||||||
|
"License"); you may not use this file except in compliance
|
||||||
|
with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing,
|
||||||
|
software distributed under the License is distributed on an
|
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations
|
||||||
|
under the License.
|
||||||
|
====================================================================
|
||||||
|
|
||||||
|
-->
|
||||||
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
|
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||||
<groupId>com.google.code.guice</groupId>
|
<groupId>com.google.code.guice</groupId>
|
||||||
|
|
|
@ -1,4 +1,29 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
|
||||||
|
|
||||||
|
Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
|
||||||
|
====================================================================
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
or more contributor license agreements. See the NOTICE file
|
||||||
|
distributed with this work for additional information
|
||||||
|
regarding copyright ownership. The ASF licenses this file
|
||||||
|
to you under the Apache License, Version 2.0 (the
|
||||||
|
"License"); you may not use this file except in compliance
|
||||||
|
with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing,
|
||||||
|
software distributed under the License is distributed on an
|
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations
|
||||||
|
under the License.
|
||||||
|
====================================================================
|
||||||
|
|
||||||
|
-->
|
||||||
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
|
<metadata xsi:schemaLocation="http://maven.apache.org/METADATA/1.0.0 http://maven.apache.org/xsd/metadata-1.0.0.xsd" xmlns="http://maven.apache.org/METADATA/1.0.0"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||||
<groupId>com.google.code.guice</groupId>
|
<groupId>com.google.code.guice</groupId>
|
||||||
|
|
|
@ -30,6 +30,7 @@ import java.util.Map;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
|
|
||||||
|
import org.jclouds.aws.s3.S3Constants;
|
||||||
import org.testng.annotations.BeforeTest;
|
import org.testng.annotations.BeforeTest;
|
||||||
import org.testng.annotations.Optional;
|
import org.testng.annotations.Optional;
|
||||||
import org.testng.annotations.Parameters;
|
import org.testng.annotations.Parameters;
|
||||||
|
@ -46,7 +47,7 @@ public class AmazonPerformance extends BasePerformance {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@BeforeTest
|
@BeforeTest
|
||||||
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" })
|
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
|
||||||
protected void setUpClient(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception {
|
protected void setUpClient(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey) throws Exception {
|
||||||
super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey);
|
super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey);
|
||||||
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey,
|
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey,
|
||||||
|
|
|
@ -34,6 +34,7 @@ import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import org.jclouds.aws.s3.S3Constants;
|
||||||
import org.jclouds.aws.s3.S3IntegrationTest;
|
import org.jclouds.aws.s3.S3IntegrationTest;
|
||||||
import org.jclouds.aws.s3.domain.S3Bucket;
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
import org.testng.annotations.AfterTest;
|
import org.testng.annotations.AfterTest;
|
||||||
|
@ -79,7 +80,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@BeforeTest
|
@BeforeTest
|
||||||
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" })
|
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
|
||||||
protected void setUpClient(@Optional String AWSAccessKeyId,
|
protected void setUpClient(@Optional String AWSAccessKeyId,
|
||||||
@Optional String AWSSecretAccessKey) throws Exception {
|
@Optional String AWSSecretAccessKey) throws Exception {
|
||||||
super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey);
|
super.setUpClient(AWSAccessKeyId, AWSSecretAccessKey);
|
||||||
|
|
|
@ -67,6 +67,11 @@
|
||||||
<artifactId>bcprov-jdk15</artifactId>
|
<artifactId>bcprov-jdk15</artifactId>
|
||||||
<version>140</version>
|
<version>140</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>eu.medsea.mimeutil</groupId>
|
||||||
|
<artifactId>mime-util</artifactId>
|
||||||
|
<version>1.3</version>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
<build>
|
<build>
|
||||||
<plugins>
|
<plugins>
|
||||||
|
|
|
@ -23,20 +23,18 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3;
|
package org.jclouds.aws.s3;
|
||||||
|
|
||||||
import org.jclouds.aws.s3.domain.S3Object;
|
|
||||||
import org.jclouds.aws.s3.domain.S3Bucket;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.Closeable;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.Future;
|
import java.util.concurrent.Future;
|
||||||
|
|
||||||
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
|
import org.jclouds.aws.s3.domain.S3Object;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* // TODO: Adrian: Document this!
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
public interface S3Connection extends Closeable {
|
public interface S3Connection {
|
||||||
Future<S3Object> getObject(S3Bucket s3Bucket, String key);
|
Future<S3Object> getObject(S3Bucket s3Bucket, String key);
|
||||||
|
|
||||||
Future<S3Object> headObject(S3Bucket s3Bucket, String key);
|
Future<S3Object> headObject(S3Bucket s3Bucket, String key);
|
||||||
|
@ -56,6 +54,4 @@ public interface S3Connection extends Closeable {
|
||||||
Future<S3Bucket> getBucket(S3Bucket s3Bucket);
|
Future<S3Bucket> getBucket(S3Bucket s3Bucket);
|
||||||
|
|
||||||
Future<List<S3Bucket>> getBuckets();
|
Future<List<S3Bucket>> getBuckets();
|
||||||
|
|
||||||
public void close() throws IOException;
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,116 +0,0 @@
|
||||||
/**
|
|
||||||
*
|
|
||||||
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
|
||||||
*
|
|
||||||
* ====================================================================
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing,
|
|
||||||
* software distributed under the License is distributed on an
|
|
||||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
* KIND, either express or implied. See the License for the
|
|
||||||
* specific language governing permissions and limitations
|
|
||||||
* under the License.
|
|
||||||
* ====================================================================
|
|
||||||
*/
|
|
||||||
package org.jclouds.aws.s3;
|
|
||||||
|
|
||||||
import com.google.inject.AbstractModule;
|
|
||||||
import com.google.inject.Guice;
|
|
||||||
import com.google.inject.Injector;
|
|
||||||
import com.google.inject.Module;
|
|
||||||
import com.google.inject.name.Names;
|
|
||||||
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* // TODO: Adrian: Document return getConnection!
|
|
||||||
*
|
|
||||||
* @author Adrian Cole
|
|
||||||
*/
|
|
||||||
public class S3ConnectionFactory {
|
|
||||||
public static final Properties DEFAULT_PROPERTIES;
|
|
||||||
|
|
||||||
static {
|
|
||||||
DEFAULT_PROPERTIES = new Properties();
|
|
||||||
DEFAULT_PROPERTIES.setProperty("jclouds.http.address", "s3.amazonaws.com");
|
|
||||||
DEFAULT_PROPERTIES.setProperty("jclouds.http.port", "443");
|
|
||||||
DEFAULT_PROPERTIES.setProperty("jclouds.http.secure", "true");
|
|
||||||
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.max_connection_reuse", "75");
|
|
||||||
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.max_session_failures", "2");
|
|
||||||
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.request_invoker_threads", "1");
|
|
||||||
DEFAULT_PROPERTIES.setProperty("jclouds.http.pool.io_worker_threads", "2");
|
|
||||||
DEFAULT_PROPERTIES.setProperty("jclouds.pool.max_connections", "12");
|
|
||||||
}
|
|
||||||
|
|
||||||
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey) {
|
|
||||||
Properties properties = new Properties(DEFAULT_PROPERTIES);
|
|
||||||
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
|
|
||||||
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
|
|
||||||
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure) {
|
|
||||||
Properties properties = new Properties(DEFAULT_PROPERTIES);
|
|
||||||
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
|
|
||||||
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
|
|
||||||
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
|
|
||||||
if (!isSecure)
|
|
||||||
properties.setProperty("jclouds.http.port", "80");
|
|
||||||
|
|
||||||
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure,
|
|
||||||
String server) {
|
|
||||||
Properties properties = new Properties(DEFAULT_PROPERTIES);
|
|
||||||
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
|
|
||||||
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
|
|
||||||
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
|
|
||||||
properties.setProperty("jclouds.http.address", server);
|
|
||||||
if (!isSecure)
|
|
||||||
properties.setProperty("jclouds.http.port", "80");
|
|
||||||
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static S3Connection getConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure,
|
|
||||||
String server, int port) {
|
|
||||||
Properties properties = new Properties(DEFAULT_PROPERTIES);
|
|
||||||
properties.setProperty("jclouds.aws.accesskeyid", awsAccessKeyId);
|
|
||||||
properties.setProperty("jclouds.aws.secretaccesskey", awsSecretAccessKey);
|
|
||||||
properties.setProperty("jclouds.http.secure", Boolean.toString(isSecure));
|
|
||||||
properties.setProperty("jclouds.http.address", server);
|
|
||||||
properties.setProperty("jclouds.http.port", port + "");
|
|
||||||
return getConnection(properties, new JavaUrlHttpFutureCommandClientModule());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new interface to interact with S3 with the given credential and connection
|
|
||||||
* parameters
|
|
||||||
*/
|
|
||||||
public static synchronized S3Connection getConnection(final Properties properties, Module httpModule) {
|
|
||||||
return getInjector(properties, httpModule).getInstance(S3Connection.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new interface to interact with S3 with the given credential and connection
|
|
||||||
* parameters
|
|
||||||
*/
|
|
||||||
public static synchronized Injector getInjector(final Properties properties, Module httpModule) {
|
|
||||||
return Guice.createInjector(new AbstractModule() {
|
|
||||||
@Override
|
|
||||||
protected void configure() {
|
|
||||||
Names.bindProperties(binder(), properties);
|
|
||||||
}
|
|
||||||
}, httpModule, new S3ConnectionModule());
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -23,6 +23,7 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3;
|
package org.jclouds.aws.s3;
|
||||||
|
|
||||||
|
import org.jclouds.command.pool.PoolConstants;
|
||||||
import org.jclouds.http.HttpConstants;
|
import org.jclouds.http.HttpConstants;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -30,6 +31,8 @@ import org.jclouds.http.HttpConstants;
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
public class S3Constants extends HttpConstants {
|
public interface S3Constants extends HttpConstants, PoolConstants {
|
||||||
public static final String AUTH = "Authorization";
|
public static final String AUTH = "Authorization";
|
||||||
|
public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey";
|
||||||
|
public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid";
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,49 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
*
|
||||||
|
* ====================================================================
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
* ====================================================================
|
||||||
|
*/
|
||||||
|
package org.jclouds.aws.s3;
|
||||||
|
|
||||||
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
|
|
||||||
|
public interface S3Context {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return a connection to S3
|
||||||
|
*/
|
||||||
|
S3Connection getConnection();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a <code>Map<String,InputStream</code> view of the specified
|
||||||
|
* bucket.
|
||||||
|
*
|
||||||
|
* @param bucket
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
S3ObjectMap createMapView(S3Bucket bucket);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Closes all connections to S3.
|
||||||
|
*/
|
||||||
|
void close();
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,148 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
*
|
||||||
|
* ====================================================================
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
* ====================================================================
|
||||||
|
*/
|
||||||
|
package org.jclouds.aws.s3;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.jclouds.aws.s3.config.S3ContextModule;
|
||||||
|
import org.jclouds.http.HttpFutureCommandClient;
|
||||||
|
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
|
||||||
|
|
||||||
|
import com.google.inject.AbstractModule;
|
||||||
|
import com.google.inject.Guice;
|
||||||
|
import com.google.inject.Injector;
|
||||||
|
import com.google.inject.Module;
|
||||||
|
import com.google.inject.name.Names;
|
||||||
|
import static org.jclouds.aws.s3.S3Constants.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* // TODO: Adrian: Document return getConnection!
|
||||||
|
*
|
||||||
|
* @author Adrian Cole
|
||||||
|
*/
|
||||||
|
public class S3ContextFactory {
|
||||||
|
|
||||||
|
public static final Properties DEFAULT_PROPERTIES;
|
||||||
|
|
||||||
|
static {
|
||||||
|
DEFAULT_PROPERTIES = new Properties();
|
||||||
|
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_ADDRESS,
|
||||||
|
"s3.amazonaws.com");
|
||||||
|
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_PORT, "443");
|
||||||
|
DEFAULT_PROPERTIES.setProperty(PROPERTY_HTTP_SECURE, "true");
|
||||||
|
DEFAULT_PROPERTIES
|
||||||
|
.setProperty(PROPERTY_POOL_MAX_CONNECTION_REUSE, "75");
|
||||||
|
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_MAX_SESSION_FAILURES, "2");
|
||||||
|
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_REQUEST_INVOKER_THREADS,
|
||||||
|
"1");
|
||||||
|
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_IO_WORKER_THREADS, "2");
|
||||||
|
DEFAULT_PROPERTIES.setProperty(PROPERTY_POOL_MAX_CONNECTIONS, "12");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static S3Context createS3Context(String awsAccessKeyId,
|
||||||
|
String awsSecretAccessKey, Module... modules) {
|
||||||
|
Properties properties = new Properties(DEFAULT_PROPERTIES);
|
||||||
|
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
|
||||||
|
properties
|
||||||
|
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
|
||||||
|
return createS3Context(properties, modules);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static S3Context createS3Context(String awsAccessKeyId,
|
||||||
|
String awsSecretAccessKey, boolean isSecure, Module... modules) {
|
||||||
|
Properties properties = new Properties(DEFAULT_PROPERTIES);
|
||||||
|
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
|
||||||
|
properties
|
||||||
|
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
|
||||||
|
properties
|
||||||
|
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
|
||||||
|
if (!isSecure)
|
||||||
|
properties.setProperty(PROPERTY_HTTP_PORT, "80");
|
||||||
|
return createS3Context(properties, modules);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static S3Context createS3Context(String awsAccessKeyId,
|
||||||
|
String awsSecretAccessKey, boolean isSecure, String server,
|
||||||
|
Module... modules) {
|
||||||
|
Properties properties = new Properties(DEFAULT_PROPERTIES);
|
||||||
|
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
|
||||||
|
properties
|
||||||
|
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
|
||||||
|
properties
|
||||||
|
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
|
||||||
|
properties.setProperty(PROPERTY_HTTP_ADDRESS, server);
|
||||||
|
if (!isSecure)
|
||||||
|
properties.setProperty(PROPERTY_HTTP_PORT, "80");
|
||||||
|
return createS3Context(properties, modules);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static S3Context createS3Context(String awsAccessKeyId,
|
||||||
|
String awsSecretAccessKey, boolean isSecure, String server,
|
||||||
|
int port, Module... modules) {
|
||||||
|
Properties properties = new Properties(DEFAULT_PROPERTIES);
|
||||||
|
properties.setProperty(PROPERTY_AWS_ACCESSKEYID, awsAccessKeyId);
|
||||||
|
properties
|
||||||
|
.setProperty(PROPERTY_AWS_SECRETACCESSKEY, awsSecretAccessKey);
|
||||||
|
properties
|
||||||
|
.setProperty(PROPERTY_HTTP_SECURE, Boolean.toString(isSecure));
|
||||||
|
properties.setProperty(PROPERTY_HTTP_ADDRESS, server);
|
||||||
|
properties.setProperty(PROPERTY_HTTP_PORT, port + "");
|
||||||
|
return createS3Context(properties, modules);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static S3Context createS3Context(Properties properties,
|
||||||
|
Module... modules) {
|
||||||
|
return getInjector(properties, modules).getInstance(S3Context.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bind the given properties and install the list of modules. If no modules
|
||||||
|
* are specified, install the default
|
||||||
|
* {@link JavaUrlHttpFutureCommandClientModule}
|
||||||
|
*
|
||||||
|
* @param properties
|
||||||
|
* - contains constants used by jclouds
|
||||||
|
* {@link #DEFAULT_PROPERTIES}
|
||||||
|
* @param httpModules
|
||||||
|
* - modules that must bind {@link HttpFutureCommandClient} if
|
||||||
|
* specified
|
||||||
|
* */
|
||||||
|
public static Injector getInjector(final Properties properties,
|
||||||
|
Module... httpModules) {
|
||||||
|
final List<? extends Module> modules = httpModules.length != 0 ? Arrays
|
||||||
|
.asList(httpModules) : Collections
|
||||||
|
.singletonList(new JavaUrlHttpFutureCommandClientModule());
|
||||||
|
return Guice.createInjector(new AbstractModule() {
|
||||||
|
@Override
|
||||||
|
protected void configure() {
|
||||||
|
Names.bindProperties(binder(), properties);
|
||||||
|
for (Module module : modules)
|
||||||
|
install(module);
|
||||||
|
}
|
||||||
|
}, new S3ContextModule());
|
||||||
|
}
|
||||||
|
}
|
|
@ -23,138 +23,32 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3;
|
package org.jclouds.aws.s3;
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import java.io.File;
|
||||||
import org.jclouds.Logger;
|
import java.io.InputStream;
|
||||||
import org.jclouds.aws.s3.domain.S3Object;
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
import org.jclouds.aws.s3.domain.S3Bucket;
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
|
import org.jclouds.aws.s3.domain.S3Object;
|
||||||
|
|
||||||
import java.util.*;
|
public interface S3ObjectMap extends Map<String, InputStream> {
|
||||||
import java.util.concurrent.ConcurrentMap;
|
|
||||||
import java.util.concurrent.Future;
|
|
||||||
|
|
||||||
/**
|
InputStream putString(String key, String value);
|
||||||
* // TODO: Adrian: Document this!
|
|
||||||
*
|
|
||||||
* @author Adrian Cole
|
|
||||||
*/
|
|
||||||
public class S3ObjectMap implements ConcurrentMap<String, Object> {
|
|
||||||
private Logger logger;
|
|
||||||
private S3Connection connection;
|
|
||||||
private S3Bucket bucket;
|
|
||||||
private S3Utils utils;
|
|
||||||
|
|
||||||
@Inject
|
InputStream putFile(String key, File value);
|
||||||
public S3ObjectMap(java.util.logging.Logger logger, S3Connection connection, S3Bucket bucket, S3Utils utils) {
|
|
||||||
this.logger = new Logger(logger);
|
|
||||||
this.connection = connection;
|
|
||||||
this.bucket = bucket;
|
|
||||||
this.utils = utils;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
InputStream putBytes(String key, byte[] value);
|
||||||
|
|
||||||
public Object putIfAbsent(String s, Object o) {
|
void putAllStrings(Map<? extends String, ? extends String> map);
|
||||||
return null; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean remove(Object o, Object o1) {
|
void putAllBytes(Map<? extends String, ? extends byte[]> map);
|
||||||
return false; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean replace(String s, Object o, Object o1) {
|
void putAllFiles(Map<? extends String, ? extends File> map);
|
||||||
return false; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object replace(String s, Object o) {
|
InputStream put(S3Object object);
|
||||||
return null; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public int size() {
|
void putAll(Set<S3Object> objects);
|
||||||
try {
|
|
||||||
bucket = connection.getBucket(bucket).get();
|
|
||||||
return bucket.getContents().size();
|
|
||||||
} catch (Exception e) {
|
|
||||||
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
|
|
||||||
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isEmpty() {
|
S3Bucket getBucket();
|
||||||
return false; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean containsKey(Object o) {
|
|
||||||
return false; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean containsValue(Object o) {
|
|
||||||
return false; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object get(Object o) {
|
|
||||||
try {
|
|
||||||
return connection.getObject(bucket, o.toString()).get();
|
|
||||||
} catch (Exception e) {
|
|
||||||
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
|
|
||||||
throw new S3RuntimeException(String.format("Error geting object %1s:%2s", bucket, o), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object put(String s, Object o) {
|
|
||||||
S3Object object = new S3Object();
|
|
||||||
try {
|
|
||||||
object.setKey(s);
|
|
||||||
object.setContent(o);
|
|
||||||
return connection.addObject(bucket, object).get();
|
|
||||||
} catch (Exception e) {
|
|
||||||
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
|
|
||||||
throw new S3RuntimeException(String.format("Error adding object %1s:%2s", bucket, object), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object remove(Object o) {
|
|
||||||
return null; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public void putAll(Map<? extends String, ? extends Object> map) {
|
|
||||||
// TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
private class S3RuntimeException extends RuntimeException {
|
|
||||||
public S3RuntimeException(String s) {
|
|
||||||
super(s); // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public S3RuntimeException(String s, Throwable throwable) {
|
|
||||||
super(s, throwable); // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void clear() {
|
|
||||||
try {
|
|
||||||
bucket = connection.getBucket(bucket).get();
|
|
||||||
List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
|
|
||||||
for (S3Object object : bucket.getContents()) {
|
|
||||||
deletes.add(connection.deleteObject(bucket, object.getKey()));
|
|
||||||
}
|
|
||||||
for (Future<Boolean> isdeleted : deletes)
|
|
||||||
if (!isdeleted.get()) {
|
|
||||||
throw new S3RuntimeException("failed to delete entry");
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
S3Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
|
|
||||||
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public Set<String> keySet() {
|
|
||||||
return null; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public Collection<Object> values() {
|
|
||||||
return null; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
|
|
||||||
public Set<Entry<String, Object>> entrySet() {
|
|
||||||
return null; // TODO: Adrian: Customise this generated block
|
|
||||||
}
|
|
||||||
}
|
}
|
|
@ -23,6 +23,14 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3;
|
package org.jclouds.aws.s3;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.security.InvalidKeyException;
|
||||||
|
import java.security.NoSuchAlgorithmException;
|
||||||
|
import java.security.NoSuchProviderException;
|
||||||
|
|
||||||
|
import org.bouncycastle.crypto.digests.MD5Digest;
|
||||||
import org.bouncycastle.crypto.digests.SHA1Digest;
|
import org.bouncycastle.crypto.digests.SHA1Digest;
|
||||||
import org.bouncycastle.crypto.macs.HMac;
|
import org.bouncycastle.crypto.macs.HMac;
|
||||||
import org.bouncycastle.crypto.params.KeyParameter;
|
import org.bouncycastle.crypto.params.KeyParameter;
|
||||||
|
@ -30,16 +38,29 @@ import org.bouncycastle.util.encoders.Base64;
|
||||||
import org.jclouds.Utils;
|
import org.jclouds.Utils;
|
||||||
import org.jclouds.aws.s3.domain.S3Object;
|
import org.jclouds.aws.s3.domain.S3Object;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.security.InvalidKeyException;
|
|
||||||
import java.security.NoSuchAlgorithmException;
|
|
||||||
import java.security.NoSuchProviderException;
|
|
||||||
|
|
||||||
public class S3Utils extends Utils {
|
public class S3Utils extends Utils {
|
||||||
|
|
||||||
|
static final byte[] HEX_CHAR_TABLE = { (byte) '0', (byte) '1', (byte) '2',
|
||||||
|
(byte) '3', (byte) '4', (byte) '5', (byte) '6', (byte) '7',
|
||||||
|
(byte) '8', (byte) '9', (byte) 'a', (byte) 'b', (byte) 'c',
|
||||||
|
(byte) 'd', (byte) 'e', (byte) 'f' };
|
||||||
|
|
||||||
public static String digest(String toEncode, byte[] key) throws NoSuchAlgorithmException, NoSuchProviderException, InvalidKeyException {
|
public static String getHexString(byte[] raw)
|
||||||
|
throws UnsupportedEncodingException {
|
||||||
|
byte[] hex = new byte[2 * raw.length];
|
||||||
|
int index = 0;
|
||||||
|
|
||||||
|
for (byte b : raw) {
|
||||||
|
int v = b & 0xFF;
|
||||||
|
hex[index++] = HEX_CHAR_TABLE[v >>> 4];
|
||||||
|
hex[index++] = HEX_CHAR_TABLE[v & 0xF];
|
||||||
|
}
|
||||||
|
return new String(hex, "ASCII");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String hmacSha1Base64(String toEncode, byte[] key)
|
||||||
|
throws NoSuchAlgorithmException, NoSuchProviderException,
|
||||||
|
InvalidKeyException {
|
||||||
HMac hmac = new HMac(new SHA1Digest());
|
HMac hmac = new HMac(new SHA1Digest());
|
||||||
byte[] resBuf = new byte[hmac.getMacSize()];
|
byte[] resBuf = new byte[hmac.getMacSize()];
|
||||||
byte[] plainBytes = toEncode.getBytes();
|
byte[] plainBytes = toEncode.getBytes();
|
||||||
|
@ -50,8 +71,45 @@ public class S3Utils extends Utils {
|
||||||
return new String(Base64.encode(resBuf));
|
return new String(Base64.encode(resBuf));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String md5Hex(byte [] toEncode)
|
||||||
|
throws NoSuchAlgorithmException, NoSuchProviderException,
|
||||||
|
InvalidKeyException, UnsupportedEncodingException {
|
||||||
|
byte[] resBuf = md5(toEncode);
|
||||||
|
return getHexString(resBuf);
|
||||||
|
}
|
||||||
|
|
||||||
public static String getContentAsStringAndClose(S3Object object) throws IOException {
|
public static String md5Base64(byte [] toEncode)
|
||||||
|
throws NoSuchAlgorithmException, NoSuchProviderException,
|
||||||
|
InvalidKeyException {
|
||||||
|
byte[] resBuf = md5(toEncode);
|
||||||
|
return new String(Base64.encode(resBuf));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] md5(byte[] plainBytes) {
|
||||||
|
MD5Digest md5 = new MD5Digest();
|
||||||
|
byte[] resBuf = new byte[md5.getDigestSize()];
|
||||||
|
md5.update(plainBytes, 0, plainBytes.length);
|
||||||
|
md5.doFinal(resBuf, 0);
|
||||||
|
return resBuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] md5(InputStream toEncode) throws IOException {
|
||||||
|
MD5Digest md5 = new MD5Digest();
|
||||||
|
byte[] resBuf = new byte[md5.getDigestSize()];
|
||||||
|
byte[] buffer = new byte[1024];
|
||||||
|
int numRead = -1;
|
||||||
|
do {
|
||||||
|
numRead = toEncode.read(buffer);
|
||||||
|
if (numRead > 0) {
|
||||||
|
md5.update(buffer, 0, numRead);
|
||||||
|
}
|
||||||
|
} while (numRead != -1);
|
||||||
|
md5.doFinal(resBuf, 0);
|
||||||
|
return resBuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getContentAsStringAndClose(S3Object object)
|
||||||
|
throws IOException {
|
||||||
Object o = object.getContent();
|
Object o = object.getContent();
|
||||||
|
|
||||||
if (o instanceof InputStream) {
|
if (o instanceof InputStream) {
|
||||||
|
@ -61,7 +119,8 @@ public class S3Utils extends Utils {
|
||||||
}
|
}
|
||||||
return returnVal;
|
return returnVal;
|
||||||
} else {
|
} else {
|
||||||
throw new IllegalArgumentException("Object type not supported: " + o.getClass().getName());
|
throw new IllegalArgumentException("Object type not supported: "
|
||||||
|
+ o.getClass().getName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -67,7 +67,10 @@ public class RetrieveObjectCallable extends
|
||||||
object.setLastModified(dateParser
|
object.setLastModified(dateParser
|
||||||
.dateTimeFromHeaderFormat(getResponse()
|
.dateTimeFromHeaderFormat(getResponse()
|
||||||
.getFirstHeaderOrNull("Last-Modified")));
|
.getFirstHeaderOrNull("Last-Modified")));
|
||||||
object.setETag(getResponse().getFirstHeaderOrNull("ETag"));
|
String eTag = getResponse().getFirstHeaderOrNull("ETag");
|
||||||
|
if (eTag != null) {
|
||||||
|
object.setETag(eTag.replaceAll("\"", ""));
|
||||||
|
}
|
||||||
object.setContentType(getResponse().getFirstHeaderOrNull(
|
object.setContentType(getResponse().getFirstHeaderOrNull(
|
||||||
"Content-Type"));
|
"Content-Type"));
|
||||||
object.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull(
|
object.setSize(Long.parseLong(getResponse().getFirstHeaderOrNull(
|
||||||
|
|
|
@ -68,7 +68,7 @@ public class ListBucketHandler extends ParseSax.HandlerWithResult<S3Bucket> {
|
||||||
} else if (qName.equals("LastModified")) {
|
} else if (qName.equals("LastModified")) {
|
||||||
currentObject.setLastModified(dateParser.dateTimeFromXMLFormat(currentText.toString()));
|
currentObject.setLastModified(dateParser.dateTimeFromXMLFormat(currentText.toString()));
|
||||||
} else if (qName.equals("ETag")) {
|
} else if (qName.equals("ETag")) {
|
||||||
currentObject.setETag(currentText.toString());
|
currentObject.setETag(currentText.toString().replaceAll("\"", ""));
|
||||||
} else if (qName.equals("Size")) {
|
} else if (qName.equals("Size")) {
|
||||||
currentObject.setSize(Long.parseLong(currentText.toString()));
|
currentObject.setSize(Long.parseLong(currentText.toString()));
|
||||||
} else if (qName.equals("Owner")) {
|
} else if (qName.equals("Owner")) {
|
||||||
|
|
|
@ -21,39 +21,55 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
* ====================================================================
|
* ====================================================================
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3;
|
package org.jclouds.aws.s3.config;
|
||||||
|
|
||||||
import com.google.inject.AbstractModule;
|
|
||||||
import com.google.inject.Provides;
|
|
||||||
import com.google.inject.Singleton;
|
|
||||||
import com.google.inject.Scopes;
|
|
||||||
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
|
|
||||||
import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader;
|
|
||||||
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
|
|
||||||
import org.jclouds.http.HttpRequestFilter;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.jclouds.aws.s3.S3Connection;
|
||||||
|
import org.jclouds.aws.s3.S3Context;
|
||||||
|
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
|
||||||
|
import org.jclouds.aws.s3.filters.RemoveTransferEncodingHeader;
|
||||||
|
import org.jclouds.aws.s3.filters.RequestAuthorizeSignature;
|
||||||
|
import org.jclouds.aws.s3.internal.GuiceS3Context;
|
||||||
|
import org.jclouds.aws.s3.internal.LiveS3Connection;
|
||||||
|
import org.jclouds.aws.s3.internal.LiveS3ObjectMap;
|
||||||
|
import org.jclouds.aws.s3.internal.GuiceS3Context.S3ObjectMapFactory;
|
||||||
|
import org.jclouds.http.HttpRequestFilter;
|
||||||
|
|
||||||
|
import com.google.inject.AbstractModule;
|
||||||
|
import com.google.inject.Provides;
|
||||||
|
import com.google.inject.Scopes;
|
||||||
|
import com.google.inject.Singleton;
|
||||||
|
import com.google.inject.assistedinject.FactoryProvider;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* // TODO: Adrian: Document this!
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
public class S3ConnectionModule extends AbstractModule {
|
public class S3ContextModule extends AbstractModule {
|
||||||
|
|
||||||
|
@Override
|
||||||
protected void configure() {
|
protected void configure() {
|
||||||
install(new S3CommandsModule());
|
install(new S3CommandsModule());
|
||||||
bind(S3Connection.class).to(LiveS3Connection.class).in(Scopes.SINGLETON);
|
bind(S3Connection.class).to(LiveS3Connection.class)
|
||||||
|
.in(Scopes.SINGLETON);
|
||||||
|
bind(GuiceS3Context.S3ObjectMapFactory.class).toProvider(
|
||||||
|
FactoryProvider.newFactory(
|
||||||
|
GuiceS3Context.S3ObjectMapFactory.class,
|
||||||
|
LiveS3ObjectMap.class));
|
||||||
|
bind(S3Context.class).to(GuiceS3Context.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Singleton
|
@Singleton
|
||||||
List<HttpRequestFilter> provideRequestFilters(RemoveTransferEncodingHeader removTransferEncodingHeader, RequestAuthorizeSignature requestAuthorizeSignature) {
|
List<HttpRequestFilter> provideRequestFilters(
|
||||||
|
RemoveTransferEncodingHeader removTransferEncodingHeader,
|
||||||
|
RequestAuthorizeSignature requestAuthorizeSignature) {
|
||||||
List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>();
|
List<HttpRequestFilter> filters = new ArrayList<HttpRequestFilter>();
|
||||||
filters.add(removTransferEncodingHeader);
|
filters.add(removTransferEncodingHeader);
|
||||||
filters.add(requestAuthorizeSignature);
|
filters.add(requestAuthorizeSignature);
|
||||||
return filters;
|
return filters;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -99,8 +99,8 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
public RequestAuthorizeSignature(
|
public RequestAuthorizeSignature(
|
||||||
@Named("jclouds.aws.accesskeyid") String accessKey,
|
@Named(S3Constants.PROPERTY_AWS_ACCESSKEYID) String accessKey,
|
||||||
@Named("jclouds.aws.secretaccesskey") String secretKey,
|
@Named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY) String secretKey,
|
||||||
DateService dateService) {
|
DateService dateService) {
|
||||||
this.accessKey = accessKey;
|
this.accessKey = accessKey;
|
||||||
this.secretKey = secretKey;
|
this.secretKey = secretKey;
|
||||||
|
@ -142,7 +142,7 @@ public class RequestAuthorizeSignature implements HttpRequestFilter {
|
||||||
toSign.append(request.getUri());
|
toSign.append(request.getUri());
|
||||||
String signature;
|
String signature;
|
||||||
try {
|
try {
|
||||||
signature = S3Utils.digest(toSign.toString(), secretKey.getBytes());
|
signature = S3Utils.hmacSha1Base64(toSign.toString(), secretKey.getBytes());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new HttpException("error signing request", e);
|
throw new HttpException("error signing request", e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,88 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
*
|
||||||
|
* ====================================================================
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
* ====================================================================
|
||||||
|
*/
|
||||||
|
package org.jclouds.aws.s3.internal;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.jclouds.Logger;
|
||||||
|
import org.jclouds.aws.s3.S3Connection;
|
||||||
|
import org.jclouds.aws.s3.S3Context;
|
||||||
|
import org.jclouds.aws.s3.S3ObjectMap;
|
||||||
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
|
import org.jclouds.lifecycle.Closer;
|
||||||
|
|
||||||
|
import com.google.inject.Inject;
|
||||||
|
import com.google.inject.Injector;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* // TODO: Adrian: Document return getConnection!
|
||||||
|
*
|
||||||
|
* @author Adrian Cole
|
||||||
|
*/
|
||||||
|
public class GuiceS3Context implements S3Context {
|
||||||
|
public interface S3ObjectMapFactory {
|
||||||
|
S3ObjectMap createMapView(S3Bucket bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
private final Logger logger;
|
||||||
|
private final Injector injector;
|
||||||
|
private final S3ObjectMapFactory s3ObjectMapFactory;
|
||||||
|
private final Closer closer;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
private GuiceS3Context(java.util.logging.Logger logger, Injector injector,
|
||||||
|
Closer closer, S3ObjectMapFactory s3ObjectMapFactory) {
|
||||||
|
this.logger = new Logger(logger);
|
||||||
|
this.injector = injector;
|
||||||
|
this.s3ObjectMapFactory = s3ObjectMapFactory;
|
||||||
|
this.closer = closer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*/
|
||||||
|
public S3Connection getConnection() {
|
||||||
|
return injector.getInstance(S3Connection.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*/
|
||||||
|
public S3ObjectMap createMapView(S3Bucket bucket) {
|
||||||
|
getConnection().createBucketIfNotExists(bucket);
|
||||||
|
return s3ObjectMapFactory.createMapView(bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*/
|
||||||
|
public void close() {
|
||||||
|
try {
|
||||||
|
closer.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error(e, "error closing content");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -21,12 +21,12 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
* ====================================================================
|
* ====================================================================
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3;
|
package org.jclouds.aws.s3.internal;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.Future;
|
import java.util.concurrent.Future;
|
||||||
|
|
||||||
|
import org.jclouds.aws.s3.S3Connection;
|
||||||
import org.jclouds.aws.s3.commands.CopyObject;
|
import org.jclouds.aws.s3.commands.CopyObject;
|
||||||
import org.jclouds.aws.s3.commands.DeleteBucket;
|
import org.jclouds.aws.s3.commands.DeleteBucket;
|
||||||
import org.jclouds.aws.s3.commands.DeleteObject;
|
import org.jclouds.aws.s3.commands.DeleteObject;
|
||||||
|
@ -127,9 +127,4 @@ public class LiveS3Connection implements S3Connection {
|
||||||
client.submit(listRequest);
|
client.submit(listRequest);
|
||||||
return listRequest;
|
return listRequest;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close() throws IOException {
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
|
@ -0,0 +1,433 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Copyright (C) 2009 Adrian Cole <adriancole@jclouds.org>
|
||||||
|
*
|
||||||
|
* ====================================================================
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
* ====================================================================
|
||||||
|
*/
|
||||||
|
package org.jclouds.aws.s3.internal;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.ExecutionException;
|
||||||
|
import java.util.concurrent.Future;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.jclouds.Utils;
|
||||||
|
import org.jclouds.aws.s3.S3Connection;
|
||||||
|
import org.jclouds.aws.s3.S3ObjectMap;
|
||||||
|
import org.jclouds.aws.s3.S3Utils;
|
||||||
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
|
import org.jclouds.aws.s3.domain.S3Object;
|
||||||
|
|
||||||
|
import com.google.inject.Inject;
|
||||||
|
import com.google.inject.assistedinject.Assisted;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Map representation of a live connection to S3.
|
||||||
|
*
|
||||||
|
* @author Adrian Cole
|
||||||
|
*/
|
||||||
|
public class LiveS3ObjectMap implements S3ObjectMap {
|
||||||
|
|
||||||
|
private final S3Connection connection;
|
||||||
|
private final S3Bucket bucket;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
public LiveS3ObjectMap(S3Connection connection, @Assisted S3Bucket bucket) {
|
||||||
|
this.connection = connection;
|
||||||
|
this.bucket = bucket;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#size()
|
||||||
|
*/
|
||||||
|
public int size() {
|
||||||
|
try {
|
||||||
|
return refreshBucket().getContents().size();
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#get(java.lang.Object)
|
||||||
|
*/
|
||||||
|
public InputStream get(Object o) {
|
||||||
|
try {
|
||||||
|
return (InputStream) (connection.getObject(bucket, o.toString())
|
||||||
|
.get()).getContent();
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException(String.format(
|
||||||
|
"Error geting object %1s:%2s", bucket, o), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#remove(java.lang.Object)
|
||||||
|
*/
|
||||||
|
public InputStream remove(Object o) {
|
||||||
|
InputStream old = get(o);
|
||||||
|
try {
|
||||||
|
connection.deleteObject(bucket, o.toString()).get();
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException(String.format(
|
||||||
|
"Error removing object %1s:%2s", bucket, o), e);
|
||||||
|
}
|
||||||
|
return old;
|
||||||
|
}
|
||||||
|
|
||||||
|
public class S3RuntimeException extends RuntimeException {
|
||||||
|
S3RuntimeException(String s) {
|
||||||
|
super(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
public S3RuntimeException(String s, Throwable throwable) {
|
||||||
|
super(s, throwable);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#clear()
|
||||||
|
*/
|
||||||
|
public void clear() {
|
||||||
|
try {
|
||||||
|
List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
|
||||||
|
for (String key : keySet()) {
|
||||||
|
deletes.add(connection.deleteObject(bucket, key));
|
||||||
|
}
|
||||||
|
for (Future<Boolean> isdeleted : deletes)
|
||||||
|
if (!isdeleted.get()) {
|
||||||
|
throw new S3RuntimeException("failed to delete entry");
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException("Error clearing bucket" + bucket, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private S3Bucket refreshBucket() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
|
return connection.getBucket(bucket).get();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#keySet()
|
||||||
|
*/
|
||||||
|
public Set<String> keySet() {
|
||||||
|
try {
|
||||||
|
Set<String> keys = new HashSet<String>();
|
||||||
|
for (S3Object object : refreshBucket().getContents())
|
||||||
|
keys.add(object.getKey());
|
||||||
|
return keys;
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException("Error getting keys in bucket: "
|
||||||
|
+ bucket, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#values()
|
||||||
|
*/
|
||||||
|
public Collection<InputStream> values() {
|
||||||
|
Collection<InputStream> values = new LinkedList<InputStream>();
|
||||||
|
Set<Future<S3Object>> futureObjects = new HashSet<Future<S3Object>>();
|
||||||
|
for (String key : keySet()) {
|
||||||
|
futureObjects.add(connection.getObject(bucket, key));
|
||||||
|
}
|
||||||
|
for (Future<S3Object> futureObject : futureObjects) {
|
||||||
|
S3Object object = null;
|
||||||
|
try {
|
||||||
|
object = futureObject.get();
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException(String.format(
|
||||||
|
"Error getting value from bucket %1s:%2s", bucket,
|
||||||
|
object != null ? object.getKey() : "unknown"), e);
|
||||||
|
}
|
||||||
|
System.err.printf("key: %1s, MD5: %2s", object.getKey(), object
|
||||||
|
.getContentMD5());
|
||||||
|
values.add((InputStream) object.getContent());
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#entrySet()
|
||||||
|
*/
|
||||||
|
public Set<Map.Entry<String, InputStream>> entrySet() {
|
||||||
|
Set<Map.Entry<String, InputStream>> entrySet = new HashSet<Map.Entry<String, InputStream>>();
|
||||||
|
for (String key : keySet()) {
|
||||||
|
Map.Entry<String, InputStream> entry = new Entry(key, get(key));
|
||||||
|
entrySet.add(entry);
|
||||||
|
}
|
||||||
|
return entrySet;
|
||||||
|
}
|
||||||
|
|
||||||
|
public class Entry implements java.util.Map.Entry<String, InputStream> {
|
||||||
|
|
||||||
|
private InputStream value;
|
||||||
|
private String key;
|
||||||
|
|
||||||
|
Entry(String key, InputStream value) {
|
||||||
|
this.key = key;
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getKey() {
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
|
||||||
|
public InputStream getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public InputStream setValue(InputStream value) {
|
||||||
|
return put(key, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#containsKey(java.lang.Object)
|
||||||
|
*/
|
||||||
|
public boolean containsKey(Object key) {
|
||||||
|
try {
|
||||||
|
return connection.headObject(bucket, key.toString()).get() != S3Object.NOT_FOUND;
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException(String.format(
|
||||||
|
"Error searching for %1s:%2s", bucket, key), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#containsValue(java.lang.Object)
|
||||||
|
*/
|
||||||
|
public boolean containsValue(Object value) {
|
||||||
|
|
||||||
|
try {
|
||||||
|
byte[] md5;
|
||||||
|
|
||||||
|
if (value instanceof InputStream) {
|
||||||
|
md5 = S3Utils.md5((InputStream) value);
|
||||||
|
} else if (value instanceof byte[]) {
|
||||||
|
md5 = S3Utils.md5((byte[]) value);
|
||||||
|
} else if (value instanceof String) {
|
||||||
|
md5 = S3Utils.md5(((String) value).getBytes());
|
||||||
|
} else if (value instanceof File) {
|
||||||
|
md5 = S3Utils.md5(new FileInputStream((File) value));
|
||||||
|
} else {
|
||||||
|
throw new IllegalArgumentException("unsupported value type: "
|
||||||
|
+ value.getClass());
|
||||||
|
}
|
||||||
|
String eTagOfValue = S3Utils.getHexString(md5);
|
||||||
|
|
||||||
|
for (S3Object object : refreshBucket().getContents()) {
|
||||||
|
if (object.getETag().equals(eTagOfValue))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException(String.format(
|
||||||
|
"Error searching for ETAG of value: [%2s] in bucket:%1s",
|
||||||
|
bucket, value), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMapi#isEmpty()
|
||||||
|
*/
|
||||||
|
public boolean isEmpty() {
|
||||||
|
return keySet().size() == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private InputStream putInternal(String s, Object o) {
|
||||||
|
S3Object object = new S3Object();
|
||||||
|
try {
|
||||||
|
|
||||||
|
InputStream returnVal = containsKey(s) ? get(s) : null;
|
||||||
|
object.setKey(s);
|
||||||
|
object.setContent(o);
|
||||||
|
setSizeIfContentIsInputStream(object);
|
||||||
|
connection.addObject(bucket, object).get();
|
||||||
|
return returnVal;
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException(String.format(
|
||||||
|
"Error adding object %1s:%2s", bucket, object), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMap#putAll(java.util.Map)
|
||||||
|
*/
|
||||||
|
public void putAll(Map<? extends String, ? extends InputStream> map) {
|
||||||
|
putAllInternal(map);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void putAllBytes(Map<? extends String, ? extends byte[]> map) {
|
||||||
|
putAllInternal(map);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void putAllFiles(Map<? extends String, ? extends File> map) {
|
||||||
|
putAllInternal(map);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void putAllStrings(Map<? extends String, ? extends String> map) {
|
||||||
|
putAllInternal(map);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void putAllInternal(Map<? extends String, ? extends Object> map) {
|
||||||
|
try {
|
||||||
|
List<Future<String>> puts = new ArrayList<Future<String>>();
|
||||||
|
for (String key : map.keySet()) {
|
||||||
|
S3Object object = new S3Object();
|
||||||
|
object.setKey(key);
|
||||||
|
object.setContent(map.get(key));
|
||||||
|
setSizeIfContentIsInputStream(object);
|
||||||
|
puts.add(connection.addObject(bucket, object));
|
||||||
|
}
|
||||||
|
for (Future<String> put : puts)
|
||||||
|
put.get();// this will throw an exception if there was a problem
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException("Error putting into bucket" + bucket,
|
||||||
|
e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void setSizeIfContentIsInputStream(S3Object object)
|
||||||
|
throws IOException {
|
||||||
|
if (object.getContent() instanceof InputStream) {
|
||||||
|
byte[] buffer = IOUtils.toByteArray((InputStream) object
|
||||||
|
.getContent());
|
||||||
|
object.setSize(buffer.length);
|
||||||
|
object.setContent(new ByteArrayInputStream(buffer));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMap#putString(java.lang.String,
|
||||||
|
* java.lang.String)
|
||||||
|
*/
|
||||||
|
public InputStream putString(String key, String value) {
|
||||||
|
return putInternal(key, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMap#putFile(java.lang.String,
|
||||||
|
* java.io.File)
|
||||||
|
*/
|
||||||
|
public InputStream putFile(String key, File value) {
|
||||||
|
return putInternal(key, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMap#putBytes(java.lang.String, byte[])
|
||||||
|
*/
|
||||||
|
public InputStream putBytes(String key, byte[] value) {
|
||||||
|
return putInternal(key, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (non-Javadoc)
|
||||||
|
*
|
||||||
|
* @see org.jclouds.aws.s3.S3ObjectMap#put(java.lang.String,
|
||||||
|
* java.io.InputStream)
|
||||||
|
*/
|
||||||
|
public InputStream put(String key, InputStream value) {
|
||||||
|
return putInternal(key, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
public S3Bucket getBucket() {
|
||||||
|
return bucket;
|
||||||
|
}
|
||||||
|
|
||||||
|
public InputStream put(S3Object object) {
|
||||||
|
InputStream returnVal = get(object.getKey());
|
||||||
|
try {
|
||||||
|
connection.addObject(bucket, object).get();
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException(String.format(
|
||||||
|
"Error putting object %1s:%2s", bucket, object.getKey()), e);
|
||||||
|
}
|
||||||
|
return returnVal;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void putAll(Set<S3Object> objects) {
|
||||||
|
try {
|
||||||
|
List<Future<String>> puts = new ArrayList<Future<String>>();
|
||||||
|
for (S3Object object : objects) {
|
||||||
|
puts.add(connection.addObject(bucket, object));
|
||||||
|
}
|
||||||
|
for (Future<String> put : puts)
|
||||||
|
put.get();// this will throw an exception if there was a problem
|
||||||
|
} catch (Exception e) {
|
||||||
|
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
|
||||||
|
throw new S3RuntimeException("Error putting into bucket" + bucket,
|
||||||
|
e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -41,6 +41,7 @@ import java.util.logging.Logger;
|
||||||
|
|
||||||
import org.jclouds.aws.s3.domain.S3Bucket;
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
import org.jclouds.aws.s3.domain.S3Object;
|
import org.jclouds.aws.s3.domain.S3Object;
|
||||||
|
import org.jclouds.http.HttpConstants;
|
||||||
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
|
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
|
||||||
import org.testng.annotations.AfterTest;
|
import org.testng.annotations.AfterTest;
|
||||||
import org.testng.annotations.BeforeTest;
|
import org.testng.annotations.BeforeTest;
|
||||||
|
@ -48,7 +49,6 @@ import org.testng.annotations.Optional;
|
||||||
import org.testng.annotations.Parameters;
|
import org.testng.annotations.Parameters;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
import com.google.inject.Injector;
|
|
||||||
import com.google.inject.Module;
|
import com.google.inject.Module;
|
||||||
|
|
||||||
@Test(sequential = true)
|
@Test(sequential = true)
|
||||||
|
@ -90,44 +90,45 @@ public class S3IntegrationTest {
|
||||||
+ "/adriancole.s3.amazons3test.filetestsforadrian/file</StringToSign><AWSAccessKeyId>0101100101001001</AWSAccessKeyId></Error>";
|
+ "/adriancole.s3.amazons3test.filetestsforadrian/file</StringToSign><AWSAccessKeyId>0101100101001001</AWSAccessKeyId></Error>";
|
||||||
String amazonHadAnError = "<Error><Code>InternalError</Code><Message>We encountered an internal error. Please try again.</Message><RequestId>EF6FA7A639CAFF15</RequestId><HostId>tBkX23mIeq2riHsNw2YShupMlZ9+iy3V/uN+lRhqCR4qHTE07ujFeyAUPTowvuH/</HostId></Error>";
|
String amazonHadAnError = "<Error><Code>InternalError</Code><Message>We encountered an internal error. Please try again.</Message><RequestId>EF6FA7A639CAFF15</RequestId><HostId>tBkX23mIeq2riHsNw2YShupMlZ9+iy3V/uN+lRhqCR4qHTE07ujFeyAUPTowvuH/</HostId></Error>";
|
||||||
protected S3Connection client;
|
protected S3Connection client;
|
||||||
Injector i = null;
|
protected S3Context context = null;
|
||||||
|
|
||||||
protected String bucketPrefix = System.getProperty("user.name") + "."
|
protected String bucketPrefix = System.getProperty("user.name") + "."
|
||||||
+ this.getClass().getName();
|
+ this.getClass().getName();
|
||||||
|
|
||||||
private static final String sysAWSAccessKeyId = System
|
private static final String sysAWSAccessKeyId = System
|
||||||
.getProperty("jclouds.aws.accesskeyid");
|
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
|
||||||
private static final String sysAWSSecretAccessKey = System
|
private static final String sysAWSSecretAccessKey = System
|
||||||
.getProperty("jclouds.aws.secretaccesskey");
|
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
|
||||||
|
|
||||||
@BeforeTest
|
@BeforeTest
|
||||||
@Parameters( { "jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" })
|
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID,
|
||||||
|
S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
|
||||||
protected void setUpClient(@Optional String AWSAccessKeyId,
|
protected void setUpClient(@Optional String AWSAccessKeyId,
|
||||||
@Optional String AWSSecretAccessKey) throws Exception {
|
@Optional String AWSSecretAccessKey) throws Exception {
|
||||||
i = createInject(AWSAccessKeyId != null ? AWSAccessKeyId
|
context = createS3Context(AWSAccessKeyId != null ? AWSAccessKeyId
|
||||||
: sysAWSAccessKeyId,
|
: sysAWSAccessKeyId,
|
||||||
AWSSecretAccessKey != null ? AWSSecretAccessKey
|
AWSSecretAccessKey != null ? AWSSecretAccessKey
|
||||||
: sysAWSSecretAccessKey);
|
: sysAWSSecretAccessKey);
|
||||||
client = i.getInstance(LiveS3Connection.class);
|
client = context.getConnection();
|
||||||
deleteEverything();
|
deleteEverything();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected Injector createInject(String AWSAccessKeyId,
|
protected S3Context createS3Context(String AWSAccessKeyId,
|
||||||
String AWSSecretAccessKey) {
|
String AWSSecretAccessKey) {
|
||||||
return S3ConnectionFactory.getInjector(buildS3Properties(
|
return S3ContextFactory.createS3Context(buildS3Properties(
|
||||||
AWSAccessKeyId, AWSSecretAccessKey), createHttpModule());
|
AWSAccessKeyId, AWSSecretAccessKey), createHttpModule());
|
||||||
}
|
}
|
||||||
|
|
||||||
protected Properties buildS3Properties(String AWSAccessKeyId,
|
protected Properties buildS3Properties(String AWSAccessKeyId,
|
||||||
String AWSSecretAccessKey) {
|
String AWSSecretAccessKey) {
|
||||||
Properties properties = new Properties(
|
Properties properties = new Properties(
|
||||||
S3ConnectionFactory.DEFAULT_PROPERTIES);
|
S3ContextFactory.DEFAULT_PROPERTIES);
|
||||||
properties.setProperty("jclouds.aws.accesskeyid", checkNotNull(
|
properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID,
|
||||||
AWSAccessKeyId, "AWSAccessKeyId"));
|
checkNotNull(AWSAccessKeyId, "AWSAccessKeyId"));
|
||||||
properties.setProperty("jclouds.aws.secretaccesskey", checkNotNull(
|
properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY,
|
||||||
AWSSecretAccessKey, "AWSSecretAccessKey"));
|
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey"));
|
||||||
properties.setProperty("jclouds.http.secure", "false");
|
properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false");
|
||||||
properties.setProperty("jclouds.http.port", "80");
|
properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80");
|
||||||
return properties;
|
return properties;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -163,8 +164,8 @@ public class S3IntegrationTest {
|
||||||
@AfterTest
|
@AfterTest
|
||||||
protected void tearDownClient() throws Exception {
|
protected void tearDownClient() throws Exception {
|
||||||
deleteEverything();
|
deleteEverything();
|
||||||
client.close();
|
context.close();
|
||||||
i = null;
|
context = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -23,59 +23,307 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3;
|
package org.jclouds.aws.s3;
|
||||||
|
|
||||||
|
import static org.testng.Assert.assertEquals;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.TreeSet;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
import java.util.concurrent.ExecutionException;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.jclouds.Utils;
|
||||||
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
|
import org.jclouds.aws.s3.domain.S3Object;
|
||||||
import org.testng.annotations.AfterMethod;
|
import org.testng.annotations.AfterMethod;
|
||||||
import org.testng.annotations.BeforeMethod;
|
import org.testng.annotations.BeforeMethod;
|
||||||
|
import org.testng.annotations.Parameters;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
import org.jclouds.aws.s3.domain.S3Bucket;
|
|
||||||
|
|
||||||
import java.util.Map;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import java.util.logging.Logger;
|
import com.google.common.collect.ImmutableSet;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* Tests to cover @{link LiveS3ObjectMap}
|
||||||
*
|
*
|
||||||
* @author Adrian Cole
|
* @author Adrian Cole
|
||||||
*/
|
*/
|
||||||
@Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest")
|
@Test(groups = "unit", sequential = true, testName = "s3.S3ObjectMapTest")
|
||||||
public class S3ObjectMapTest {
|
public class S3ObjectMapTest extends S3IntegrationTest {
|
||||||
private S3Connection connection;
|
|
||||||
private S3Bucket bucket;
|
private S3Bucket bucket;
|
||||||
private Map map;
|
private S3ObjectMap map;
|
||||||
|
|
||||||
|
private Map<String, String> fiveStrings = ImmutableMap.of("one", "apple",
|
||||||
|
"two", "bear", "three", "candy", "four", "dogma", "five", "emma");
|
||||||
|
private Map<String, byte[]> fiveBytes = ImmutableMap.of("one", "apple"
|
||||||
|
.getBytes(), "two", "bear".getBytes(), "three", "candy".getBytes(),
|
||||||
|
"four", "dogma".getBytes(), "five", "emma".getBytes());
|
||||||
|
private Map<String, InputStream> fiveInputs;
|
||||||
|
private Map<String, File> fiveFiles;
|
||||||
|
|
||||||
|
String tmpDirectory;
|
||||||
|
|
||||||
@BeforeMethod
|
@BeforeMethod
|
||||||
public void setupExecutorService() throws Exception {
|
@Parameters( { "basedir" })
|
||||||
connection = new StubS3Connection();
|
protected void setUpTempDir(String basedir) throws InterruptedException,
|
||||||
|
ExecutionException, FileNotFoundException, IOException {
|
||||||
|
tmpDirectory = basedir + File.separator + "target" + File.separator
|
||||||
|
+ "testFiles" + File.separator + getClass().getSimpleName();
|
||||||
|
new File(tmpDirectory).mkdirs();
|
||||||
|
|
||||||
|
fiveFiles = ImmutableMap.of("one", new File(tmpDirectory, "apple"),
|
||||||
|
"two", new File(tmpDirectory, "bear"), "three", new File(
|
||||||
|
tmpDirectory, "candy"), "four", new File(tmpDirectory,
|
||||||
|
"dogma"), "five", new File(tmpDirectory, "emma"));
|
||||||
|
|
||||||
|
for (File file : fiveFiles.values()) {
|
||||||
|
IOUtils.write(file.getName(), new FileOutputStream(file));
|
||||||
|
}
|
||||||
|
|
||||||
|
fiveInputs = ImmutableMap.of("one", IOUtils.toInputStream("apple"),
|
||||||
|
"two", IOUtils.toInputStream("bear"), "three", IOUtils
|
||||||
|
.toInputStream("candy"), "four", IOUtils
|
||||||
|
.toInputStream("dogma"), "five", IOUtils
|
||||||
|
.toInputStream("emma"));
|
||||||
bucket = new S3Bucket();
|
bucket = new S3Bucket();
|
||||||
bucket.setName("mimi");
|
bucket.setName(bucketPrefix + ".mimi");
|
||||||
connection.createBucketIfNotExists(bucket).get();
|
client.createBucketIfNotExists(bucket).get();
|
||||||
map = new S3ObjectMap(Logger.getLogger("test"), connection, bucket, new S3Utils());
|
map = context.createMapView(bucket);
|
||||||
|
map.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterMethod
|
@AfterMethod
|
||||||
public void teardownExecutorService() {
|
public void tearDown() {
|
||||||
|
map.clear();
|
||||||
map = null;
|
map = null;
|
||||||
bucket = null;
|
bucket = null;
|
||||||
connection = null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testClearWhenNothingInMap(){
|
public void testClear() {
|
||||||
|
map.clear();
|
||||||
|
assert map.size() == 0;
|
||||||
|
map.putString("one", "apple");
|
||||||
map.clear();
|
map.clear();
|
||||||
assert map.size() == 0;
|
assert map.size() == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// @Test
|
@Test()
|
||||||
// public void testGetReturnsWhatWasPut(){
|
public void testRemove() throws IOException {
|
||||||
// map.put("hello","goodbye");
|
map.putString("one", "two");
|
||||||
// assert "goodbye".equals(map.get("hello"));
|
InputStream old = map.remove("one");
|
||||||
// assert map.size() == 0;
|
assertEquals(Utils.toStringAndClose(old), "two");
|
||||||
// }
|
old = map.remove("one");
|
||||||
|
assert old == null;
|
||||||
|
old = map.get("one");
|
||||||
|
assert old == null;
|
||||||
|
assertEquals(map.keySet().size(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testKeySet() {
|
||||||
|
assertEquals(map.keySet().size(), 0);
|
||||||
|
map.putString("one", "two");
|
||||||
|
assertEquals(map.keySet(), ImmutableSet.of("one"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testValues() throws IOException {
|
||||||
|
map.putAll(this.fiveInputs);
|
||||||
|
Collection<InputStream> values = map.values();
|
||||||
|
assertEquals(values.size(), 5);
|
||||||
|
Set<String> valuesAsString = new HashSet<String>();
|
||||||
|
for (InputStream stream : values) {
|
||||||
|
valuesAsString.add(Utils.toStringAndClose(stream));
|
||||||
|
}
|
||||||
|
valuesAsString.removeAll(fiveStrings.values());
|
||||||
|
assert valuesAsString.size() == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testEntrySet() throws IOException {
|
||||||
|
map.putAllStrings(this.fiveStrings);
|
||||||
|
Set<Entry<String, InputStream>> entries = map.entrySet();
|
||||||
|
assertEquals(entries.size(), 5);
|
||||||
|
for (Entry<String, InputStream> entry : entries) {
|
||||||
|
assertEquals(IOUtils.toString(entry.getValue()), fiveStrings
|
||||||
|
.get(entry.getKey()));
|
||||||
|
entry.setValue(IOUtils.toInputStream(""));
|
||||||
|
}
|
||||||
|
assertEquals(map.size(), 5);
|
||||||
|
for (InputStream value : map.values()) {
|
||||||
|
assertEquals(IOUtils.toString(value), "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testContainsKey() {
|
||||||
|
assert !map.containsKey("one");
|
||||||
|
map.putString("one", "apple");
|
||||||
|
assert map.containsKey("one");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testContainsStringValue() {
|
||||||
|
map.putString("one", "apple");
|
||||||
|
assert map.containsValue(fiveStrings.get("one"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testContainsFileValue() {
|
||||||
|
map.putString("one", "apple");
|
||||||
|
assert map.containsValue(fiveFiles.get("one"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testContainsInputStreamValue() {
|
||||||
|
map.putString("one", "apple");
|
||||||
|
assert map.containsValue(this.fiveInputs.get("one"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testContainsBytesValue() {
|
||||||
|
map.putString("one", "apple");
|
||||||
|
assert map.containsValue(this.fiveBytes.get("one"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testIsEmpty() {
|
||||||
|
assert map.isEmpty();
|
||||||
|
map.putString("one", "apple");
|
||||||
|
assert !map.isEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testPutAll() {
|
||||||
|
map.putAll(this.fiveInputs);
|
||||||
|
assertEquals(map.size(), 5);
|
||||||
|
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
|
||||||
|
fiveInputs.keySet()));
|
||||||
|
fourLeftRemovingOne();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testPutAllBytes() {
|
||||||
|
map.putAllBytes(this.fiveBytes);
|
||||||
|
assertEquals(map.size(), 5);
|
||||||
|
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
|
||||||
|
fiveBytes.keySet()));
|
||||||
|
fourLeftRemovingOne();
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testClearWhenSomethingInMap(){
|
public void testPutAllFiles() {
|
||||||
map.put("hello","goodbye");
|
map.putAllFiles(this.fiveFiles);
|
||||||
map.clear();
|
assertEquals(map.size(), 5);
|
||||||
assert map.size() == 0;
|
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
|
||||||
|
fiveFiles.keySet()));
|
||||||
|
fourLeftRemovingOne();
|
||||||
|
}
|
||||||
|
|
||||||
|
void fourLeftRemovingOne() {
|
||||||
|
map.remove("one");
|
||||||
|
assertEquals(map.size(), 4);
|
||||||
|
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
|
||||||
|
ImmutableSet.of("two", "three", "four", "five")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testPutAllStrings() {
|
||||||
|
map.putAllStrings(this.fiveStrings);
|
||||||
|
assertEquals(map.size(), 5);
|
||||||
|
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
|
||||||
|
fiveStrings.keySet()));
|
||||||
|
fourLeftRemovingOne();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testPutString() throws IOException {
|
||||||
|
InputStream old = map.putString("one", "apple");
|
||||||
|
getOneReturnsAppleAndOldValueIsNull(old);
|
||||||
|
InputStream apple = map.putString("one", "bear");
|
||||||
|
getOneReturnsBearAndOldValueIsApple(apple);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void getOneReturnsAppleAndOldValueIsNull(InputStream old)
|
||||||
|
throws IOException {
|
||||||
|
assert old == null;
|
||||||
|
assertEquals(Utils.toStringAndClose(map.get("one")), "apple");
|
||||||
|
assert map.size() == 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void getOneReturnsBearAndOldValueIsApple(InputStream oldValue)
|
||||||
|
throws IOException {
|
||||||
|
assertEquals(Utils.toStringAndClose(map.get("one")), "bear");
|
||||||
|
assertEquals(Utils.toStringAndClose(oldValue), "apple");
|
||||||
|
assert map.size() == 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testPutFile() throws IOException {
|
||||||
|
InputStream old = map.putFile("one", fiveFiles.get("one"));
|
||||||
|
getOneReturnsAppleAndOldValueIsNull(old);
|
||||||
|
InputStream apple = map.putFile("one", fiveFiles.get("two"));
|
||||||
|
getOneReturnsBearAndOldValueIsApple(apple);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testPutBytes() throws IOException {
|
||||||
|
InputStream old = map.putBytes("one", "apple".getBytes());
|
||||||
|
getOneReturnsAppleAndOldValueIsNull(old);
|
||||||
|
InputStream apple = map.putBytes("one", "bear".getBytes());
|
||||||
|
getOneReturnsBearAndOldValueIsApple(apple);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testPut() throws IOException {
|
||||||
|
InputStream old = map.put("one", IOUtils.toInputStream("apple"));
|
||||||
|
getOneReturnsAppleAndOldValueIsNull(old);
|
||||||
|
InputStream apple = map.put("one", IOUtils.toInputStream("bear"));
|
||||||
|
getOneReturnsBearAndOldValueIsApple(apple);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void testGetBucket() {
|
||||||
|
assertEquals(map.getBucket().getName(), bucket.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testPutS3Object() throws IOException {
|
||||||
|
S3Object object = new S3Object();
|
||||||
|
object.setKey("one");
|
||||||
|
object.setContent(IOUtils.toInputStream("apple"));
|
||||||
|
object.setSize("apple".getBytes().length);
|
||||||
|
InputStream old = map.put(object);
|
||||||
|
getOneReturnsAppleAndOldValueIsNull(old);
|
||||||
|
object.setContent(IOUtils.toInputStream("bear"));
|
||||||
|
object.setSize("bear".getBytes().length);
|
||||||
|
InputStream apple = map.put(object);
|
||||||
|
getOneReturnsBearAndOldValueIsApple(apple);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testPutAllS3Objects() {
|
||||||
|
Set<S3Object> set = new HashSet<S3Object>();
|
||||||
|
for (String key : fiveInputs.keySet()) {
|
||||||
|
S3Object object = new S3Object();
|
||||||
|
object.setKey(key);
|
||||||
|
object.setContent(fiveInputs.get(key));
|
||||||
|
object.setSize(fiveBytes.get(key).length);
|
||||||
|
set.add(object);
|
||||||
|
}
|
||||||
|
map.putAll(set);
|
||||||
|
assertEquals(map.size(), 5);
|
||||||
|
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
|
||||||
|
fiveInputs.keySet()));
|
||||||
|
fourLeftRemovingOne();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,6 +28,7 @@ import org.jclouds.aws.PerformanceTest;
|
||||||
import org.testng.annotations.DataProvider;
|
import org.testng.annotations.DataProvider;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
import java.security.InvalidKeyException;
|
import java.security.InvalidKeyException;
|
||||||
import java.security.NoSuchAlgorithmException;
|
import java.security.NoSuchAlgorithmException;
|
||||||
import java.security.NoSuchProviderException;
|
import java.security.NoSuchProviderException;
|
||||||
|
@ -51,7 +52,7 @@ public class S3UtilsTest extends PerformanceTest {
|
||||||
String base64Digest) throws NoSuchProviderException,
|
String base64Digest) throws NoSuchProviderException,
|
||||||
NoSuchAlgorithmException, InvalidKeyException {
|
NoSuchAlgorithmException, InvalidKeyException {
|
||||||
for (int i = 0; i < 10000; i++)
|
for (int i = 0; i < 10000; i++)
|
||||||
testBouncyCastleDigest(key, message, base64Digest);
|
testBouncyCastleHmacSha1Base64(key, message, base64Digest);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(dataProvider = "hmacsha1")
|
@Test(dataProvider = "hmacsha1")
|
||||||
|
@ -64,7 +65,7 @@ public class S3UtilsTest extends PerformanceTest {
|
||||||
for (int i = 0; i < 10000; i++)
|
for (int i = 0; i < 10000; i++)
|
||||||
completer.submit(new Callable<Boolean>() {
|
completer.submit(new Callable<Boolean>() {
|
||||||
public Boolean call() throws Exception {
|
public Boolean call() throws Exception {
|
||||||
testBouncyCastleDigest(key, message, base64Digest);
|
testBouncyCastleHmacSha1Base64(key, message, base64Digest);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -72,6 +73,17 @@ public class S3UtilsTest extends PerformanceTest {
|
||||||
assert completer.take().get();
|
assert completer.take().get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@DataProvider(name = "md5")
|
||||||
|
public Object[][] createMD5Data() {
|
||||||
|
return base64MD5MessageDigest;
|
||||||
|
}
|
||||||
|
public final static Object[][] base64MD5MessageDigest = {
|
||||||
|
{ "apple", "1f3870be274f6c49b3e31a0c6728957f" },
|
||||||
|
{ "bear", "893b56e3cfe153fb770a120b83bac20c" },
|
||||||
|
{ "candy", "c48ba993d35c3abe0380f91738fe2a34" },
|
||||||
|
{ "dogma", "95eb470e4faee302e9cd3063b1923dab" },
|
||||||
|
{ "emma", "00a809937eddc44521da9521269e75c6" } };
|
||||||
|
|
||||||
public final static Object[][] base64KeyMessageDigest = {
|
public final static Object[][] base64KeyMessageDigest = {
|
||||||
{ Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There",
|
{ Base64.decode("CwsLCwsLCwsLCwsLCwsLCwsLCws="), "Hi There",
|
||||||
"thcxhlUFcmTii8C2+zeMjvFGvgA=" },
|
"thcxhlUFcmTii8C2+zeMjvFGvgA=" },
|
||||||
|
@ -96,11 +108,19 @@ public class S3UtilsTest extends PerformanceTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(dataProvider = "hmacsha1")
|
@Test(dataProvider = "hmacsha1")
|
||||||
public void testBouncyCastleDigest(byte[] key, String message,
|
public void testBouncyCastleHmacSha1Base64(byte[] key, String message,
|
||||||
String base64Digest) throws NoSuchProviderException,
|
String base64Digest) throws NoSuchProviderException,
|
||||||
NoSuchAlgorithmException, InvalidKeyException {
|
NoSuchAlgorithmException, InvalidKeyException {
|
||||||
String b64 = S3Utils.digest(message, key);
|
String b64 = S3Utils.hmacSha1Base64(message, key);
|
||||||
assertEquals(b64, base64Digest);
|
assertEquals(b64, base64Digest);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test(dataProvider = "md5")
|
||||||
|
public void testBouncyCastleMD5Digest(String message,
|
||||||
|
String base64Digest) throws NoSuchProviderException,
|
||||||
|
NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException {
|
||||||
|
String b64 = S3Utils.md5Hex(message.getBytes());
|
||||||
|
assertEquals(base64Digest,b64);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,12 +23,19 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3;
|
package org.jclouds.aws.s3;
|
||||||
|
|
||||||
import org.jclouds.aws.s3.domain.S3Object;
|
import java.util.ArrayList;
|
||||||
import org.jclouds.aws.s3.domain.S3Bucket;
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.ExecutionException;
|
||||||
|
import java.util.concurrent.Future;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.TimeoutException;
|
||||||
|
|
||||||
import java.util.*;
|
import org.jclouds.aws.s3.domain.S3Bucket;
|
||||||
import java.util.concurrent.*;
|
import org.jclouds.aws.s3.domain.S3Object;
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* // TODO: Adrian: Document this!
|
* // TODO: Adrian: Document this!
|
||||||
|
@ -38,13 +45,14 @@ import java.io.IOException;
|
||||||
public class StubS3Connection implements S3Connection {
|
public class StubS3Connection implements S3Connection {
|
||||||
private static Map<S3Bucket, Map<String, Object>> bucketToContents = new ConcurrentHashMap<S3Bucket, Map<String, Object>>();
|
private static Map<S3Bucket, Map<String, Object>> bucketToContents = new ConcurrentHashMap<S3Bucket, Map<String, Object>>();
|
||||||
|
|
||||||
|
|
||||||
public Future<S3Object> getObject(final S3Bucket s3Bucket, final String key) {
|
public Future<S3Object> getObject(final S3Bucket s3Bucket, final String key) {
|
||||||
return new FutureBase<S3Object>() {
|
return new FutureBase<S3Object>() {
|
||||||
public S3Object get() throws InterruptedException, ExecutionException {
|
public S3Object get() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
if (!bucketToContents.containsKey(s3Bucket))
|
if (!bucketToContents.containsKey(s3Bucket))
|
||||||
return S3Object.NOT_FOUND;
|
return S3Object.NOT_FOUND;
|
||||||
Map<String, Object> realContents = bucketToContents.get(s3Bucket);
|
Map<String, Object> realContents = bucketToContents
|
||||||
|
.get(s3Bucket);
|
||||||
if (!realContents.containsKey(key))
|
if (!realContents.containsKey(key))
|
||||||
return S3Object.NOT_FOUND;
|
return S3Object.NOT_FOUND;
|
||||||
S3Object object = new S3Object();
|
S3Object object = new S3Object();
|
||||||
|
@ -59,9 +67,11 @@ public class StubS3Connection implements S3Connection {
|
||||||
return getObject(s3Bucket, key);
|
return getObject(s3Bucket, key);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Future<Boolean> deleteObject(final S3Bucket s3Bucket, final String key) {
|
public Future<Boolean> deleteObject(final S3Bucket s3Bucket,
|
||||||
|
final String key) {
|
||||||
return new FutureBase<Boolean>() {
|
return new FutureBase<Boolean>() {
|
||||||
public Boolean get() throws InterruptedException, ExecutionException {
|
public Boolean get() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
if (bucketToContents.containsKey(s3Bucket)) {
|
if (bucketToContents.containsKey(s3Bucket)) {
|
||||||
bucketToContents.get(s3Bucket).remove(key);
|
bucketToContents.get(s3Bucket).remove(key);
|
||||||
}
|
}
|
||||||
|
@ -70,13 +80,16 @@ public class StubS3Connection implements S3Connection {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public Future<String> addObject(final S3Bucket s3Bucket, final S3Object object) {
|
public Future<String> addObject(final S3Bucket s3Bucket,
|
||||||
|
final S3Object object) {
|
||||||
return new FutureBase<String>() {
|
return new FutureBase<String>() {
|
||||||
public String get() throws InterruptedException, ExecutionException {
|
public String get() throws InterruptedException, ExecutionException {
|
||||||
if (!bucketToContents.containsKey(s3Bucket)) {
|
if (!bucketToContents.containsKey(s3Bucket)) {
|
||||||
throw new ExecutionException(new RuntimeException("bucket not found: " + s3Bucket.getName()));
|
throw new ExecutionException(new RuntimeException(
|
||||||
|
"bucket not found: " + s3Bucket.getName()));
|
||||||
}
|
}
|
||||||
bucketToContents.get(s3Bucket).put(object.getKey(), object.getClass());
|
bucketToContents.get(s3Bucket).put(object.getKey(),
|
||||||
|
object.getClass());
|
||||||
return object.getKey();
|
return object.getKey();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -84,9 +97,11 @@ public class StubS3Connection implements S3Connection {
|
||||||
|
|
||||||
public Future<Boolean> createBucketIfNotExists(final S3Bucket s3Bucket) {
|
public Future<Boolean> createBucketIfNotExists(final S3Bucket s3Bucket) {
|
||||||
return new FutureBase<Boolean>() {
|
return new FutureBase<Boolean>() {
|
||||||
public Boolean get() throws InterruptedException, ExecutionException {
|
public Boolean get() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
if (!bucketToContents.containsKey(s3Bucket)) {
|
if (!bucketToContents.containsKey(s3Bucket)) {
|
||||||
bucketToContents.put(s3Bucket, new ConcurrentHashMap<String, Object>());
|
bucketToContents.put(s3Bucket,
|
||||||
|
new ConcurrentHashMap<String, Object>());
|
||||||
}
|
}
|
||||||
return bucketToContents.containsKey(s3Bucket);
|
return bucketToContents.containsKey(s3Bucket);
|
||||||
}
|
}
|
||||||
|
@ -95,7 +110,8 @@ public class StubS3Connection implements S3Connection {
|
||||||
|
|
||||||
public Future<Boolean> deleteBucket(final S3Bucket s3Bucket) {
|
public Future<Boolean> deleteBucket(final S3Bucket s3Bucket) {
|
||||||
return new FutureBase<Boolean>() {
|
return new FutureBase<Boolean>() {
|
||||||
public Boolean get() throws InterruptedException, ExecutionException {
|
public Boolean get() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
if (bucketToContents.containsKey(s3Bucket)) {
|
if (bucketToContents.containsKey(s3Bucket)) {
|
||||||
if (bucketToContents.get(s3Bucket).size() == 0)
|
if (bucketToContents.get(s3Bucket).size() == 0)
|
||||||
return true;
|
return true;
|
||||||
|
@ -105,13 +121,18 @@ public class StubS3Connection implements S3Connection {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public Future<Boolean> copyObject(final S3Bucket sourceBucket, final S3Object sourceObject, final S3Bucket destinationBucket, final S3Object destinationObject) {
|
public Future<Boolean> copyObject(final S3Bucket sourceBucket,
|
||||||
|
final S3Object sourceObject, final S3Bucket destinationBucket,
|
||||||
|
final S3Object destinationObject) {
|
||||||
return new FutureBase<Boolean>() {
|
return new FutureBase<Boolean>() {
|
||||||
public Boolean get() throws InterruptedException, ExecutionException {
|
public Boolean get() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
Map<String, Object> source = bucketToContents.get(sourceBucket);
|
Map<String, Object> source = bucketToContents.get(sourceBucket);
|
||||||
Map<String, Object> dest = bucketToContents.get(destinationBucket);
|
Map<String, Object> dest = bucketToContents
|
||||||
|
.get(destinationBucket);
|
||||||
if (source.containsKey(sourceObject.getKey())) {
|
if (source.containsKey(sourceObject.getKey())) {
|
||||||
dest.put(destinationObject.getKey(), source.get(sourceObject.getKey()));
|
dest.put(destinationObject.getKey(), source
|
||||||
|
.get(sourceObject.getKey()));
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
|
@ -121,7 +142,8 @@ public class StubS3Connection implements S3Connection {
|
||||||
|
|
||||||
public Future<Boolean> bucketExists(final S3Bucket s3Bucket) {
|
public Future<Boolean> bucketExists(final S3Bucket s3Bucket) {
|
||||||
return new FutureBase<Boolean>() {
|
return new FutureBase<Boolean>() {
|
||||||
public Boolean get() throws InterruptedException, ExecutionException {
|
public Boolean get() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
return bucketToContents.containsKey(s3Bucket);
|
return bucketToContents.containsKey(s3Bucket);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -129,9 +151,11 @@ public class StubS3Connection implements S3Connection {
|
||||||
|
|
||||||
public Future<S3Bucket> getBucket(final S3Bucket s3Bucket) {
|
public Future<S3Bucket> getBucket(final S3Bucket s3Bucket) {
|
||||||
return new FutureBase<S3Bucket>() {
|
return new FutureBase<S3Bucket>() {
|
||||||
public S3Bucket get() throws InterruptedException, ExecutionException {
|
public S3Bucket get() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
Set<S3Object> contents = new HashSet<S3Object>();
|
Set<S3Object> contents = new HashSet<S3Object>();
|
||||||
Map<String, Object> realContents = bucketToContents.get(s3Bucket);
|
Map<String, Object> realContents = bucketToContents
|
||||||
|
.get(s3Bucket);
|
||||||
if (realContents != null) {
|
if (realContents != null) {
|
||||||
for (String key : realContents.keySet()) {
|
for (String key : realContents.keySet()) {
|
||||||
S3Object object = new S3Object();
|
S3Object object = new S3Object();
|
||||||
|
@ -150,16 +174,13 @@ public class StubS3Connection implements S3Connection {
|
||||||
|
|
||||||
public Future<List<S3Bucket>> getBuckets() {
|
public Future<List<S3Bucket>> getBuckets() {
|
||||||
return new FutureBase<List<S3Bucket>>() {
|
return new FutureBase<List<S3Bucket>>() {
|
||||||
public List<S3Bucket> get() throws InterruptedException, ExecutionException {
|
public List<S3Bucket> get() throws InterruptedException,
|
||||||
|
ExecutionException {
|
||||||
return new ArrayList<S3Bucket>(bucketToContents.keySet());
|
return new ArrayList<S3Bucket>(bucketToContents.keySet());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close() throws IOException {
|
|
||||||
// nothing to close
|
|
||||||
}
|
|
||||||
|
|
||||||
private abstract class FutureBase<V> implements Future<V> {
|
private abstract class FutureBase<V> implements Future<V> {
|
||||||
public boolean cancel(boolean b) {
|
public boolean cancel(boolean b) {
|
||||||
return false;
|
return false;
|
||||||
|
@ -173,7 +194,8 @@ public class StubS3Connection implements S3Connection {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public V get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException {
|
public V get(long l, TimeUnit timeUnit) throws InterruptedException,
|
||||||
|
ExecutionException, TimeoutException {
|
||||||
return get();
|
return get();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,8 @@
|
||||||
*/
|
*/
|
||||||
package org.jclouds.aws.s3.commands;
|
package org.jclouds.aws.s3.commands;
|
||||||
|
|
||||||
|
import static org.testng.Assert.assertEquals;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
|
@ -148,7 +150,7 @@ public class S3ParserTest extends PerformanceTest {
|
||||||
assert object.getLastModified().equals(expected) : String
|
assert object.getLastModified().equals(expected) : String
|
||||||
.format("expected %1s, but got %1s", expected, object
|
.format("expected %1s, but got %1s", expected, object
|
||||||
.getLastModified());
|
.getLastModified());
|
||||||
assert object.getETag().equals("\"9d7bb64e8e18ee34eec06dd2cf37b766\"");
|
assertEquals(object.getETag(), "9d7bb64e8e18ee34eec06dd2cf37b766");
|
||||||
assert object.getSize() == 136;
|
assert object.getSize() == 136;
|
||||||
S3Owner owner = new S3Owner();
|
S3Owner owner = new S3Owner();
|
||||||
owner
|
owner
|
||||||
|
|
|
@ -27,6 +27,7 @@ import com.google.inject.AbstractModule;
|
||||||
import com.google.inject.Guice;
|
import com.google.inject.Guice;
|
||||||
import com.google.inject.name.Names;
|
import com.google.inject.name.Names;
|
||||||
import org.jclouds.aws.s3.DateService;
|
import org.jclouds.aws.s3.DateService;
|
||||||
|
import org.jclouds.aws.s3.S3Constants;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
|
|
||||||
|
@ -40,8 +41,8 @@ public class RequestAuthorizeSignatureTest {
|
||||||
filter = Guice.createInjector(new AbstractModule() {
|
filter = Guice.createInjector(new AbstractModule() {
|
||||||
|
|
||||||
protected void configure() {
|
protected void configure() {
|
||||||
bindConstant().annotatedWith(Names.named("jclouds.aws.accesskeyid")).to("foo");
|
bindConstant().annotatedWith(Names.named(S3Constants.PROPERTY_AWS_ACCESSKEYID)).to("foo");
|
||||||
bindConstant().annotatedWith(Names.named("jclouds.aws.secretaccesskey")).to("bar");
|
bindConstant().annotatedWith(Names.named(S3Constants.PROPERTY_AWS_SECRETACCESSKEY)).to("bar");
|
||||||
bind(DateService.class);
|
bind(DateService.class);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,6 +31,7 @@ import java.net.URL;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.jclouds.aws.s3.S3Constants;
|
||||||
import org.testng.annotations.BeforeTest;
|
import org.testng.annotations.BeforeTest;
|
||||||
import org.testng.annotations.Optional;
|
import org.testng.annotations.Optional;
|
||||||
import org.testng.annotations.Parameters;
|
import org.testng.annotations.Parameters;
|
||||||
|
@ -47,13 +48,14 @@ import org.testng.annotations.Test;
|
||||||
public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
|
public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
|
||||||
|
|
||||||
private static final String sysAWSAccessKeyId = System
|
private static final String sysAWSAccessKeyId = System
|
||||||
.getProperty("jclouds.aws.accesskeyid");
|
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
|
||||||
private static final String sysAWSSecretAccessKey = System
|
private static final String sysAWSSecretAccessKey = System
|
||||||
.getProperty("jclouds.aws.secretaccesskey");
|
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
|
||||||
|
|
||||||
@BeforeTest
|
@BeforeTest
|
||||||
@Parameters( { "warfile", "devappserver.address", "devappserver.port",
|
@Parameters( { "warfile", "devappserver.address", "devappserver.port",
|
||||||
"jclouds.aws.accesskeyid", "jclouds.aws.secretaccesskey" })
|
S3Constants.PROPERTY_AWS_ACCESSKEYID,
|
||||||
|
S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
|
||||||
public void startDevAppServer(final String warfile, final String address,
|
public void startDevAppServer(final String warfile, final String address,
|
||||||
final String port, @Optional String AWSAccessKeyId,
|
final String port, @Optional String AWSAccessKeyId,
|
||||||
@Optional String AWSSecretAccessKey) throws Exception {
|
@Optional String AWSSecretAccessKey) throws Exception {
|
||||||
|
@ -66,8 +68,8 @@ public class GoogleAppEngineTest extends BaseGoogleAppEngineTest {
|
||||||
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey");
|
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey");
|
||||||
|
|
||||||
Properties props = new Properties();
|
Properties props = new Properties();
|
||||||
props.put("jclouds.aws.accesskeyid", AWSAccessKeyId);
|
props.put(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId);
|
||||||
props.put("jclouds.aws.secretaccesskey", AWSSecretAccessKey);
|
props.put(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey);
|
||||||
writePropertiesAndStartServer(address, port, warfile, props);
|
writePropertiesAndStartServer(address, port, warfile, props);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -31,8 +31,8 @@ import javax.servlet.ServletContext;
|
||||||
import javax.servlet.ServletContextEvent;
|
import javax.servlet.ServletContextEvent;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.jclouds.aws.s3.S3ConnectionFactory;
|
import org.jclouds.aws.s3.S3ContextFactory;
|
||||||
import org.jclouds.aws.s3.S3ConnectionModule;
|
import org.jclouds.aws.s3.config.S3ContextModule;
|
||||||
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
|
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
|
||||||
import org.jclouds.lifecycle.Closer;
|
import org.jclouds.lifecycle.Closer;
|
||||||
import org.jclouds.samples.googleappengine.JCloudsServlet;
|
import org.jclouds.samples.googleappengine.JCloudsServlet;
|
||||||
|
@ -84,11 +84,11 @@ public class GuiceServletConfig extends GuiceServletContextListener {
|
||||||
} finally {
|
} finally {
|
||||||
IOUtils.closeQuietly(input);
|
IOUtils.closeQuietly(input);
|
||||||
}
|
}
|
||||||
props.putAll(S3ConnectionFactory.DEFAULT_PROPERTIES);
|
props.putAll(S3ContextFactory.DEFAULT_PROPERTIES);
|
||||||
Names.bindProperties(binder(), props);
|
Names.bindProperties(binder(), props);
|
||||||
}
|
}
|
||||||
}, new JavaUrlHttpFutureCommandClientModule(),
|
}, new JavaUrlHttpFutureCommandClientModule(),
|
||||||
new S3ConnectionModule(), new ServletModule() {
|
new S3ContextModule(), new ServletModule() {
|
||||||
@Override
|
@Override
|
||||||
protected void configureServlets() {
|
protected void configureServlets() {
|
||||||
serve("*.s3").with(JCloudsServlet.class);
|
serve("*.s3").with(JCloudsServlet.class);
|
||||||
|
|
Loading…
Reference in New Issue