Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

made tests parallel and fixed some transient errors in BaseS3Map 

git-svn-id: http://jclouds.googlecode.com/svn/trunk@834 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-05-22 15:35:57 +00:00
parent 6b9c2431d6
commit f8da659695
29 changed files with 2068 additions and 2025 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
aws/s3/core/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java
View File

@ -24,9 +24,19 @@
package org.jclouds.aws.s3.internal; package org.jclouds.aws.s3.internal;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted; import java.io.FileNotFoundException;
import com.google.inject.name.Named; import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3Connection; import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Map; import org.jclouds.aws.s3.S3Map;
import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Bucket;
@ -34,19 +44,16 @@ import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.reference.S3Constants; import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.util.Utils; import org.jclouds.util.Utils;
import java.io.FileNotFoundException; import com.google.common.annotations.VisibleForTesting;
import java.io.IOException; import com.google.inject.Inject;
import java.util.*; import com.google.inject.assistedinject.Assisted;
import java.util.concurrent.ExecutionException; import com.google.inject.name.Named;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/** /**
* Implements core Map functionality with an {@link S3Connection} * Implements core Map functionality with an {@link S3Connection}
* <p/> * <p/>
* All commands will wait a maximum of ${jclouds.s3.map.timeout} milliseconds to * All commands will wait a maximum of ${jclouds.s3.map.timeout} milliseconds to complete before
* complete before throwing an exception. * throwing an exception.
* *
* @author Adrian Cole * @author Adrian Cole
* @param <V> * @param <V>
@ -64,6 +71,13 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
@Named(S3Constants.PROPERTY_S3_MAP_TIMEOUT) @Named(S3Constants.PROPERTY_S3_MAP_TIMEOUT)
protected long requestTimeoutMilliseconds = 10000; protected long requestTimeoutMilliseconds = 10000;
/**
* time to pause before retrying a transient failure
*/
@Inject(optional = true)
@Named(S3Constants.PROPERTY_S3_MAP_RETRY)
protected long requestRetryMilliseconds = 10;
@Inject @Inject
public BaseS3Map(S3Connection connection, @Assisted String bucket) { public BaseS3Map(S3Connection connection, @Assisted String bucket) {
this.connection = checkNotNull(connection, "connection"); this.connection = checkNotNull(connection, "connection");
@ -83,14 +97,13 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
Set<S3Object.Metadata> contents = bucket.getContents(); Set<S3Object.Metadata> contents = bucket.getContents();
return contents.size(); return contents.size();
} catch (Exception e) { } catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting size of bucketName" throw new S3RuntimeException("Error getting size of bucketName" + bucket, e);
+ bucket, e);
} }
} }
protected boolean containsMd5(byte[] md5) throws InterruptedException, protected boolean containsMd5(byte[] md5) throws InterruptedException, ExecutionException,
ExecutionException, TimeoutException { TimeoutException {
for (S3Object.Metadata metadata : refreshBucket().getContents()) { for (S3Object.Metadata metadata : refreshBucket().getContents()) {
if (Arrays.equals(md5, metadata.getMd5())) if (Arrays.equals(md5, metadata.getMd5()))
return true; return true;
@ -98,9 +111,8 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
return false; return false;
} }
protected byte[] getMd5(Object value) throws IOException, protected byte[] getMd5(Object value) throws IOException, FileNotFoundException,
FileNotFoundException, InterruptedException, ExecutionException, InterruptedException, ExecutionException, TimeoutException {
TimeoutException {
S3Object object = null; S3Object object = null;
if (value instanceof S3Object) { if (value instanceof S3Object) {
object = (S3Object) value; object = (S3Object) value;
@ -124,37 +136,47 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
futureObjects.add(connection.getObject(bucket, key)); futureObjects.add(connection.getObject(bucket, key));
} }
for (Future<S3Object> futureObject : futureObjects) { for (Future<S3Object> futureObject : futureObjects) {
S3Object object = null;
try { try {
object = futureObject.get(requestTimeoutMilliseconds, ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
TimeUnit.MILLISECONDS);
} catch (Exception e) { } catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format( throw new S3RuntimeException(String.format("Error getting value from bucket %1$s",
"Error getting value from bucket %1$s", bucket), e); bucket), e);
} }
if (object != S3Object.NOT_FOUND)
objects.add(object);
} }
return objects; return objects;
} }
@VisibleForTesting
void ifNotFoundRetryOtherwiseAddToSet(Future<S3Object> futureObject, Set<S3Object> objects)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < 3; i++) {
S3Object object = futureObject.get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
if (object != S3Object.NOT_FOUND) {
objects.add(object);
break;
} else {
Thread.sleep(requestRetryMilliseconds);
}
}
}
/** /**
* {@inheritDoc} * {@inheritDoc}
* <p/> * <p/>
* Note that if value is an instance of InputStream, it will be read and * Note that if value is an instance of InputStream, it will be read and closed following this
* closed following this method. To reuse data from InputStreams, pass * method. To reuse data from InputStreams, pass {@link java.io.InputStream}s inside
* {@link java.io.InputStream}s inside {@link S3Object}s * {@link S3Object}s
*/ */
public boolean containsValue(Object value) { public boolean containsValue(Object value) {
try { try {
byte[] md5 = getMd5(value); byte[] md5 = getMd5(value);
return containsMd5(md5); return containsMd5(md5);
} catch (Exception e) { } catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format( throw new S3RuntimeException(String.format(
"Error searching for ETAG of value: [%2$s] in bucketName:%1$s", "Error searching for ETAG of value: [%2$s] in bucketName:%1$s", bucket, value), e);
bucket, value), e);
} }
} }
@ -177,20 +199,19 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
deletes.add(connection.deleteObject(bucket, key)); deletes.add(connection.deleteObject(bucket, key));
} }
for (Future<Boolean> isdeleted : deletes) for (Future<Boolean> isdeleted : deletes)
if (!isdeleted.get(requestTimeoutMilliseconds, if (!isdeleted.get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)) {
TimeUnit.MILLISECONDS)) {
throw new S3RuntimeException("failed to delete entry"); throw new S3RuntimeException("failed to delete entry");
} }
} catch (Exception e) { } catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucketName" + bucket, e); throw new S3RuntimeException("Error clearing bucketName" + bucket, e);
} }
} }
protected S3Bucket refreshBucket() throws InterruptedException, protected S3Bucket refreshBucket() throws InterruptedException, ExecutionException,
ExecutionException, TimeoutException { TimeoutException {
S3Bucket currentBucket = connection.listBucket(bucket).get( S3Bucket currentBucket = connection.listBucket(bucket).get(requestTimeoutMilliseconds,
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); TimeUnit.MILLISECONDS);
if (currentBucket == S3Bucket.NOT_FOUND) if (currentBucket == S3Bucket.NOT_FOUND)
throw new S3RuntimeException("bucketName not found: " + bucket); throw new S3RuntimeException("bucketName not found: " + bucket);
else else
@ -204,20 +225,19 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
keys.add(object.getKey()); keys.add(object.getKey());
return keys; return keys;
} catch (Exception e) { } catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting keys in bucketName: " throw new S3RuntimeException("Error getting keys in bucketName: " + bucket, e);
+ bucket, e);
} }
} }
public boolean containsKey(Object key) { public boolean containsKey(Object key) {
try { try {
return connection.headObject(bucket, key.toString()).get( return connection.headObject(bucket, key.toString()).get(requestTimeoutMilliseconds,
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS) != S3Object.Metadata.NOT_FOUND; TimeUnit.MILLISECONDS) != S3Object.Metadata.NOT_FOUND;
} catch (Exception e) { } catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format( throw new S3RuntimeException(String.format("Error searching for %1$s:%2$s", bucket, key),
"Error searching for %1$s:%2$s", bucket, key), e); e);
} }
} }
@ -229,7 +249,7 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
try { try {
return refreshBucket(); return refreshBucket();
} catch (Exception e) { } catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting bucketName" + bucket, e); throw new S3RuntimeException("Error getting bucketName" + bucket, e);
} }
} }

7
aws/s3/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java
View File

@ -36,9 +36,12 @@ public interface S3Constants extends HttpConstants, PoolConstants, S3Headers {
public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey"; public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey";
public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid"; public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid";
/** /**
* longest time a single Map operation can take before throwing an * longest time a single Map operation can take before throwing an exception.
* exception.
*/ */
public static final String PROPERTY_S3_MAP_TIMEOUT = "jclouds.s3.map.timeout"; public static final String PROPERTY_S3_MAP_TIMEOUT = "jclouds.s3.map.timeout";
/**
* time to pause before retrying a transient failure
*/
public static final String PROPERTY_S3_MAP_RETRY = "jclouds.s3.map.retry";
} }

14
aws/s3/core/src/test/java/org/jclouds/aws/PerformanceTest.java
View File

@ -23,29 +23,29 @@
*/ */
package org.jclouds.aws; package org.jclouds.aws;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
/** /**
* // TODO: Adrian: Document this! * // TODO: Adrian: Document this!
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = "performance", sequential = true, testName = "s3.PerformanceTest") @Test(groups = "performance")
public class PerformanceTest { public class PerformanceTest {
protected static int LOOP_COUNT = 1000; protected static int LOOP_COUNT = 1000;
protected ExecutorService exec; protected ExecutorService exec;
@BeforeMethod @BeforeTest
public void setupExecutorService() { public void setupExecutorService() {
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
} }
@AfterMethod @AfterTest
public void teardownExecutorService() { public void teardownExecutorService() {
exec.shutdownNow(); exec.shutdownNow();
exec = null; exec = null;

49
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3ConnectionIntegrationTest.java
View File

@ -24,20 +24,19 @@
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.util.S3Utils;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import java.io.InputStream; import java.io.InputStream;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.util.S3Utils;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
/** /**
* Tests connection by listing all the buckets and their size * Tests connection by listing all the buckets and their size
* *
@ -46,37 +45,30 @@ import java.util.concurrent.TimeUnit;
@Test(testName = "s3.S3ConnectionIntegrationTest") @Test(testName = "s3.S3ConnectionIntegrationTest")
public class S3ConnectionIntegrationTest extends S3IntegrationTest { public class S3ConnectionIntegrationTest extends S3IntegrationTest {
@Test(groups = {"integration"}) @Test(groups = { "integration" })
void testListBuckets() throws Exception { void testListBuckets() throws Exception {
List<S3Bucket.Metadata> myBuckets = client.listOwnedBuckets().get(10, client.listOwnedBuckets().get(10, TimeUnit.SECONDS);
TimeUnit.SECONDS);
for (S3Bucket.Metadata bucket : myBuckets) {
context.createInputStreamMap(bucket.getName()).size();
}
} }
private static final String sysHttpStreamUrl = System private static final String sysHttpStreamUrl = System.getProperty("jclouds.s3.httpstream.url");
.getProperty("jclouds.s3.httpstream.url"); private static final String sysHttpStreamMd5 = System.getProperty("jclouds.s3.httpstream.md5");
private static final String sysHttpStreamMd5 = System
.getProperty("jclouds.s3.httpstream.md5");
@Test(groups = {"integration"}) @Test(groups = { "integration" })
@Parameters({"jclouds.s3.httpstream.url", "jclouds.s3.httpstream.md5"}) @Parameters( { "jclouds.s3.httpstream.url", "jclouds.s3.httpstream.md5" })
public void testCopyUrl(@Optional String httpStreamUrl, public void testCopyUrl(@Optional String httpStreamUrl, @Optional String httpStreamMd5)
@Optional String httpStreamMd5) throws Exception { throws Exception {
httpStreamUrl = checkNotNull(httpStreamUrl != null ? httpStreamUrl httpStreamUrl = checkNotNull(httpStreamUrl != null ? httpStreamUrl : sysHttpStreamUrl,
: sysHttpStreamUrl, "httpStreamUrl"); "httpStreamUrl");
httpStreamMd5 = checkNotNull(httpStreamMd5 != null ? httpStreamMd5 httpStreamMd5 = checkNotNull(httpStreamMd5 != null ? httpStreamMd5 : sysHttpStreamMd5,
: sysHttpStreamMd5, "httpStreamMd5"); "httpStreamMd5");
String bucketName = bucketPrefix + "tcu"; String bucketName = bucketPrefix + "tcu";
createBucketAndEnsureEmpty(bucketName); createBucketAndEnsureEmpty(bucketName);
String key = "hello"; String key = "hello";
URL url = new URL(httpStreamUrl); URL url = new URL(httpStreamUrl);
byte[] md5 = S3Utils byte[] md5 = S3Utils.fromHexString(httpStreamMd5);
.fromHexString(httpStreamMd5);
URLConnection connection = url.openConnection(); URLConnection connection = url.openConnection();
int length = connection.getContentLength(); int length = connection.getContentLength();
@ -87,8 +79,7 @@ public class S3ConnectionIntegrationTest extends S3IntegrationTest {
object.getMetadata().setMd5(md5); object.getMetadata().setMd5(md5);
object.getMetadata().setSize(length); object.getMetadata().setSize(length);
byte[] newMd5 = client.putObject(bucketName, object).get(30, byte[] newMd5 = client.putObject(bucketName, object).get(30, TimeUnit.SECONDS);
TimeUnit.SECONDS);
assertEquals(newMd5, md5); assertEquals(newMd5, md5);
} }
} }

146
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java
View File

@ -24,7 +24,27 @@
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import com.google.inject.Module; import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.ConsoleHandler;
import java.util.logging.Formatter;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import org.jclouds.aws.s3.config.StubS3ConnectionModule; import org.jclouds.aws.s3.config.StubS3ConnectionModule;
import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.aws.s3.domain.S3Object;
@ -32,16 +52,15 @@ import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.aws.s3.util.S3Utils; import org.jclouds.aws.s3.util.S3Utils;
import org.jclouds.http.HttpConstants; import org.jclouds.http.HttpConstants;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import static org.testng.Assert.assertEquals;
import org.testng.ITestContext; import org.testng.ITestContext;
import org.testng.annotations.*; import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import java.io.IOException; import com.google.inject.Module;
import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.*;
import java.util.logging.*;
import java.util.logging.Formatter;
public class S3IntegrationTest { public class S3IntegrationTest {
protected static final String TEST_STRING = "<apples><apple name=\"fuji\"></apple> </apples>"; protected static final String TEST_STRING = "<apples><apple name=\"fuji\"></apple> </apples>";
@ -50,16 +69,16 @@ public class S3IntegrationTest {
protected byte[] badMd5; protected byte[] badMd5;
protected String bucketName; protected String bucketName;
protected void createBucketAndEnsureEmpty(String sourceBucket) protected void createBucketAndEnsureEmpty(String sourceBucket) throws InterruptedException,
throws InterruptedException, ExecutionException, TimeoutException { ExecutionException, TimeoutException {
client.deleteBucketIfEmpty(sourceBucket).get(10, TimeUnit.SECONDS);
client.putBucketIfNotExists(sourceBucket).get(10, TimeUnit.SECONDS); client.putBucketIfNotExists(sourceBucket).get(10, TimeUnit.SECONDS);
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS) assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS).getContents().size(),
.getContents().size(), 0); 0, "bucket " + sourceBucket + "wasn't empty");
} }
protected void addObjectToBucket(String sourceBucket, String key) protected void addObjectToBucket(String sourceBucket, String key) throws InterruptedException,
throws InterruptedException, ExecutionException, TimeoutException, ExecutionException, TimeoutException, IOException {
IOException {
S3Object sourceObject = new S3Object(key); S3Object sourceObject = new S3Object(key);
sourceObject.getMetadata().setContentType("text/xml"); sourceObject.getMetadata().setContentType("text/xml");
sourceObject.setData(TEST_STRING); sourceObject.setData(TEST_STRING);
@ -67,24 +86,21 @@ public class S3IntegrationTest {
} }
protected void addObjectToBucket(String sourceBucket, S3Object object) protected void addObjectToBucket(String sourceBucket, S3Object object)
throws InterruptedException, ExecutionException, TimeoutException, throws InterruptedException, ExecutionException, TimeoutException, IOException {
IOException {
client.putObject(sourceBucket, object).get(10, TimeUnit.SECONDS); client.putObject(sourceBucket, object).get(10, TimeUnit.SECONDS);
} }
protected S3Object validateContent(String sourceBucket, String key) protected S3Object validateContent(String sourceBucket, String key) throws InterruptedException,
throws InterruptedException, ExecutionException, TimeoutException, ExecutionException, TimeoutException, IOException {
IOException { assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS).getContents().size(),
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS) 1);
.getContents().size(), 1); S3Object newObject = client.getObject(sourceBucket, key).get(10, TimeUnit.SECONDS);
S3Object newObject = client.getObject(sourceBucket, key).get(10,
TimeUnit.SECONDS);
assert newObject != S3Object.NOT_FOUND; assert newObject != S3Object.NOT_FOUND;
assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING); assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING);
return newObject; return newObject;
} }
@BeforeClass(groups = {"integration", "live"}) @BeforeClass(groups = { "integration", "live" })
void enableDebug() { void enableDebug() {
if (debugEnabled()) { if (debugEnabled()) {
Handler HANDLER = new ConsoleHandler() { Handler HANDLER = new ConsoleHandler() {
@ -94,14 +110,10 @@ public class S3IntegrationTest {
@Override @Override
public String format(LogRecord record) { public String format(LogRecord record) {
return String.format( return String.format("[%tT %-7s] [%-7s] [%s]: %s %s\n", new Date(record
"[%tT %-7s] [%-7s] [%s]: %s %s\n", .getMillis()), record.getLevel(), Thread.currentThread().getName(),
new Date(record.getMillis()), record record.getLoggerName(), record.getMessage(),
.getLevel(), Thread.currentThread() record.getThrown() == null ? "" : record.getThrown());
.getName(), record.getLoggerName(),
record.getMessage(),
record.getThrown() == null ? "" : record
.getThrown());
} }
}); });
} }
@ -115,34 +127,33 @@ public class S3IntegrationTest {
protected S3Connection client; protected S3Connection client;
protected S3Context context = null; protected S3Context context = null;
protected String bucketPrefix = (System.getProperty("user.name") + "." + this protected String bucketPrefix = (System.getProperty("user.name") + "." + this.getClass()
.getClass().getSimpleName()).toLowerCase(); .getSimpleName()).toLowerCase();
private static final String sysAWSAccessKeyId = System private static final String sysAWSAccessKeyId = System
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID); .getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
private static final String sysAWSSecretAccessKey = System private static final String sysAWSSecretAccessKey = System
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY); .getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
@BeforeClass(inheritGroups = false, groups = {"integration", "live"}) @BeforeClass(inheritGroups = false, groups = { "integration", "live" })
@Parameters({S3Constants.PROPERTY_AWS_ACCESSKEYID, @Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
S3Constants.PROPERTY_AWS_SECRETACCESSKEY})
protected void setUpCredentials(@Optional String AWSAccessKeyId, protected void setUpCredentials(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey, ITestContext testContext) throws Exception { @Optional String AWSSecretAccessKey, ITestContext testContext) throws Exception {
AWSAccessKeyId = AWSAccessKeyId != null ? AWSAccessKeyId AWSAccessKeyId = AWSAccessKeyId != null ? AWSAccessKeyId : sysAWSAccessKeyId;
: sysAWSAccessKeyId; AWSSecretAccessKey = AWSSecretAccessKey != null ? AWSSecretAccessKey : sysAWSSecretAccessKey;
AWSSecretAccessKey = AWSSecretAccessKey != null ? AWSSecretAccessKey
: sysAWSSecretAccessKey;
if (AWSAccessKeyId != null) if (AWSAccessKeyId != null)
testContext.setAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId); testContext.setAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId);
if (AWSSecretAccessKey != null) if (AWSSecretAccessKey != null)
testContext.setAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey); testContext.setAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey);
} }
@BeforeClass(dependsOnMethods = {"setUpCredentials"}, groups = {"integration", "live"}) @BeforeClass(dependsOnMethods = { "setUpCredentials" }, groups = { "integration", "live" })
protected void setUpClient(ITestContext testContext) throws Exception { protected void setUpClient(ITestContext testContext) throws Exception {
if (testContext.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID) != null) { if (testContext.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID) != null) {
String AWSAccessKeyId = (String) testContext.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID); String AWSAccessKeyId = (String) testContext
String AWSSecretAccessKey = (String) testContext.getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY); .getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID);
String AWSSecretAccessKey = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
createLiveS3Context(AWSAccessKeyId, AWSSecretAccessKey); createLiveS3Context(AWSAccessKeyId, AWSSecretAccessKey);
} else { } else {
createStubS3Context(); createStubS3Context();
@ -161,20 +172,20 @@ public class S3IntegrationTest {
} }
protected void createLiveS3Context(String AWSAccessKeyId, String AWSSecretAccessKey) { protected void createLiveS3Context(String AWSAccessKeyId, String AWSSecretAccessKey) {
context = S3ContextFactory.createS3Context(buildS3Properties( context = S3ContextFactory.createS3Context(buildS3Properties(checkNotNull(AWSAccessKeyId,
checkNotNull(AWSAccessKeyId, "AWSAccessKeyId"), "AWSAccessKeyId"), checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey")),
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey")),
createHttpModule()); createHttpModule());
} }
@BeforeMethod(dependsOnMethods = "deleteBucket", groups = {"integration", "live"}) @BeforeMethod(dependsOnMethods = "deleteBucket", groups = { "integration", "live" })
public void setUpBucket(Method method) throws TimeoutException, ExecutionException, InterruptedException { public void setUpBucket(Method method) throws TimeoutException, ExecutionException,
InterruptedException {
bucketName = (bucketPrefix + method.getName()).toLowerCase(); bucketName = (bucketPrefix + method.getName()).toLowerCase();
createBucketAndEnsureEmpty(bucketName); createBucketAndEnsureEmpty(bucketName);
} }
@BeforeMethod(groups = {"integration", "live"}) @BeforeMethod(groups = { "integration", "live" })
@AfterMethod(groups = {"integration", "live"}) @AfterMethod(groups = { "integration", "live" })
public void deleteBucket() throws TimeoutException, ExecutionException, InterruptedException { public void deleteBucket() throws TimeoutException, ExecutionException, InterruptedException {
if (bucketName != null) if (bucketName != null)
deleteBucket(bucketName); deleteBucket(bucketName);
@ -184,14 +195,12 @@ public class S3IntegrationTest {
return false; return false;
} }
protected Properties buildS3Properties(String AWSAccessKeyId, protected Properties buildS3Properties(String AWSAccessKeyId, String AWSSecretAccessKey) {
String AWSSecretAccessKey) { Properties properties = new Properties(S3ContextFactory.DEFAULT_PROPERTIES);
Properties properties = new Properties( properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID, checkNotNull(AWSAccessKeyId,
S3ContextFactory.DEFAULT_PROPERTIES); "AWSAccessKeyId"));
properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID, properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, checkNotNull(
checkNotNull(AWSAccessKeyId, "AWSAccessKeyId")); AWSSecretAccessKey, "AWSSecretAccessKey"));
properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY,
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey"));
properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false"); properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false");
properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80"); properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80");
return properties; return properties;
@ -203,8 +212,7 @@ public class S3IntegrationTest {
protected void deleteEverything() throws Exception { protected void deleteEverything() throws Exception {
try { try {
List<S3Bucket.Metadata> metadata = client.listOwnedBuckets().get( List<S3Bucket.Metadata> metadata = client.listOwnedBuckets().get(10, TimeUnit.SECONDS);
10, TimeUnit.SECONDS);
for (S3Bucket.Metadata metaDatum : metadata) { for (S3Bucket.Metadata metaDatum : metadata) {
if (metaDatum.getName().startsWith(bucketPrefix.toLowerCase())) { if (metaDatum.getName().startsWith(bucketPrefix.toLowerCase())) {
deleteBucket(metaDatum.getName()); deleteBucket(metaDatum.getName());
@ -216,23 +224,21 @@ public class S3IntegrationTest {
} }
} }
private void deleteBucket(String name) throws InterruptedException, ExecutionException, TimeoutException { private void deleteBucket(String name) throws InterruptedException, ExecutionException,
TimeoutException {
if (client.bucketExists(name).get(10, TimeUnit.SECONDS)) { if (client.bucketExists(name).get(10, TimeUnit.SECONDS)) {
List<Future<Boolean>> results = new ArrayList<Future<Boolean>>(); List<Future<Boolean>> results = new ArrayList<Future<Boolean>>();
S3Bucket bucket = client.listBucket(name) S3Bucket bucket = client.listBucket(name).get(10, TimeUnit.SECONDS);
.get(10, TimeUnit.SECONDS);
for (S3Object.Metadata objectMeta : bucket.getContents()) { for (S3Object.Metadata objectMeta : bucket.getContents()) {
results.add(client.deleteObject(name, results.add(client.deleteObject(name, objectMeta.getKey()));
objectMeta.getKey()));
} }
Iterator<Future<Boolean>> iterator = results.iterator(); Iterator<Future<Boolean>> iterator = results.iterator();
while (iterator.hasNext()) { while (iterator.hasNext()) {
iterator.next().get(10, TimeUnit.SECONDS); iterator.next().get(10, TimeUnit.SECONDS);
iterator.remove(); iterator.remove();
} }
client.deleteBucketIfEmpty(name).get(10, client.deleteBucketIfEmpty(name).get(10, TimeUnit.SECONDS);
TimeUnit.SECONDS);
} }
} }

2
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3UtilsTest.java
View File

@ -44,7 +44,7 @@ import java.util.concurrent.ExecutorCompletionService;
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = "performance", sequential = true, testName = "s3.PerformanceTest") @Test(groups = "performance", sequential = true, testName = "s3.S3UtilsTest")
public class S3UtilsTest extends PerformanceTest { public class S3UtilsTest extends PerformanceTest {
@Test(dataProvider = "hmacsha1") @Test(dataProvider = "hmacsha1")

58
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java
View File

@ -23,45 +23,50 @@
*/ */
package org.jclouds.aws.s3.commands; package org.jclouds.aws.s3.commands;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.commands.options.CopyObjectOptions;
import org.jclouds.aws.s3.commands.options.GetObjectOptions;
import org.jclouds.aws.s3.commands.options.ListBucketOptions;
import org.jclouds.aws.s3.commands.options.PutBucketOptions;
import org.jclouds.aws.s3.commands.options.PutObjectOptions;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Bucket.Metadata.LocationConstraint;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap; import com.google.common.collect.Multimap;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Injector; import com.google.inject.Injector;
import com.google.inject.name.Names; import com.google.inject.name.Names;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.commands.options.*;
import org.jclouds.aws.s3.domain.S3Bucket.Metadata.LocationConstraint;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/** /**
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = {"unit"}, testName = "s3.S3CommandFactoryTest") @Test(groups = { "unit" }, testName = "s3.S3CommandFactoryTest")
public class S3CommandFactoryTest { public class S3CommandFactoryTest {
Injector injector = null; Injector injector = null;
S3CommandFactory commandFactory = null; S3CommandFactory commandFactory = null;
@BeforeMethod @BeforeTest
void setUpInjector() { void setUpInjector() {
injector = Guice.createInjector(new S3CommandsModule() { injector = Guice.createInjector(new S3CommandsModule() {
@Override @Override
protected void configure() { protected void configure() {
bindConstant().annotatedWith( bindConstant().annotatedWith(Names.named("jclouds.http.address")).to("localhost");
Names.named("jclouds.http.address")).to("localhost");
super.configure(); super.configure();
} }
}); });
commandFactory = injector.getInstance(S3CommandFactory.class); commandFactory = injector.getInstance(S3CommandFactory.class);
} }
@AfterMethod @AfterTest
void tearDownInjector() { void tearDownInjector() {
commandFactory = null; commandFactory = null;
injector = null; injector = null;
@ -69,14 +74,14 @@ public class S3CommandFactoryTest {
@Test @Test
void testCreateCopyObject() { void testCreateCopyObject() {
assert commandFactory.createCopyObject("sourcebucket", "sourceObject", assert commandFactory.createCopyObject("sourcebucket", "sourceObject", "destbucket",
"destbucket", "destObject", CopyObjectOptions.NONE) != null; "destObject", CopyObjectOptions.NONE) != null;
} }
@Test @Test
void testCreateCopyObjectOptions() { void testCreateCopyObjectOptions() {
assert commandFactory.createCopyObject("sourcebucket", "sourceObject", assert commandFactory.createCopyObject("sourcebucket", "sourceObject", "destbucket",
"destbucket", "destObject", new CopyObjectOptions()) != null; "destObject", new CopyObjectOptions()) != null;
} }
@Test @Test
@ -113,26 +118,21 @@ public class S3CommandFactoryTest {
expect(metadata.getKey()).andReturn("rawr"); expect(metadata.getKey()).andReturn("rawr");
expect(metadata.getContentType()).andReturn("text/xml").atLeastOnce(); expect(metadata.getContentType()).andReturn("text/xml").atLeastOnce();
expect(metadata.getCacheControl()).andReturn("no-cache").atLeastOnce(); expect(metadata.getCacheControl()).andReturn("no-cache").atLeastOnce();
expect(metadata.getContentDisposition()).andReturn("disposition") expect(metadata.getContentDisposition()).andReturn("disposition").atLeastOnce();
.atLeastOnce(); expect(metadata.getContentEncoding()).andReturn("encoding").atLeastOnce();
expect(metadata.getContentEncoding()).andReturn("encoding") expect(metadata.getMd5()).andReturn("encoding".getBytes()).atLeastOnce();
.atLeastOnce();
expect(metadata.getMd5()).andReturn("encoding".getBytes())
.atLeastOnce();
Multimap<String, String> userMdata = HashMultimap.create(); Multimap<String, String> userMdata = HashMultimap.create();
expect(metadata.getUserMetadata()).andReturn(userMdata).atLeastOnce(); expect(metadata.getUserMetadata()).andReturn(userMdata).atLeastOnce();
replay(metadata); replay(metadata);
object.setData("<a></a>"); object.setData("<a></a>");
assert commandFactory.createPutObject("test", object, assert commandFactory.createPutObject("test", object, PutObjectOptions.NONE) != null;
PutObjectOptions.NONE) != null;
} }
@Test @Test
void testCreateGetObject() { void testCreateGetObject() {
assert commandFactory.createGetObject("test", "blah", assert commandFactory.createGetObject("test", "blah", GetObjectOptions.NONE) != null;
GetObjectOptions.NONE) != null;
} }
@Test @Test

70
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/S3ParserTest.java
View File

@ -23,8 +23,16 @@
*/ */
package org.jclouds.aws.s3.commands; package org.jclouds.aws.s3.commands;
import com.google.inject.Guice; import static org.testng.Assert.assertEquals;
import com.google.inject.Injector;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.jclouds.aws.PerformanceTest; import org.jclouds.aws.PerformanceTest;
import org.jclouds.aws.s3.domain.CanonicalUser; import org.jclouds.aws.s3.domain.CanonicalUser;
@ -38,26 +46,20 @@ import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.jclouds.http.HttpException; import org.jclouds.http.HttpException;
import org.jclouds.http.commands.callables.xml.ParseSax; import org.jclouds.http.commands.callables.xml.ParseSax;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import static org.testng.Assert.assertEquals; import org.testng.annotations.AfterTest;
import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import java.io.IOException; import com.google.inject.Guice;
import java.io.UnsupportedEncodingException; import com.google.inject.Injector;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
/** /**
* Tests parsing of S3 responses * Tests parsing of S3 responses
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = {"performance"}, testName = "s3.S3ParserTest") @Test(groups = { "performance" }, testName = "s3.S3ParserTest")
public class S3ParserTest extends PerformanceTest { public class S3ParserTest extends PerformanceTest {
Injector injector = null; Injector injector = null;
@ -65,14 +67,14 @@ public class S3ParserTest extends PerformanceTest {
S3ParserFactory parserFactory = null; S3ParserFactory parserFactory = null;
@BeforeMethod @BeforeTest
protected void setUpInjector() { protected void setUpInjector() {
injector = Guice.createInjector(new S3ParserModule()); injector = Guice.createInjector(new S3ParserModule());
parserFactory = injector.getInstance(S3ParserFactory.class); parserFactory = injector.getInstance(S3ParserFactory.class);
assert parserFactory != null; assert parserFactory != null;
} }
@AfterMethod @AfterTest
protected void tearDownInjector() { protected void tearDownInjector() {
parserFactory = null; parserFactory = null;
injector = null; injector = null;
@ -84,21 +86,19 @@ public class S3ParserTest extends PerformanceTest {
runParseListAllMyBuckets(); runParseListAllMyBuckets();
} }
private List<S3Bucket.Metadata> runParseListAllMyBuckets() private List<S3Bucket.Metadata> runParseListAllMyBuckets() throws HttpException {
throws HttpException {
return parserFactory.createListBucketsParser().parse( return parserFactory.createListBucketsParser().parse(
IOUtils.toInputStream(listAllMyBucketsResultOn200)); IOUtils.toInputStream(listAllMyBucketsResultOn200));
} }
@Test @Test
void testParseListAllMyBucketsParallelResponseTime() void testParseListAllMyBucketsParallelResponseTime() throws InterruptedException,
throws InterruptedException, ExecutionException { ExecutionException {
CompletionService<List<S3Bucket.Metadata>> completer = new ExecutorCompletionService<List<S3Bucket.Metadata>>( CompletionService<List<S3Bucket.Metadata>> completer = new ExecutorCompletionService<List<S3Bucket.Metadata>>(
exec); exec);
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<List<S3Bucket.Metadata>>() { completer.submit(new Callable<List<S3Bucket.Metadata>>() {
public List<S3Bucket.Metadata> call() throws IOException, public List<S3Bucket.Metadata> call() throws IOException, SAXException, HttpException {
SAXException, HttpException {
return runParseListAllMyBuckets(); return runParseListAllMyBuckets();
} }
}); });
@ -128,8 +128,7 @@ public class S3ParserTest extends PerformanceTest {
public static final String listBucketResult = "<ListBucketHandler xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>adrianjbosstest</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>3366</Key><LastModified>2009-03-12T02:00:13.000Z</LastModified><ETag>&quot;9d7bb64e8e18ee34eec06dd2cf37b766&quot;</ETag><Size>136</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketHandler>"; public static final String listBucketResult = "<ListBucketHandler xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>adrianjbosstest</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>3366</Key><LastModified>2009-03-12T02:00:13.000Z</LastModified><ETag>&quot;9d7bb64e8e18ee34eec06dd2cf37b766&quot;</ETag><Size>136</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketHandler>";
public void testCanParseListBucketResult() throws HttpException, public void testCanParseListBucketResult() throws HttpException, UnsupportedEncodingException {
UnsupportedEncodingException {
S3Bucket bucket = runParseListBucketResult(); S3Bucket bucket = runParseListBucketResult();
assert !bucket.isTruncated(); assert !bucket.isTruncated();
assert bucket.getName().equals("adrianjbosstest"); assert bucket.getName().equals("adrianjbosstest");
@ -138,10 +137,8 @@ public class S3ParserTest extends PerformanceTest {
assert object.getKey().equals("3366"); assert object.getKey().equals("3366");
DateTime expected = new DateTime("2009-03-12T02:00:13.000Z"); DateTime expected = new DateTime("2009-03-12T02:00:13.000Z");
assert object.getLastModified().equals(expected) : String.format( assert object.getLastModified().equals(expected) : String.format(
"expected %1$s, but got %1$s", expected, object "expected %1$s, but got %1$s", expected, object.getLastModified());
.getLastModified()); assertEquals(S3Utils.toHexString(object.getMd5()), "9d7bb64e8e18ee34eec06dd2cf37b766");
assertEquals(S3Utils.toHexString(object.getMd5()),
"9d7bb64e8e18ee34eec06dd2cf37b766");
assert object.getSize() == 136; assert object.getSize() == 136;
CanonicalUser owner = new CanonicalUser( CanonicalUser owner = new CanonicalUser(
"e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0"); "e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0");
@ -160,20 +157,17 @@ public class S3ParserTest extends PerformanceTest {
public static final String successfulCopyObject200 = "<CopyObjectResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><LastModified>2009-03-19T13:23:27.000Z</LastModified><ETag>\"92836a3ea45a6984d1b4d23a747d46bb\"</ETag></CopyObjectResult>"; public static final String successfulCopyObject200 = "<CopyObjectResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><LastModified>2009-03-19T13:23:27.000Z</LastModified><ETag>\"92836a3ea45a6984d1b4d23a747d46bb\"</ETag></CopyObjectResult>";
private S3Object.Metadata runParseCopyObjectResult() throws HttpException { private S3Object.Metadata runParseCopyObjectResult() throws HttpException {
ParseSax<S3Object.Metadata> parser = parserFactory ParseSax<S3Object.Metadata> parser = parserFactory.createCopyObjectParser();
.createCopyObjectParser();
CopyObjectHandler handler = (CopyObjectHandler) parser.getHandler(); CopyObjectHandler handler = (CopyObjectHandler) parser.getHandler();
handler.setKey("adrianjbosstest"); handler.setKey("adrianjbosstest");
return parser.parse(IOUtils.toInputStream(successfulCopyObject200)); return parser.parse(IOUtils.toInputStream(successfulCopyObject200));
} }
public void testCanParseCopyObjectResult() throws HttpException, public void testCanParseCopyObjectResult() throws HttpException, UnsupportedEncodingException {
UnsupportedEncodingException {
S3Object.Metadata metadata = runParseCopyObjectResult(); S3Object.Metadata metadata = runParseCopyObjectResult();
DateTime expected = new DateTime("2009-03-19T13:23:27.000Z"); DateTime expected = new DateTime("2009-03-19T13:23:27.000Z");
assertEquals(metadata.getLastModified(), expected); assertEquals(metadata.getLastModified(), expected);
assertEquals(S3Utils.toHexString(metadata.getMd5()), assertEquals(S3Utils.toHexString(metadata.getMd5()), "92836a3ea45a6984d1b4d23a747d46bb");
"92836a3ea45a6984d1b4d23a747d46bb");
assertEquals(metadata.getKey(), "adrianjbosstest"); assertEquals(metadata.getKey(), "adrianjbosstest");
} }
@ -184,14 +178,12 @@ public class S3ParserTest extends PerformanceTest {
} }
@Test @Test
void testParseListBucketResultParallelResponseTime() void testParseListBucketResultParallelResponseTime() throws InterruptedException,
throws InterruptedException, ExecutionException { ExecutionException {
CompletionService<S3Bucket> completer = new ExecutorCompletionService<S3Bucket>( CompletionService<S3Bucket> completer = new ExecutorCompletionService<S3Bucket>(exec);
exec);
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<S3Bucket>() { completer.submit(new Callable<S3Bucket>() {
public S3Bucket call() throws IOException, SAXException, public S3Bucket call() throws IOException, SAXException, HttpException {
HttpException {
return runParseListBucketResult(); return runParseListBucketResult();
} }
}); });

15
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/callables/ParseObjectFromHeadersAndHttpContentTest.java
View File

@ -23,16 +23,19 @@
*/ */
package org.jclouds.aws.s3.commands.callables; package org.jclouds.aws.s3.commands.callables;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import static org.testng.Assert.assertEquals;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import static org.easymock.classextension.EasyMock.*;
import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Object.Metadata; import org.jclouds.aws.s3.domain.S3Object.Metadata;
import org.jclouds.http.HttpException; import org.jclouds.http.HttpException;
import org.jclouds.http.HttpHeaders; import org.jclouds.http.HttpHeaders;
import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpResponse;
import static org.testng.Assert.assertEquals; import org.testng.annotations.AfterTest;
import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
/** /**
@ -43,13 +46,13 @@ public class ParseObjectFromHeadersAndHttpContentTest {
ParseObjectFromHeadersAndHttpContent callable; ParseObjectFromHeadersAndHttpContent callable;
ParseMetadataFromHeaders metadataParser; ParseMetadataFromHeaders metadataParser;
@BeforeMethod @BeforeTest
void setUp() { void setUp() {
metadataParser = createMock(ParseMetadataFromHeaders.class); metadataParser = createMock(ParseMetadataFromHeaders.class);
callable = new ParseObjectFromHeadersAndHttpContent(metadataParser); callable = new ParseObjectFromHeadersAndHttpContent(metadataParser);
} }
@AfterMethod @AfterTest
void tearDown() { void tearDown() {
callable = null; callable = null;
} }

121
aws/s3/core/src/test/java/org/jclouds/aws/s3/internal/BaseS3MapTest.java Normal file
View File

@ -0,0 +1,121 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.internal;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.createNiceMock;
import static org.easymock.classextension.EasyMock.replay;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.Test;
/**
*
* Tests retry logic.
*
* @author Adrian Cole
*/
@Test(groups = { "unit" }, testName = "s3.BaseS3MapTest")
public class BaseS3MapTest {
class MockBaseS3Map extends BaseS3Map<String> {
public MockBaseS3Map() {
super(createNiceMock(S3Connection.class), "bucket");
}
public Set<java.util.Map.Entry<String, String>> entrySet() {
return null;
}
public String get(Object key) {
return null;
}
public String put(String key, String value) {
return null;
}
public void putAll(Map<? extends String, ? extends String> t) {
}
public String remove(Object key) {
return null;
}
public Collection<String> values() {
return null;
}
}
public void testIfNotFoundRetryOtherwiseAddToSet() throws InterruptedException,
ExecutionException, TimeoutException {
BaseS3Map<String> map = new MockBaseS3Map();
Future<S3Object> futureObject = createMock(Future.class);
S3Object object = createNiceMock(S3Object.class);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
S3Object.NOT_FOUND);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
object);
replay(futureObject);
Set<S3Object> objects = new HashSet<S3Object>();
long time = System.currentTimeMillis();
map.ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
// should have retried once
assert System.currentTimeMillis() >= time + map.requestRetryMilliseconds;
assert objects.contains(object);
assert !objects.contains(S3Object.NOT_FOUND);
}
public void testIfNotFoundRetryOtherwiseAddToSetButNeverGetsIt() throws InterruptedException,
ExecutionException, TimeoutException {
BaseS3Map<String> map = new MockBaseS3Map();
Future<S3Object> futureObject = createMock(Future.class);
S3Object object = createNiceMock(S3Object.class);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
S3Object.NOT_FOUND).atLeastOnce();
replay(futureObject);
Set<S3Object> objects = new HashSet<S3Object>();
long time = System.currentTimeMillis();
map.ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
// should have retried thrice
assert System.currentTimeMillis() >= time + map.requestRetryMilliseconds * 3;
assert !objects.contains(object);
assert !objects.contains(S3Object.NOT_FOUND);
}
}

13
aws/s3/core/src/test/java/org/jclouds/aws/s3/xml/S3ParserFactoryTest.java
View File

@ -23,12 +23,13 @@
*/ */
package org.jclouds.aws.s3.xml; package org.jclouds.aws.s3.xml;
import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Injector; import com.google.inject.Injector;
import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/** /**
* @author Adrian Cole * @author Adrian Cole
@ -39,13 +40,13 @@ public class S3ParserFactoryTest {
Injector injector = null; Injector injector = null;
S3ParserFactory parserFactory = null; S3ParserFactory parserFactory = null;
@BeforeMethod @BeforeTest
void setUpInjector() { void setUpInjector() {
injector = Guice.createInjector(new S3ParserModule()); injector = Guice.createInjector(new S3ParserModule());
parserFactory = injector.getInstance(S3ParserFactory.class); parserFactory = injector.getInstance(S3ParserFactory.class);
} }
@AfterMethod @AfterTest
void tearDownInjector() { void tearDownInjector() {
parserFactory = null; parserFactory = null;
injector = null; injector = null;

147
aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java
View File

@ -40,7 +40,6 @@ import org.jets3t.service.acl.AccessControlList;
import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3BucketLoggingStatus; import org.jets3t.service.model.S3BucketLoggingStatus;
import org.jets3t.service.model.S3Object; import org.jets3t.service.model.S3Object;
import org.jets3t.service.model.S3Owner;
import org.jets3t.service.security.AWSCredentials; import org.jets3t.service.security.AWSCredentials;
import com.google.inject.Module; import com.google.inject.Module;
@ -63,16 +62,18 @@ public class JCloudsS3Service extends S3Service {
/** /**
* Initializes a JClouds context to S3. * Initializes a JClouds context to S3.
* *
* @param awsCredentials - credentials to access S3 * @param awsCredentials
* @param modules - Module that configures a FutureHttpClient, if not specified, * - credentials to access S3
* default is URLFetchServiceClientModule * @param modules
* - Module that configures a FutureHttpClient, if not specified, default is
* URLFetchServiceClientModule
* @throws S3ServiceException * @throws S3ServiceException
*/ */
protected JCloudsS3Service(AWSCredentials awsCredentials, Module... modules) protected JCloudsS3Service(AWSCredentials awsCredentials, Module... modules)
throws S3ServiceException { throws S3ServiceException {
super(awsCredentials); super(awsCredentials);
context = S3ContextFactory.createS3Context(awsCredentials context = S3ContextFactory.createS3Context(awsCredentials.getAccessKey(), awsCredentials
.getAccessKey(), awsCredentials.getSecretKey(), modules); .getSecretKey(), modules);
connection = context.getConnection(); connection = context.getConnection();
} }
@ -87,20 +88,17 @@ public class JCloudsS3Service extends S3Service {
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override
protected Map copyObjectImpl(String sourceBucketName, protected Map copyObjectImpl(String sourceBucketName, String sourceObjectKey,
String sourceObjectKey, String destinationBucketName, String destinationBucketName, String destinationObjectKey, AccessControlList acl,
String destinationObjectKey, AccessControlList acl, Map destinationMetadata, Calendar ifModifiedSince, Calendar ifUnmodifiedSince,
Map destinationMetadata, Calendar ifModifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags) throws S3ServiceException {
Calendar ifUnmodifiedSince, String[] ifMatchTags,
String[] ifNoneMatchTags) throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected S3Bucket createBucketImpl(String bucketName, String location, protected S3Bucket createBucketImpl(String bucketName, String location, AccessControlList acl)
AccessControlList acl) throws S3ServiceException throws S3ServiceException {
{
if (location != null) if (location != null)
throw new UnsupportedOperationException("Bucket location is not yet supported"); throw new UnsupportedOperationException("Bucket location is not yet supported");
if (acl != null) if (acl != null)
@ -111,9 +109,8 @@ public class JCloudsS3Service extends S3Service {
// Bucket created. // Bucket created.
} }
} catch (Exception e) { } catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e); Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException( throw new S3ServiceException("error creating bucket: " + bucketName, e);
"error creating bucket: " + bucketName, e);
} }
return new S3Bucket(bucketName); return new S3Bucket(bucketName);
} }
@ -124,15 +121,13 @@ public class JCloudsS3Service extends S3Service {
* @see S3Connection#deleteBucketIfEmpty(String) * @see S3Connection#deleteBucketIfEmpty(String)
*/ */
@Override @Override
protected void deleteBucketImpl(String bucketName) protected void deleteBucketImpl(String bucketName) throws S3ServiceException {
throws S3ServiceException {
try { try {
connection.deleteBucketIfEmpty(bucketName).get( connection.deleteBucketIfEmpty(bucketName).get(requestTimeoutMilliseconds,
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); TimeUnit.MILLISECONDS);
} catch (Exception e) { } catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e); Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException( throw new S3ServiceException("error deleting bucket: " + bucketName, e);
"error deleting bucket: " + bucketName, e);
} }
} }
@ -142,28 +137,25 @@ public class JCloudsS3Service extends S3Service {
* @see S3Connection#deleteObject(String, String) * @see S3Connection#deleteObject(String, String)
*/ */
@Override @Override
protected void deleteObjectImpl(String bucketName, String objectKey) protected void deleteObjectImpl(String bucketName, String objectKey) throws S3ServiceException {
throws S3ServiceException {
try { try {
connection.deleteObject(bucketName, objectKey).get( connection.deleteObject(bucketName, objectKey).get(requestTimeoutMilliseconds,
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); TimeUnit.MILLISECONDS);
} catch (Exception e) { } catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e); Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format( throw new S3ServiceException(String.format("error deleting object: %1$s:%2$s", bucketName,
"error deleting object: %1$s:%2$s", bucketName, objectKey), e); objectKey), e);
} }
} }
@Override @Override
protected AccessControlList getBucketAclImpl(String bucketName) protected AccessControlList getBucketAclImpl(String bucketName) throws S3ServiceException {
throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected String getBucketLocationImpl(String bucketName) protected String getBucketLocationImpl(String bucketName) throws S3ServiceException {
throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@ -176,18 +168,16 @@ public class JCloudsS3Service extends S3Service {
} }
@Override @Override
protected AccessControlList getObjectAclImpl(String bucketName, protected AccessControlList getObjectAclImpl(String bucketName, String objectKey)
String objectKey) throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected S3Object getObjectDetailsImpl(String bucketName, protected S3Object getObjectDetailsImpl(String bucketName, String objectKey,
String objectKey, Calendar ifModifiedSince, Calendar ifModifiedSince, Calendar ifUnmodifiedSince, String[] ifMatchTags,
Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags) throws S3ServiceException {
String[] ifNoneMatchTags) throws S3ServiceException
{
try { try {
if (ifModifiedSince != null) if (ifModifiedSince != null)
throw new IllegalArgumentException("ifModifiedSince"); throw new IllegalArgumentException("ifModifiedSince");
@ -198,43 +188,37 @@ public class JCloudsS3Service extends S3Service {
if (ifNoneMatchTags != null) if (ifNoneMatchTags != null)
throw new IllegalArgumentException("ifNoneMatchTags"); throw new IllegalArgumentException("ifNoneMatchTags");
return Util.convertObjectHead( return Util.convertObjectHead(connection.headObject(bucketName, objectKey).get());
connection.headObject(bucketName, objectKey).get());
} catch (Exception e) { } catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e); Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format( throw new S3ServiceException(String.format("error retrieving object head: %1$s:%2$s",
"error retrieving object head: %1$s:%2$s", bucketName, objectKey), e); bucketName, objectKey), e);
} }
} }
@Override @Override
protected S3Object getObjectImpl(String bucketName, String objectKey, protected S3Object getObjectImpl(String bucketName, String objectKey, Calendar ifModifiedSince,
Calendar ifModifiedSince, Calendar ifUnmodifiedSince, Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags,
String[] ifMatchTags, String[] ifNoneMatchTags, Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException {
Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException
{
try { try {
GetObjectOptions options = Util.convertOptions( GetObjectOptions options = Util.convertOptions(ifModifiedSince, ifUnmodifiedSince,
ifModifiedSince, ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags); ifMatchTags, ifNoneMatchTags);
return Util.convertObject( return Util.convertObject(connection.getObject(bucketName, objectKey, options).get());
connection.getObject(bucketName, objectKey, options).get());
} catch (Exception e) { } catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e); Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format( throw new S3ServiceException(String.format("error retrieving object: %1$s:%2$s",
"error retrieving object: %1$s:%2$s", bucketName, objectKey), e); bucketName, objectKey), e);
} }
} }
@Override @Override
public boolean isBucketAccessible(String bucketName) public boolean isBucketAccessible(String bucketName) throws S3ServiceException {
throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected boolean isRequesterPaysBucketImpl(String bucketName) protected boolean isRequesterPaysBucketImpl(String bucketName) throws S3ServiceException {
throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@ -243,27 +227,25 @@ public class JCloudsS3Service extends S3Service {
protected S3Bucket[] listAllBucketsImpl() throws S3ServiceException { protected S3Bucket[] listAllBucketsImpl() throws S3ServiceException {
try { try {
List<org.jclouds.aws.s3.domain.S3Bucket.Metadata> jcBucketList = connection List<org.jclouds.aws.s3.domain.S3Bucket.Metadata> jcBucketList = connection
.listOwnedBuckets().get(requestTimeoutMilliseconds, .listOwnedBuckets().get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
TimeUnit.MILLISECONDS);
return Util.convertBuckets(jcBucketList); return Util.convertBuckets(jcBucketList);
} catch (Exception e) { } catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e); Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error listing buckets", e); throw new S3ServiceException("error listing buckets", e);
} }
} }
@Override @Override
protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, String prefix,
String prefix, String delimiter, long maxListingLength, String delimiter, long maxListingLength, String priorLastKey, boolean completeListing)
String priorLastKey, boolean completeListing)
throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected S3Object[] listObjectsImpl(String bucketName, String prefix, protected S3Object[] listObjectsImpl(String bucketName, String prefix, String delimiter,
String delimiter, long maxListingLength) throws S3ServiceException { long maxListingLength) throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@ -277,40 +259,33 @@ public class JCloudsS3Service extends S3Service {
} }
@Override @Override
protected void putObjectAclImpl(String bucketName, String objectKey, protected void putObjectAclImpl(String bucketName, String objectKey, AccessControlList acl)
AccessControlList acl) throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected S3Object putObjectImpl(String bucketName, S3Object object) protected S3Object putObjectImpl(String bucketName, S3Object object) throws S3ServiceException {
throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
return null; return null;
} }
@Override @Override
protected void setBucketLoggingStatusImpl(String bucketName, protected void setBucketLoggingStatusImpl(String bucketName, S3BucketLoggingStatus status)
S3BucketLoggingStatus status) throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected void setRequesterPaysBucketImpl(String bucketName, protected void setRequesterPaysBucketImpl(String bucketName, boolean requesterPays)
boolean requesterPays) throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override
protected S3Owner getAccountOwnerImpl() throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
} }

27
aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java
View File

@ -23,6 +23,19 @@
*/ */
package org.jclouds.aws.s3.jets3t; package org.jclouds.aws.s3.jets3t;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3IntegrationTest; import org.jclouds.aws.s3.S3IntegrationTest;
import org.jclouds.aws.s3.config.StubS3ConnectionModule; import org.jclouds.aws.s3.config.StubS3ConnectionModule;
@ -31,26 +44,12 @@ import org.jets3t.service.S3ServiceException;
import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object; import org.jets3t.service.model.S3Object;
import org.jets3t.service.security.AWSCredentials; import org.jets3t.service.security.AWSCredentials;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.fail;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap; import com.google.common.collect.Multimap;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/** /**
* Tests to cover JCloudsS3Service * Tests to cover JCloudsS3Service
* *

78
aws/s3/perftest/pom.xml
View File

@ -1,42 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!-- <!--
$HeadURL$ $HeadURL$ $Revision$ $Date$ Copyright (C) 2009 Adrian Cole
$Revision$ <adrian@jclouds.org>
$Date$
Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
==================================================================== ====================================================================
Licensed to the Apache Software Foundation (ASF) under one Licensed to the Apache Software Foundation (ASF) under one or more
or more contributor license agreements. See the NOTICE file contributor license agreements. See the NOTICE file distributed with
distributed with this work for additional information this work for additional information regarding copyright ownership.
regarding copyright ownership. The ASF licenses this file The ASF licenses this file to you under the Apache License, Version
to you under the Apache License, Version 2.0 (the 2.0 (the "License"); you may not use this file except in compliance
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0.html http://www.apache.org/licenses/LICENSE-2.0.html Unless required by
applicable law or agreed to in writing, software distributed under the
Unless required by applicable law or agreed to in writing, License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
software distributed under the License is distributed on an CONDITIONS OF ANY KIND, either express or implied. See the License for
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY the specific language governing permissions and limitations under the
KIND, either express or implied. See the License for the License.
specific language governing permissions and limitations
under the License.
==================================================================== ====================================================================
--> -->
<!-- <!--
Note that the code that these performance tests are evaluated against exist in src/main. The code in that Note that the code that these performance tests are evaluated against
location was copied straight from Amazon's website. There have been small modifications to make unit testing exist in src/main. The code in that location was copied straight from
possible. That code exists with the following license: Amazon's website. There have been small modifications to make unit
testing possible. That code exists with the following license: This
This software code is made available "AS IS" without warranties of any software code is made available "AS IS" without warranties of any
kind. You may copy, display, modify and redistribute the software kind. You may copy, display, modify and redistribute the software code
code either by itself or as incorporated into your code; provided that either by itself or as incorporated into your code; provided that you
you do not remove any proprietary notices. Your use of this software do not remove any proprietary notices. Your use of this software code
code is at your own risk and you waive any claim against Amazon is at your own risk and you waive any claim against Amazon Digital
Digital Services, Inc. or its affiliates with respect to your use of Services, Inc. or its affiliates with respect to your use of this
this software code. (c) 2006 Amazon Digital Services, Inc. or its software code. (c) 2006 Amazon Digital Services, Inc. or its
affiliates. affiliates.
--> -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@ -90,14 +84,16 @@
<developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk/s3core/perftest</developerConnection> <developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk/s3core/perftest</developerConnection>
<url>http://jclouds.googlecode.com/svn/trunk/s3core/perftest</url> <url>http://jclouds.googlecode.com/svn/trunk/s3core/perftest</url>
</scm> </scm>
<build>
<repositories> <plugins>
<!-- For Amazon S3 artifacts --> <plugin>
<repository> <groupId>org.apache.maven.plugins</groupId>
<name>jets3t</name> <artifactId>maven-surefire-plugin</artifactId>
<id>jets3t</id> <configuration>
<url>http://jets3t.s3.amazonaws.com/maven2</url> <parallel>classes</parallel>
</repository> </configuration>
</repositories> </plugin>
</plugins>
</build>
</project> </project>

131
aws/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformanceLiveTest.java
View File

@ -1,131 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.jclouds.aws.s3.reference.S3Constants;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import java.io.File;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
/**
* Runs operations that amazon s3 sample code is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformanceLiveTest", groups = {"live"})
public class AmazonPerformanceLiveTest extends BasePerformance {
private AWSAuthConnection amzClient;
@Override
@BeforeTest
@Parameters({S3Constants.PROPERTY_AWS_ACCESSKEYID,
S3Constants.PROPERTY_AWS_SECRETACCESSKEY})
protected void setUpCredentials(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey, ITestContext context) throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey,
false);
}
@Override
@Test(enabled = false)
public void testPutFileSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
com.amazon.s3.S3Object object = new com.amazon.s3.S3Object(data, null);
Map<String, List<String>> headers = new TreeMap<String, List<String>>();
headers
.put("Content-Type", Arrays
.asList(new String[]{contentType}));
return amzClient.put(bucket, key, object, headers).connection
.getResponseMessage() != null;
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
}

94
aws/s3/perftest/src/test/java/com/amazon/s3/BaseJCloudsPerformance.java
View File

@ -1,94 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.TimeUnit;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public abstract class BaseJCloudsPerformance extends BasePerformance {
// boolean get
// (
// int id) throws Exception {
// S3Bucket s3Bucket = new S3Bucket();
// s3Bucket.setName(bucketPrefix + "-jclouds-puts");
// org.jclouds.aws.s3.domain.S3Object object = new
// org.jclouds.aws.s3.domain.S3Object();
// object.setKey(id + "");
// //object.setContentType("text/plain");
// object.setContentType("application/octetstream");
// //object.setData("this is a test");
// object.setData(test);
// return clientProvider.getObject(s3Bucket,
// object.getKey()).get(120,TimeUnit.SECONDS) !=
// org.jclouds.aws.s3.domain.S3Object.NOT_FOUND;
// }
@Override
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
object.getMetadata().setSize(data.available());
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
}

252
aws/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java
View File

@ -1,252 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import com.google.inject.Provider;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.testng.ITestContext;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test(groups = {"live"})
public abstract class BasePerformance extends S3IntegrationTest {
protected boolean debugEnabled() {
return false;
}
protected static int LOOP_COUNT = 100;
protected ExecutorService exec;
protected final String BUCKET_BYTES = bucketPrefix + "-bytes";
protected final String BUCKET_INPUTSTREAM = bucketPrefix + "-inputstream";
protected final String BUCKET_STRING = bucketPrefix + "-string";
protected final String BUCKET_FILE = bucketPrefix + "-file";
protected final String[] BUCKETS = {BUCKET_BYTES, BUCKET_INPUTSTREAM,
BUCKET_STRING, BUCKET_FILE};
protected PutBytesCallable putBytesCallable;
protected PutFileCallable putFileCallable;
protected PutInputStreamCallable putInputStreamCallable;
protected PutStringCallable putStringCallable;
protected CompletionService<Boolean> completer;
@BeforeTest
protected void setUpCallables() {
putBytesCallable = new PutBytesCallable();
putFileCallable = new PutFileCallable();
putInputStreamCallable = new PutInputStreamCallable();
putStringCallable = new PutStringCallable();
exec = Executors.newCachedThreadPool();
completer = new ExecutorCompletionService<Boolean>(exec);
}
@Override
@BeforeTest
protected void setUpClient(ITestContext context) throws Exception {
super.setUpClient(context);
for (String bucket : BUCKETS) {
client.putBucketIfNotExists(bucket).get(10, TimeUnit.SECONDS);
}
}
@AfterTest
protected void tearDownExecutor() throws Exception {
exec.shutdownNow();
exec = null;
}
@Test(enabled = true)
public void testPutBytesSerial() throws Exception {
doSerial(putBytesCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutBytesParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putBytesCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutFileSerial() throws Exception {
doSerial(putFileCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutFileParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putFileCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutInputStreamSerial() throws Exception {
doSerial(putInputStreamCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putInputStreamCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutStringSerial() throws Exception {
doSerial(putStringCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutStringParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putStringCallable, LOOP_COUNT);
}
private void doSerial(Provider<Callable<Boolean>> provider, int loopCount)
throws Exception, ExecutionException {
for (int i = 0; i < loopCount; i++)
assert provider.get().call();
}
private void doParallel(Provider<Callable<Boolean>> provider, int loopCount)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < loopCount; i++)
completer.submit(provider.get());
for (int i = 0; i < loopCount; i++)
assert completer.take().get(10, TimeUnit.SECONDS);
}
class PutBytesCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected byte[] test = new byte[1024 * 2];
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putByteArray(BUCKET_BYTES, key.getAndIncrement()
+ "", test, "application/octetstring");
}
};
}
}
class PutFileCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml");
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putFile(BUCKET_FILE, key.getAndIncrement() + "",
file, "text/xml");
}
};
}
}
class PutInputStreamCallable extends PutBytesCallable {
final AtomicInteger key = new AtomicInteger(0);
@Override
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putInputStream(BUCKET_INPUTSTREAM, key
.getAndIncrement()
+ "", new ByteArrayInputStream(test),
"application/octetstring");
}
};
}
}
class PutStringCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected String testString = "hello world!";
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putString(BUCKET_STRING, key.getAndIncrement() + "",
testString, "text/plain");
}
};
}
}
protected abstract boolean putByteArray(String bucket, String key,
byte[] data, String contentType) throws Exception;
protected abstract boolean putFile(String bucket, String key, File data,
String contentType) throws Exception;
protected abstract boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception;
protected abstract boolean putString(String bucket, String key,
String data, String contentType) throws Exception;
// private class BucketDeleter implements Callable<Boolean> {
// private BucketDeleter(S3Bucket bucket) {
// this.bucket = bucket;
// }
//
// private S3Bucket bucket;
//
// @Override
// public String toString() {
// return "BucketDeleter{" + "bucket=" + bucket + '}';
// }
//
// public Boolean call() throws Exception {
// bucket =
// clientProvider.get(10,TimeUnit.SECONDS).getBucket(bucket).get(10,TimeUnit.SECONDS);
// List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
// for (org.jclouds.aws.s3.domain.S3Object object : bucket
// .getContents()) {
// deletes.add(clientProvider.get(10,TimeUnit.SECONDS).deleteObject(bucket,
// object.getKey()));
// }
// for (Future<Boolean> isdeleted : deletes)
// assert isdeleted.get(10,TimeUnit.SECONDS) :
// String.format("failed to delete %1$ss",
// isdeleted);
// return
// clientProvider.get(10,TimeUnit.SECONDS).deleteBucket(bucket).get(10,TimeUnit.SECONDS);
// }
// }
}

107
aws/s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java
View File

@ -23,6 +23,9 @@
*/ */
package com.amazon.s3; package com.amazon.s3;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import java.util.Date; import java.util.Date;
import java.util.Locale; import java.util.Locale;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
@ -41,9 +44,6 @@ import org.testng.annotations.Test;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Injector; import com.google.inject.Injector;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
/* /*
* TODO: Scrap any non-DateService references (eg Joda & Amazon) if/when * TODO: Scrap any non-DateService references (eg Joda & Amazon) if/when
* we confirm that the DateService is fast enough. * we confirm that the DateService is fast enough.
@ -62,6 +62,7 @@ public class DateServiceTest extends PerformanceTest {
DateService dateService = i.getInstance(DateService.class); DateService dateService = i.getInstance(DateService.class);
private TestData[] testData; private TestData[] testData;
class TestData { class TestData {
public final String iso8601DateString; public final String iso8601DateString;
public final String rfc822DateString; public final String rfc822DateString;
@ -76,22 +77,24 @@ public class DateServiceTest extends PerformanceTest {
private long startTime; private long startTime;
public DateServiceTest() { public DateServiceTest() {
// Constant time test values, each TestData item must contain matching times! // Constant time test values, each TestData item must contain matching times!
testData = new TestData[] { testData = new TestData[] {
new TestData("2009-03-12T02:00:07.000Z", "Thu, 12 Mar 2009 02:00:07 GMT", new Date(1236823207000l)), new TestData("2009-03-12T02:00:07.000Z", "Thu, 12 Mar 2009 02:00:07 GMT", new Date(
new TestData("2009-03-14T04:00:07.000Z", "Sat, 14 Mar 2009 04:00:07 GMT", new Date(1237003207000l)), 1236823207000l)),
new TestData("2009-03-16T06:00:07.000Z", "Mon, 16 Mar 2009 06:00:07 GMT", new Date(1237183207000l)), new TestData("2009-03-14T04:00:07.000Z", "Sat, 14 Mar 2009 04:00:07 GMT", new Date(
new TestData("2009-03-18T08:00:07.000Z", "Wed, 18 Mar 2009 08:00:07 GMT", new Date(1237363207000l)), 1237003207000l)),
new TestData("2009-03-20T10:00:07.000Z", "Fri, 20 Mar 2009 10:00:07 GMT", new Date(1237543207000l)) new TestData("2009-03-16T06:00:07.000Z", "Mon, 16 Mar 2009 06:00:07 GMT", new Date(
}; 1237183207000l)),
new TestData("2009-03-18T08:00:07.000Z", "Wed, 18 Mar 2009 08:00:07 GMT", new Date(
1237363207000l)),
new TestData("2009-03-20T10:00:07.000Z", "Fri, 20 Mar 2009 10:00:07 GMT", new Date(
1237543207000l)) };
} }
// Joda items for performance comparisons // Joda items for performance comparisons
private DateTimeFormatter headerDateFormat = DateTimeFormat private DateTimeFormatter headerDateFormat = DateTimeFormat.forPattern(
.forPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.US); "EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.US);
private DateTime jodaParseIso8601(String toParse) { private DateTime jodaParseIso8601(String toParse) {
return new DateTime(toParse); return new DateTime(toParse);
@ -109,79 +112,63 @@ public class DateServiceTest extends PerformanceTest {
return headerDateFormat.print(date.withZone(DateTimeZone.UTC)); return headerDateFormat.print(date.withZone(DateTimeZone.UTC));
} }
private void startClock() { private void startClock() {
startTime = System.currentTimeMillis(); startTime = System.currentTimeMillis();
} }
private void printElapsedClockTime(String testName) { private void printElapsedClockTime(String testName) {
System.out.println(testName + " took " + System.out.println(testName + " took " + (System.currentTimeMillis() - startTime) + "ms for "
(System.currentTimeMillis() - startTime) + "ms for " + LOOP_COUNT + " loops");
+ LOOP_COUNT+ " loops");
} }
@Test @Test
public void testIso8601DateParse() throws ExecutionException, public void testIso8601DateParse() throws ExecutionException, InterruptedException {
InterruptedException
{
DateTime dsDate = dateService.iso8601DateParse(testData[0].iso8601DateString); DateTime dsDate = dateService.iso8601DateParse(testData[0].iso8601DateString);
assertEquals(dsDate.toDate(), testData[0].date); assertEquals(dsDate.toDate(), testData[0].date);
} }
@Test @Test
public void testRfc822DateParse() throws ExecutionException, public void testRfc822DateParse() throws ExecutionException, InterruptedException {
InterruptedException
{
DateTime dsDate = dateService.rfc822DateParse(testData[0].rfc822DateString); DateTime dsDate = dateService.rfc822DateParse(testData[0].rfc822DateString);
assertEquals(dsDate.toDate(), testData[0].date); assertEquals(dsDate.toDate(), testData[0].date);
} }
@Test @Test
public void testIso8601DateFormat() throws ExecutionException, public void testIso8601DateFormat() throws ExecutionException, InterruptedException {
InterruptedException
{
String dsString = dateService.iso8601DateFormat(testData[0].date); String dsString = dateService.iso8601DateFormat(testData[0].date);
assertEquals(dsString, testData[0].iso8601DateString); assertEquals(dsString, testData[0].iso8601DateString);
} }
@Test @Test
public void testRfc822DateFormat() throws ExecutionException, public void testRfc822DateFormat() throws ExecutionException, InterruptedException {
InterruptedException
{
String dsString = dateService.rfc822DateFormat(testData[0].date); String dsString = dateService.rfc822DateFormat(testData[0].date);
assertEquals(dsString, testData[0].rfc822DateString); assertEquals(dsString, testData[0].rfc822DateString);
} }
@Test @Test
void testIso8601DateFormatResponseTime() throws ExecutionException, void testIso8601DateFormatResponseTime() throws ExecutionException, InterruptedException {
InterruptedException {
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
dateService.iso8601DateFormat(); dateService.iso8601DateFormat();
} }
@Test @Test
void testRfc822DateFormatResponseTime() throws ExecutionException, void testRfc822DateFormatResponseTime() throws ExecutionException, InterruptedException {
InterruptedException {
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
dateService.rfc822DateFormat(); dateService.rfc822DateFormat();
} }
@Test @Test
void testFormatIso8601DateInParallel() throws InterruptedException, void testFormatIso8601DateInParallel() throws InterruptedException, ExecutionException {
ExecutionException { CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
startClock(); startClock();
for (int i = 0; i < LOOP_COUNT; i++) { for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length]; final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() { completer.submit(new Callable<Boolean>() {
public Boolean call() throws ExecutionException, public Boolean call() throws ExecutionException, InterruptedException {
InterruptedException
{
String dsString = dateService.iso8601DateFormat(myData.date); String dsString = dateService.iso8601DateFormat(myData.date);
/* /*
* Comment-in the assert below to test thread safety. * Comment-in the assert below to test thread safety. Comment it out to test
* Comment it out to test performance * performance
*/ */
assertEquals(dsString, myData.iso8601DateString); assertEquals(dsString, myData.iso8601DateString);
return true; return true;
@ -194,10 +181,8 @@ public class DateServiceTest extends PerformanceTest {
} }
@Test @Test
void testFormatAmazonDateInParallel() throws InterruptedException, void testFormatAmazonDateInParallel() throws InterruptedException, ExecutionException {
ExecutionException { CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
startClock(); startClock();
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<Boolean>() { completer.submit(new Callable<Boolean>() {
@ -212,10 +197,8 @@ public class DateServiceTest extends PerformanceTest {
} }
@Test @Test
void testFormatJodaDateInParallel() throws InterruptedException, void testFormatJodaDateInParallel() throws InterruptedException, ExecutionException {
ExecutionException { CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
startClock(); startClock();
for (int i = 0; i < LOOP_COUNT; i++) { for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length]; final TestData myData = testData[i % testData.length];
@ -233,8 +216,7 @@ public class DateServiceTest extends PerformanceTest {
} }
@Test @Test
void testIso8601ParseDateSerialResponseTime() throws ExecutionException, void testIso8601ParseDateSerialResponseTime() throws ExecutionException, InterruptedException {
InterruptedException {
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
dateService.iso8601DateParse(testData[0].iso8601DateString); dateService.iso8601DateParse(testData[0].iso8601DateString);
} }
@ -246,16 +228,13 @@ public class DateServiceTest extends PerformanceTest {
} }
@Test @Test
void testParseIso8601DateInParallel() throws InterruptedException, void testParseIso8601DateInParallel() throws InterruptedException, ExecutionException {
ExecutionException { CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
startClock(); startClock();
for (int i = 0; i < LOOP_COUNT; i++) { for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length]; final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() { completer.submit(new Callable<Boolean>() {
public Boolean call() throws ExecutionException, public Boolean call() throws ExecutionException, InterruptedException {
InterruptedException {
DateTime dsDate = dateService.iso8601DateParse(myData.iso8601DateString); DateTime dsDate = dateService.iso8601DateParse(myData.iso8601DateString);
assertEquals(dsDate.toDate(), myData.date); assertEquals(dsDate.toDate(), myData.date);
return true; return true;
@ -268,10 +247,8 @@ public class DateServiceTest extends PerformanceTest {
} }
@Test @Test
void testParseAmazonDateInParallel() throws InterruptedException, void testParseAmazonDateInParallel() throws InterruptedException, ExecutionException {
ExecutionException { CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
startClock(); startClock();
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<Boolean>() { completer.submit(new Callable<Boolean>() {
@ -286,10 +263,8 @@ public class DateServiceTest extends PerformanceTest {
} }
@Test @Test
void testParseJodaDateInParallel() throws InterruptedException, void testParseJodaDateInParallel() throws InterruptedException, ExecutionException {
ExecutionException { CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
startClock(); startClock();
for (int i = 0; i < LOOP_COUNT; i++) { for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length]; final TestData myData = testData[i % testData.length];

130
aws/s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformanceLiveTest.java
View File

@ -1,130 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.jets3t.service.S3Service;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.security.AWSCredentials;
import org.testng.ITestContext;
import org.testng.annotations.Test;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.ExecutionException;
/**
* Runs operations that jets3t is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformanceLiveTest", groups = {"live"})
public class Jets3tPerformanceLiveTest extends BasePerformance {
private S3Service jetClient;
@Override
protected void setUpCredentials(String AWSAccessKeyId, String AWSSecretAccessKey, ITestContext context)
throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
jetClient = new RestS3Service(new AWSCredentials(AWSAccessKeyId,
AWSSecretAccessKey));
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(
key);
object.setContentType(contentType);
object.setDataInputFile(data);
object.setContentLength(data.length());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(
key);
object.setContentType(contentType);
object.setDataInputStream(data);
object.setContentLength(data.available());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
}

73
aws/s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java
View File

@ -1,73 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.testng.annotations.Test;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
/**
* Compares performance of encryption operations.
*
* @author Adrian Cole
*
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest")
public class S3UtilsTest extends org.jclouds.aws.s3.S3UtilsTest {
@Test(dataProvider = "hmacsha1")
void testAmazonSampleDigestSerialResponseTime(byte[] key, String message, String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException {
for (int i = 0; i < 10000; i++)
testAmazonSampleDigest(key, message, base64Digest);
}
@Test(dataProvider = "hmacsha1")
public void testAmazonSampleDigest(byte[] key, String message, String base64Digest) {
String encoded = Utils.encode(new String(key), message, false);
assert encoded.equals(base64Digest);
}
@Test(dataProvider = "hmacsha1")
void testAmazonSampleDigestParallelResponseTime(final byte[] key, final String message, final String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException, InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
for (int i = 0; i < 10000; i++)
completer.submit(new Callable<Boolean>() {
public Boolean call() {
try {
testAmazonSampleDigest(key, message, base64Digest);
return true;
} catch (Exception e) {
return false;
}
}
});
for (int i = 0; i < 10000; i++) assert completer.take().get();
}
}

124
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/AmazonPerformanceLiveTest.java Normal file
View File

@ -0,0 +1,124 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.File;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import org.jclouds.aws.s3.reference.S3Constants;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import com.amazon.s3.AWSAuthConnection;
/**
* Runs operations that amazon s3 sample code is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformanceLiveTest", groups = { "live" })
public class AmazonPerformanceLiveTest extends BasePerformance {
private AWSAuthConnection amzClient;
@Override
@BeforeTest
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpCredentials(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey, ITestContext context) throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey, false);
}
@Override
@Test(enabled = false)
public void testPutFileSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
com.amazon.s3.S3Object object = new com.amazon.s3.S3Object(data, null);
Map<String, List<String>> headers = new TreeMap<String, List<String>>();
headers.put("Content-Type", Arrays.asList(new String[] { contentType }));
return amzClient.put(bucket, key, object, headers).connection.getResponseMessage() != null;
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
}

90
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/BaseJCloudsPerformance.java Normal file
View File

@ -0,0 +1,90 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.TimeUnit;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public abstract class BaseJCloudsPerformance extends BasePerformance {
// boolean get
// (
// int id) throws Exception {
// S3Bucket s3Bucket = new S3Bucket();
// s3Bucket.setName(bucketPrefix + "-jclouds-puts");
// org.jclouds.aws.s3.domain.S3Object object = new
// org.jclouds.aws.s3.domain.S3Object();
// object.setKey(id + "");
// //object.setContentType("text/plain");
// object.setContentType("application/octetstream");
// //object.setData("this is a test");
// object.setData(test);
// return clientProvider.getObject(s3Bucket,
// object.getKey()).get(120,TimeUnit.SECONDS) !=
// org.jclouds.aws.s3.domain.S3Object.NOT_FOUND;
// }
@Override
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
object.getMetadata().setSize(data.available());
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
}

226
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/BasePerformance.java Normal file
View File

@ -0,0 +1,226 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.inject.Provider;
/**
* Tests relative performance of S3 functions.
*
* @author Adrian Cole
*/
@Test(groups = { "live" })
public abstract class BasePerformance extends S3IntegrationTest {
@Override
protected boolean debugEnabled() {
return false;
}
protected static int LOOP_COUNT = 100;
protected ExecutorService exec;
protected CompletionService<Boolean> completer;
@BeforeTest
protected void setUpCallables() {
exec = Executors.newCachedThreadPool();
completer = new ExecutorCompletionService<Boolean>(exec);
}
@AfterTest
protected void tearDownExecutor() throws Exception {
exec.shutdownNow();
exec = null;
}
@Test(enabled = true)
public void testPutBytesSerial() throws Exception {
doSerial(new PutBytesCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutBytesParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutBytesCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutFileSerial() throws Exception {
doSerial(new PutFileCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutFileParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutFileCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutInputStreamSerial() throws Exception {
doSerial(new PutInputStreamCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutInputStreamCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutStringSerial() throws Exception {
doSerial(new PutStringCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutStringParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutStringCallable(this.bucketName), LOOP_COUNT);
}
private void doSerial(Provider<Callable<Boolean>> provider, int loopCount) throws Exception,
ExecutionException {
for (int i = 0; i < loopCount; i++)
assert provider.get().call();
}
private void doParallel(Provider<Callable<Boolean>> provider, int loopCount)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < loopCount; i++)
completer.submit(provider.get());
for (int i = 0; i < loopCount; i++)
assert completer.take().get(10, TimeUnit.SECONDS);
}
class PutBytesCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected byte[] test = new byte[1024 * 2];
private final String bucketName;
public PutBytesCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
String bucketName2 = bucketName;
return putByteArray(bucketName2, key.getAndIncrement() + "", test,
"application/octetstring");
}
};
}
}
class PutFileCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml");
private final String bucketName;
public PutFileCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putFile(bucketName, key.getAndIncrement() + "", file, "text/xml");
}
};
}
}
class PutInputStreamCallable extends PutBytesCallable {
final AtomicInteger key = new AtomicInteger(0);
private final String bucketName;
public PutInputStreamCallable(String bucketName) {
super(bucketName);
this.bucketName = bucketName;
}
@Override
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putInputStream(bucketName, key.getAndIncrement() + "",
new ByteArrayInputStream(test), "application/octetstring");
}
};
}
}
class PutStringCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected String testString = "hello world!";
private final String bucketName;
public PutStringCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putString(bucketName, key.getAndIncrement() + "", testString, "text/plain");
}
};
}
}
protected abstract boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception;
protected abstract boolean putFile(String bucket, String key, File data, String contentType)
throws Exception;
protected abstract boolean putInputStream(String bucket, String key, InputStream data,
String contentType) throws Exception;
protected abstract boolean putString(String bucket, String key, String data, String contentType)
throws Exception;
}

6
aws/s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformanceLiveTest.java → aws/s3/perftest/src/test/java/org/jclouds/aws/s3/JCloudsNioPerformanceLiveTest.java
View File

@ -21,14 +21,14 @@
* under the License. * under the License.
* ==================================================================== * ====================================================================
*/ */
package com.amazon.s3; package org.jclouds.aws.s3;
import java.util.Properties;
import com.google.inject.Module;
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule; import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.util.Properties; import com.google.inject.Module;
@Test(sequential = true, testName = "s3.JCloudsNioPerformanceLiveTest", groups = {"live"}) @Test(sequential = true, testName = "s3.JCloudsNioPerformanceLiveTest", groups = {"live"})
public class JCloudsNioPerformanceLiveTest extends BaseJCloudsPerformance { public class JCloudsNioPerformanceLiveTest extends BaseJCloudsPerformance {

3
aws/s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformanceLiveTest.java → aws/s3/perftest/src/test/java/org/jclouds/aws/s3/JCloudsPerformanceLiveTest.java
View File

@ -21,8 +21,7 @@
* under the License. * under the License.
* ==================================================================== * ====================================================================
*/ */
package com.amazon.s3; package org.jclouds.aws.s3;
import org.testng.annotations.Test; import org.testng.annotations.Test;
/** /**

127
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/Jets3tPerformanceLiveTest.java Normal file
View File

@ -0,0 +1,127 @@
package org.jclouds.aws.s3;
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.ExecutionException;
import org.jclouds.aws.s3.reference.S3Constants;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.security.AWSCredentials;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
/**
* Runs operations that jets3t is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformanceLiveTest", groups = { "live" })
public class Jets3tPerformanceLiveTest extends BasePerformance {
private S3Service jetClient;
@BeforeTest
public void setUpJetS3t(ITestContext testContext) throws S3ServiceException {
String AWSAccessKeyId = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID);
String AWSSecretAccessKey = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
jetClient = new RestS3Service(new AWSCredentials(AWSAccessKeyId, AWSSecretAccessKey));
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
object.setContentType(contentType);
object.setDataInputFile(data);
object.setContentLength(data.length());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
object.setContentType(contentType);
object.setDataInputStream(data);
object.setContentLength(data.available());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
}

27
core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java
View File

@ -35,28 +35,15 @@ import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.jclouds.http.HttpFutureCommand; import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpResponse;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@Test @Test(groups = { "unit" })
public class ReturnStringIf200Test { public class ReturnStringIf200Test {
private HttpFutureCommand.ResponseCallable<String> callable = null;
@BeforeMethod
void setUp() {
callable = new ReturnStringIf200();
}
@AfterMethod
void tearDown() {
callable = null;
}
@Test @Test
public void testExceptionWhenNoContentOn200() throws ExecutionException, public void testExceptionWhenNoContentOn200() throws ExecutionException, InterruptedException,
InterruptedException, TimeoutException, IOException { TimeoutException, IOException {
HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
HttpResponse response = createMock(HttpResponse.class); HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce(); expect(response.getStatusCode()).andReturn(200).atLeastOnce();
expect(response.getContent()).andReturn(null); expect(response.getContent()).andReturn(null);
@ -71,8 +58,9 @@ public class ReturnStringIf200Test {
} }
@Test @Test
public void testExceptionWhenIOExceptionOn200() throws ExecutionException, public void testExceptionWhenIOExceptionOn200() throws ExecutionException, InterruptedException,
InterruptedException, TimeoutException, IOException { TimeoutException, IOException {
HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
HttpResponse response = createMock(HttpResponse.class); HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce(); expect(response.getStatusCode()).andReturn(200).atLeastOnce();
RuntimeException exception = new RuntimeException("bad"); RuntimeException exception = new RuntimeException("bad");
@ -89,6 +77,7 @@ public class ReturnStringIf200Test {
@Test @Test
public void testResponseOk() throws Exception { public void testResponseOk() throws Exception {
HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
HttpResponse response = createMock(HttpResponse.class); HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce(); expect(response.getStatusCode()).andReturn(200).atLeastOnce();
expect(response.getContent()).andReturn(IOUtils.toInputStream("hello")); expect(response.getContent()).andReturn(IOUtils.toInputStream("hello"));

76
core/src/test/java/org/jclouds/test/testng/UnitTestStatusListener.java Normal file
View File

@ -0,0 +1,76 @@
package org.jclouds.test.testng;
import java.util.concurrent.atomic.AtomicInteger;
import org.testng.IClass;
import org.testng.ITestContext;
import org.testng.ITestListener;
import org.testng.ITestResult;
/**
* adapted from the following class:
*
* @see org.UnitTestStatusListener.test.testng.UnitTestTestNGListener
* @author Adrian Cole
* @author dpospisi@redhat.com
* @author Mircea.Markus@jboss.com
*/
public class UnitTestStatusListener implements ITestListener {
/**
* Holds test classes actually running in all threads.
*/
private ThreadLocal<IClass> threadTestClass = new ThreadLocal<IClass>();
private ThreadLocal<Long> threadTestStart = new ThreadLocal<Long>();
private AtomicInteger failed = new AtomicInteger(0);
private AtomicInteger succeded = new AtomicInteger(0);
private AtomicInteger skipped = new AtomicInteger(0);
public void onTestStart(ITestResult res) {
System.out.println("Starting test " + getTestDesc(res));
threadTestClass.set(res.getTestClass());
threadTestStart.set(System.currentTimeMillis());
}
synchronized public void onTestSuccess(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " succeeded: "
+ (System.currentTimeMillis() - threadTestStart.get()) + "ms");
succeded.incrementAndGet();
printStatus();
}
synchronized public void onTestFailure(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " failed.");
failed.incrementAndGet();
printStatus();
}
synchronized public void onTestSkipped(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " skipped.");
skipped.incrementAndGet();
printStatus();
}
public void onTestFailedButWithinSuccessPercentage(ITestResult arg0) {
}
public void onStart(ITestContext arg0) {
}
public void onFinish(ITestContext arg0) {
}
private String getThreadId() {
return "[" + Thread.currentThread().getName() + "]";
}
private String getTestDesc(ITestResult res) {
return res.getMethod().getMethodName() + "(" + res.getTestClass().getName() + ")";
}
private void printStatus() {
System.out.println("Test suite progress: tests succeeded: " + succeded.get() + ", failed: "
+ failed.get() + ", skipped: " + skipped.get() + ".");
}
}

10
project/pom.xml
View File

@ -169,6 +169,8 @@
<goal>test</goal> <goal>test</goal>
</goals> </goals>
<configuration> <configuration>
<parallel>tests</parallel>
<threadCount>5</threadCount>
<!-- note that the groups/excluded groups don't work due to some problem <!-- note that the groups/excluded groups don't work due to some problem
in surefire or testng. instead, we have to exclude via file path in surefire or testng. instead, we have to exclude via file path
<groups>integration</groups> <groups>integration</groups>
@ -183,6 +185,8 @@
</execution> </execution>
</executions> </executions>
<configuration> <configuration>
<parallel>methods</parallel>
<threadCount>5</threadCount>
<!-- note that the groups/excluded groups don't work due to some problem <!-- note that the groups/excluded groups don't work due to some problem
in surefire or testng. instead, we have to exclude via file path in surefire or testng. instead, we have to exclude via file path
<groups>unit,performance</groups> <groups>unit,performance</groups>
@ -191,6 +195,12 @@
<exclude>**/*IntegrationTest.java</exclude> <exclude>**/*IntegrationTest.java</exclude>
<exclude>**/*LiveTest.java</exclude> <exclude>**/*LiveTest.java</exclude>
</excludes> </excludes>
<properties>
<property>
<name>listener</name>
<value>org.jclouds.test.testng.UnitTestStatusListener</value>
</property>
</properties>
</configuration> </configuration>
</plugin> </plugin>
<!-- Make sure we generate src jars too --> <!-- Make sure we generate src jars too -->