Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

made tests parallel and fixed some transient errors in BaseS3Map 

git-svn-id: http://jclouds.googlecode.com/svn/trunk@834 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-05-22 15:35:57 +00:00
parent 6b9c2431d6
commit f8da659695
29 changed files with 2068 additions and 2025 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

380
aws/s3/core/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java
View File

@ -24,9 +24,19 @@
package org.jclouds.aws.s3.internal; package org.jclouds.aws.s3.internal;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted; import java.io.FileNotFoundException;
import com.google.inject.name.Named; import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3Connection; import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Map; import org.jclouds.aws.s3.S3Map;
import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Bucket;
@ -34,203 +44,213 @@ import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.reference.S3Constants; import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.util.Utils; import org.jclouds.util.Utils;
import java.io.FileNotFoundException; import com.google.common.annotations.VisibleForTesting;
import java.io.IOException; import com.google.inject.Inject;
import java.util.*; import com.google.inject.assistedinject.Assisted;
import java.util.concurrent.ExecutionException; import com.google.inject.name.Named;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/** /**
* Implements core Map functionality with an {@link S3Connection} * Implements core Map functionality with an {@link S3Connection}
* <p/> * <p/>
* All commands will wait a maximum of ${jclouds.s3.map.timeout} milliseconds to * All commands will wait a maximum of ${jclouds.s3.map.timeout} milliseconds to complete before
* complete before throwing an exception. * throwing an exception.
* *
* @author Adrian Cole * @author Adrian Cole
* @param <V> * @param <V>
* value of the map * value of the map
*/ */
public abstract class BaseS3Map<V> implements S3Map<String, V> { public abstract class BaseS3Map<V> implements S3Map<String, V> {
protected final S3Connection connection; protected final S3Connection connection;
protected final String bucket; protected final String bucket;
/** /**
* maximum duration of an S3 Request * maximum duration of an S3 Request
*/ */
@Inject(optional = true) @Inject(optional = true)
@Named(S3Constants.PROPERTY_S3_MAP_TIMEOUT) @Named(S3Constants.PROPERTY_S3_MAP_TIMEOUT)
protected long requestTimeoutMilliseconds = 10000; protected long requestTimeoutMilliseconds = 10000;
@Inject /**
public BaseS3Map(S3Connection connection, @Assisted String bucket) { * time to pause before retrying a transient failure
this.connection = checkNotNull(connection, "connection"); */
this.bucket = checkNotNull(bucket, "bucketName"); @Inject(optional = true)
} @Named(S3Constants.PROPERTY_S3_MAP_RETRY)
protected long requestRetryMilliseconds = 10;
/** @Inject
* {@inheritDoc} public BaseS3Map(S3Connection connection, @Assisted String bucket) {
* <p/> this.connection = checkNotNull(connection, "connection");
* This returns the number of keys in the {@link S3Bucket} this.bucket = checkNotNull(bucket, "bucketName");
* }
* @see S3Bucket#getContents()
*/
public int size() {
try {
S3Bucket bucket = refreshBucket();
Set<S3Object.Metadata> contents = bucket.getContents();
return contents.size();
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting size of bucketName"
+ bucket, e);
}
}
protected boolean containsMd5(byte[] md5) throws InterruptedException, /**
ExecutionException, TimeoutException { * {@inheritDoc}
for (S3Object.Metadata metadata : refreshBucket().getContents()) { * <p/>
if (Arrays.equals(md5, metadata.getMd5())) * This returns the number of keys in the {@link S3Bucket}
return true; *
} * @see S3Bucket#getContents()
return false; */
} public int size() {
try {
S3Bucket bucket = refreshBucket();
Set<S3Object.Metadata> contents = bucket.getContents();
return contents.size();
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting size of bucketName" + bucket, e);
}
}
protected byte[] getMd5(Object value) throws IOException, protected boolean containsMd5(byte[] md5) throws InterruptedException, ExecutionException,
FileNotFoundException, InterruptedException, ExecutionException,
TimeoutException { TimeoutException {
S3Object object = null; for (S3Object.Metadata metadata : refreshBucket().getContents()) {
if (value instanceof S3Object) { if (Arrays.equals(md5, metadata.getMd5()))
object = (S3Object) value; return true;
} else { }
object = new S3Object("dummy", value); return false;
} }
if (object.getMetadata().getMd5() == null)
object.generateMd5();
return object.getMetadata().getMd5();
}
/** protected byte[] getMd5(Object value) throws IOException, FileNotFoundException,
* attempts asynchronous gets on all objects. InterruptedException, ExecutionException, TimeoutException {
* S3Object object = null;
* @see S3Connection#getObject(String, String) if (value instanceof S3Object) {
*/ object = (S3Object) value;
protected Set<S3Object> getAllObjects() { } else {
Set<S3Object> objects = new HashSet<S3Object>(); object = new S3Object("dummy", value);
Set<Future<S3Object>> futureObjects = new HashSet<Future<S3Object>>(); }
for (String key : keySet()) { if (object.getMetadata().getMd5() == null)
futureObjects.add(connection.getObject(bucket, key)); object.generateMd5();
} return object.getMetadata().getMd5();
for (Future<S3Object> futureObject : futureObjects) { }
S3Object object = null;
try { /**
object = futureObject.get(requestTimeoutMilliseconds, * attempts asynchronous gets on all objects.
TimeUnit.MILLISECONDS); *
} catch (Exception e) { * @see S3Connection#getObject(String, String)
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e); */
throw new S3RuntimeException(String.format( protected Set<S3Object> getAllObjects() {
"Error getting value from bucket %1$s", bucket), e); Set<S3Object> objects = new HashSet<S3Object>();
Set<Future<S3Object>> futureObjects = new HashSet<Future<S3Object>>();
for (String key : keySet()) {
futureObjects.add(connection.getObject(bucket, key));
}
for (Future<S3Object> futureObject : futureObjects) {
try {
ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format("Error getting value from bucket %1$s",
bucket), e);
}
}
return objects;
}
@VisibleForTesting
void ifNotFoundRetryOtherwiseAddToSet(Future<S3Object> futureObject, Set<S3Object> objects)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < 3; i++) {
S3Object object = futureObject.get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
if (object != S3Object.NOT_FOUND) {
objects.add(object);
break;
} else {
Thread.sleep(requestRetryMilliseconds);
}
}
}
/**
* {@inheritDoc}
* <p/>
* Note that if value is an instance of InputStream, it will be read and closed following this
* method. To reuse data from InputStreams, pass {@link java.io.InputStream}s inside
* {@link S3Object}s
*/
public boolean containsValue(Object value) {
try {
byte[] md5 = getMd5(value);
return containsMd5(md5);
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for ETAG of value: [%2$s] in bucketName:%1$s", bucket, value), e);
}
}
public static class S3RuntimeException extends RuntimeException {
private static final long serialVersionUID = 1L;
S3RuntimeException(String s) {
super(s);
}
public S3RuntimeException(String s, Throwable throwable) {
super(s, throwable);
}
}
public void clear() {
try {
List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
for (String key : keySet()) {
deletes.add(connection.deleteObject(bucket, key));
}
for (Future<Boolean> isdeleted : deletes)
if (!isdeleted.get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)) {
throw new S3RuntimeException("failed to delete entry");
} }
if (object != S3Object.NOT_FOUND) } catch (Exception e) {
objects.add(object); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
} throw new S3RuntimeException("Error clearing bucketName" + bucket, e);
return objects; }
} }
/** protected S3Bucket refreshBucket() throws InterruptedException, ExecutionException,
* {@inheritDoc} TimeoutException {
* <p/> S3Bucket currentBucket = connection.listBucket(bucket).get(requestTimeoutMilliseconds,
* Note that if value is an instance of InputStream, it will be read and TimeUnit.MILLISECONDS);
* closed following this method. To reuse data from InputStreams, pass if (currentBucket == S3Bucket.NOT_FOUND)
* {@link java.io.InputStream}s inside {@link S3Object}s throw new S3RuntimeException("bucketName not found: " + bucket);
*/ else
public boolean containsValue(Object value) { return currentBucket;
try { }
byte[] md5 = getMd5(value);
return containsMd5(md5);
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for ETAG of value: [%2$s] in bucketName:%1$s",
bucket, value), e);
}
}
public static class S3RuntimeException extends RuntimeException { public Set<String> keySet() {
private static final long serialVersionUID = 1L; try {
Set<String> keys = new HashSet<String>();
for (S3Object.Metadata object : refreshBucket().getContents())
keys.add(object.getKey());
return keys;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting keys in bucketName: " + bucket, e);
}
}
S3RuntimeException(String s) { public boolean containsKey(Object key) {
super(s); try {
} return connection.headObject(bucket, key.toString()).get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS) != S3Object.Metadata.NOT_FOUND;
} catch (Exception e) {
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format("Error searching for %1$s:%2$s", bucket, key),
e);
}
}
public S3RuntimeException(String s, Throwable throwable) { public boolean isEmpty() {
super(s, throwable); return keySet().size() == 0;
} }
}
public void clear() { public S3Bucket getBucket() {
try { try {
List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>(); return refreshBucket();
for (String key : keySet()) { } catch (Exception e) {
deletes.add(connection.deleteObject(bucket, key)); Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
} throw new S3RuntimeException("Error getting bucketName" + bucket, e);
for (Future<Boolean> isdeleted : deletes) }
if (!isdeleted.get(requestTimeoutMilliseconds, }
TimeUnit.MILLISECONDS)) {
throw new S3RuntimeException("failed to delete entry");
}
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucketName" + bucket, e);
}
}
protected S3Bucket refreshBucket() throws InterruptedException,
ExecutionException, TimeoutException {
S3Bucket currentBucket = connection.listBucket(bucket).get(
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
if (currentBucket == S3Bucket.NOT_FOUND)
throw new S3RuntimeException("bucketName not found: " + bucket);
else
return currentBucket;
}
public Set<String> keySet() {
try {
Set<String> keys = new HashSet<String>();
for (S3Object.Metadata object : refreshBucket().getContents())
keys.add(object.getKey());
return keys;
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting keys in bucketName: "
+ bucket, e);
}
}
public boolean containsKey(Object key) {
try {
return connection.headObject(bucket, key.toString()).get(
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS) != S3Object.Metadata.NOT_FOUND;
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for %1$s:%2$s", bucket, key), e);
}
}
public boolean isEmpty() {
return keySet().size() == 0;
}
public S3Bucket getBucket() {
try {
return refreshBucket();
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting bucketName" + bucket, e);
}
}
} }

17
aws/s3/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java
View File

@ -33,12 +33,15 @@ import org.jclouds.http.HttpConstants;
*/ */
public interface S3Constants extends HttpConstants, PoolConstants, S3Headers { public interface S3Constants extends HttpConstants, PoolConstants, S3Headers {
public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey"; public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey";
public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid"; public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid";
/** /**
* longest time a single Map operation can take before throwing an * longest time a single Map operation can take before throwing an exception.
* exception. */
*/ public static final String PROPERTY_S3_MAP_TIMEOUT = "jclouds.s3.map.timeout";
public static final String PROPERTY_S3_MAP_TIMEOUT = "jclouds.s3.map.timeout"; /**
* time to pause before retrying a transient failure
*/
public static final String PROPERTY_S3_MAP_RETRY = "jclouds.s3.map.retry";
} }

34
aws/s3/core/src/test/java/org/jclouds/aws/PerformanceTest.java
View File

@ -23,31 +23,31 @@
*/ */
package org.jclouds.aws; package org.jclouds.aws;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
/** /**
* // TODO: Adrian: Document this! * // TODO: Adrian: Document this!
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = "performance", sequential = true, testName = "s3.PerformanceTest") @Test(groups = "performance")
public class PerformanceTest { public class PerformanceTest {
protected static int LOOP_COUNT = 1000; protected static int LOOP_COUNT = 1000;
protected ExecutorService exec; protected ExecutorService exec;
@BeforeMethod @BeforeTest
public void setupExecutorService() { public void setupExecutorService() {
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
} }
@AfterMethod @AfterTest
public void teardownExecutorService() { public void teardownExecutorService() {
exec.shutdownNow(); exec.shutdownNow();
exec = null; exec = null;
} }
} }

81
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3ConnectionIntegrationTest.java
View File

@ -24,71 +24,62 @@
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.util.S3Utils;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import java.io.InputStream; import java.io.InputStream;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.util.S3Utils;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
/** /**
* Tests connection by listing all the buckets and their size * Tests connection by listing all the buckets and their size
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(testName = "s3.S3ConnectionIntegrationTest") @Test(testName = "s3.S3ConnectionIntegrationTest")
public class S3ConnectionIntegrationTest extends S3IntegrationTest { public class S3ConnectionIntegrationTest extends S3IntegrationTest {
@Test(groups = {"integration"}) @Test(groups = { "integration" })
void testListBuckets() throws Exception { void testListBuckets() throws Exception {
List<S3Bucket.Metadata> myBuckets = client.listOwnedBuckets().get(10, client.listOwnedBuckets().get(10, TimeUnit.SECONDS);
TimeUnit.SECONDS); }
for (S3Bucket.Metadata bucket : myBuckets) {
context.createInputStreamMap(bucket.getName()).size();
}
}
private static final String sysHttpStreamUrl = System private static final String sysHttpStreamUrl = System.getProperty("jclouds.s3.httpstream.url");
.getProperty("jclouds.s3.httpstream.url"); private static final String sysHttpStreamMd5 = System.getProperty("jclouds.s3.httpstream.md5");
private static final String sysHttpStreamMd5 = System
.getProperty("jclouds.s3.httpstream.md5");
@Test(groups = {"integration"}) @Test(groups = { "integration" })
@Parameters({"jclouds.s3.httpstream.url", "jclouds.s3.httpstream.md5"}) @Parameters( { "jclouds.s3.httpstream.url", "jclouds.s3.httpstream.md5" })
public void testCopyUrl(@Optional String httpStreamUrl, public void testCopyUrl(@Optional String httpStreamUrl, @Optional String httpStreamMd5)
@Optional String httpStreamMd5) throws Exception { throws Exception {
httpStreamUrl = checkNotNull(httpStreamUrl != null ? httpStreamUrl httpStreamUrl = checkNotNull(httpStreamUrl != null ? httpStreamUrl : sysHttpStreamUrl,
: sysHttpStreamUrl, "httpStreamUrl"); "httpStreamUrl");
httpStreamMd5 = checkNotNull(httpStreamMd5 != null ? httpStreamMd5 httpStreamMd5 = checkNotNull(httpStreamMd5 != null ? httpStreamMd5 : sysHttpStreamMd5,
: sysHttpStreamMd5, "httpStreamMd5"); "httpStreamMd5");
String bucketName = bucketPrefix + "tcu"; String bucketName = bucketPrefix + "tcu";
createBucketAndEnsureEmpty(bucketName); createBucketAndEnsureEmpty(bucketName);
String key = "hello"; String key = "hello";
URL url = new URL(httpStreamUrl); URL url = new URL(httpStreamUrl);
byte[] md5 = S3Utils byte[] md5 = S3Utils.fromHexString(httpStreamMd5);
.fromHexString(httpStreamMd5);
URLConnection connection = url.openConnection(); URLConnection connection = url.openConnection();
int length = connection.getContentLength(); int length = connection.getContentLength();
InputStream input = connection.getInputStream(); InputStream input = connection.getInputStream();
S3Object object = new S3Object(key, input); S3Object object = new S3Object(key, input);
object.setContentLength(length); object.setContentLength(length);
object.getMetadata().setMd5(md5); object.getMetadata().setMd5(md5);
object.getMetadata().setSize(length); object.getMetadata().setSize(length);
byte[] newMd5 = client.putObject(bucketName, object).get(30, byte[] newMd5 = client.putObject(bucketName, object).get(30, TimeUnit.SECONDS);
TimeUnit.SECONDS); assertEquals(newMd5, md5);
assertEquals(newMd5, md5); }
}
} }

368
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java
View File

@ -24,7 +24,27 @@
package org.jclouds.aws.s3; package org.jclouds.aws.s3;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import com.google.inject.Module; import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.ConsoleHandler;
import java.util.logging.Formatter;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import org.jclouds.aws.s3.config.StubS3ConnectionModule; import org.jclouds.aws.s3.config.StubS3ConnectionModule;
import org.jclouds.aws.s3.domain.S3Bucket; import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.aws.s3.domain.S3Object;
@ -32,215 +52,201 @@ import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.aws.s3.util.S3Utils; import org.jclouds.aws.s3.util.S3Utils;
import org.jclouds.http.HttpConstants; import org.jclouds.http.HttpConstants;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule; import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import static org.testng.Assert.assertEquals;
import org.testng.ITestContext; import org.testng.ITestContext;
import org.testng.annotations.*; import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import java.io.IOException; import com.google.inject.Module;
import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.*;
import java.util.logging.*;
import java.util.logging.Formatter;
public class S3IntegrationTest { public class S3IntegrationTest {
protected static final String TEST_STRING = "<apples><apple name=\"fuji\"></apple> </apples>"; protected static final String TEST_STRING = "<apples><apple name=\"fuji\"></apple> </apples>";
protected byte[] goodMd5; protected byte[] goodMd5;
protected byte[] badMd5; protected byte[] badMd5;
protected String bucketName; protected String bucketName;
protected void createBucketAndEnsureEmpty(String sourceBucket) protected void createBucketAndEnsureEmpty(String sourceBucket) throws InterruptedException,
throws InterruptedException, ExecutionException, TimeoutException { ExecutionException, TimeoutException {
client.putBucketIfNotExists(sourceBucket).get(10, TimeUnit.SECONDS); client.deleteBucketIfEmpty(sourceBucket).get(10, TimeUnit.SECONDS);
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS) client.putBucketIfNotExists(sourceBucket).get(10, TimeUnit.SECONDS);
.getContents().size(), 0); assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS).getContents().size(),
} 0, "bucket " + sourceBucket + "wasn't empty");
}
protected void addObjectToBucket(String sourceBucket, String key) protected void addObjectToBucket(String sourceBucket, String key) throws InterruptedException,
throws InterruptedException, ExecutionException, TimeoutException, ExecutionException, TimeoutException, IOException {
IOException { S3Object sourceObject = new S3Object(key);
S3Object sourceObject = new S3Object(key); sourceObject.getMetadata().setContentType("text/xml");
sourceObject.getMetadata().setContentType("text/xml"); sourceObject.setData(TEST_STRING);
sourceObject.setData(TEST_STRING); addObjectToBucket(sourceBucket, sourceObject);
addObjectToBucket(sourceBucket, sourceObject); }
}
protected void addObjectToBucket(String sourceBucket, S3Object object) protected void addObjectToBucket(String sourceBucket, S3Object object)
throws InterruptedException, ExecutionException, TimeoutException, throws InterruptedException, ExecutionException, TimeoutException, IOException {
IOException { client.putObject(sourceBucket, object).get(10, TimeUnit.SECONDS);
client.putObject(sourceBucket, object).get(10, TimeUnit.SECONDS); }
}
protected S3Object validateContent(String sourceBucket, String key) protected S3Object validateContent(String sourceBucket, String key) throws InterruptedException,
throws InterruptedException, ExecutionException, TimeoutException, ExecutionException, TimeoutException, IOException {
IOException { assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS).getContents().size(),
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS) 1);
.getContents().size(), 1); S3Object newObject = client.getObject(sourceBucket, key).get(10, TimeUnit.SECONDS);
S3Object newObject = client.getObject(sourceBucket, key).get(10, assert newObject != S3Object.NOT_FOUND;
TimeUnit.SECONDS); assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING);
assert newObject != S3Object.NOT_FOUND; return newObject;
assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING); }
return newObject;
}
@BeforeClass(groups = {"integration", "live"}) @BeforeClass(groups = { "integration", "live" })
void enableDebug() { void enableDebug() {
if (debugEnabled()) { if (debugEnabled()) {
Handler HANDLER = new ConsoleHandler() { Handler HANDLER = new ConsoleHandler() {
{ {
setLevel(Level.ALL); setLevel(Level.ALL);
setFormatter(new Formatter() { setFormatter(new Formatter() {
@Override @Override
public String format(LogRecord record) { public String format(LogRecord record) {
return String.format( return String.format("[%tT %-7s] [%-7s] [%s]: %s %s\n", new Date(record
"[%tT %-7s] [%-7s] [%s]: %s %s\n", .getMillis()), record.getLevel(), Thread.currentThread().getName(),
new Date(record.getMillis()), record record.getLoggerName(), record.getMessage(),
.getLevel(), Thread.currentThread() record.getThrown() == null ? "" : record.getThrown());
.getName(), record.getLoggerName(), }
record.getMessage(), });
record.getThrown() == null ? "" : record }
.getThrown()); };
} Logger guiceLogger = Logger.getLogger("org.jclouds");
}); guiceLogger.addHandler(HANDLER);
} guiceLogger.setLevel(Level.ALL);
}; }
Logger guiceLogger = Logger.getLogger("org.jclouds"); }
guiceLogger.addHandler(HANDLER);
guiceLogger.setLevel(Level.ALL);
}
}
protected S3Connection client; protected S3Connection client;
protected S3Context context = null; protected S3Context context = null;
protected String bucketPrefix = (System.getProperty("user.name") + "." + this protected String bucketPrefix = (System.getProperty("user.name") + "." + this.getClass()
.getClass().getSimpleName()).toLowerCase(); .getSimpleName()).toLowerCase();
private static final String sysAWSAccessKeyId = System private static final String sysAWSAccessKeyId = System
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID); .getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
private static final String sysAWSSecretAccessKey = System private static final String sysAWSSecretAccessKey = System
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY); .getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
@BeforeClass(inheritGroups = false, groups = {"integration", "live"}) @BeforeClass(inheritGroups = false, groups = { "integration", "live" })
@Parameters({S3Constants.PROPERTY_AWS_ACCESSKEYID, @Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
S3Constants.PROPERTY_AWS_SECRETACCESSKEY}) protected void setUpCredentials(@Optional String AWSAccessKeyId,
protected void setUpCredentials(@Optional String AWSAccessKeyId, @Optional String AWSSecretAccessKey, ITestContext testContext) throws Exception {
@Optional String AWSSecretAccessKey, ITestContext testContext) throws Exception { AWSAccessKeyId = AWSAccessKeyId != null ? AWSAccessKeyId : sysAWSAccessKeyId;
AWSAccessKeyId = AWSAccessKeyId != null ? AWSAccessKeyId AWSSecretAccessKey = AWSSecretAccessKey != null ? AWSSecretAccessKey : sysAWSSecretAccessKey;
: sysAWSAccessKeyId; if (AWSAccessKeyId != null)
AWSSecretAccessKey = AWSSecretAccessKey != null ? AWSSecretAccessKey testContext.setAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId);
: sysAWSSecretAccessKey; if (AWSSecretAccessKey != null)
if (AWSAccessKeyId != null) testContext.setAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey);
testContext.setAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId); }
if (AWSSecretAccessKey != null)
testContext.setAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey);
}
@BeforeClass(dependsOnMethods = {"setUpCredentials"}, groups = {"integration", "live"}) @BeforeClass(dependsOnMethods = { "setUpCredentials" }, groups = { "integration", "live" })
protected void setUpClient(ITestContext testContext) throws Exception { protected void setUpClient(ITestContext testContext) throws Exception {
if (testContext.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID) != null) { if (testContext.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID) != null) {
String AWSAccessKeyId = (String) testContext.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID); String AWSAccessKeyId = (String) testContext
String AWSSecretAccessKey = (String) testContext.getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY); .getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID);
createLiveS3Context(AWSAccessKeyId, AWSSecretAccessKey); String AWSSecretAccessKey = (String) testContext
} else { .getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
createStubS3Context(); createLiveS3Context(AWSAccessKeyId, AWSSecretAccessKey);
} } else {
client = context.getConnection(); createStubS3Context();
assert client != null; }
deleteEverything(); client = context.getConnection();
goodMd5 = S3Utils.md5(TEST_STRING); assert client != null;
badMd5 = S3Utils.md5("alf"); deleteEverything();
} goodMd5 = S3Utils.md5(TEST_STRING);
badMd5 = S3Utils.md5("alf");
}
protected void createStubS3Context() { protected void createStubS3Context() {
Properties props = new Properties(); Properties props = new Properties();
props.setProperty(S3Constants.PROPERTY_HTTP_ADDRESS, "stub"); props.setProperty(S3Constants.PROPERTY_HTTP_ADDRESS, "stub");
context = S3ContextFactory.createS3Context(props, new StubS3ConnectionModule()); context = S3ContextFactory.createS3Context(props, new StubS3ConnectionModule());
} }
protected void createLiveS3Context(String AWSAccessKeyId, String AWSSecretAccessKey) { protected void createLiveS3Context(String AWSAccessKeyId, String AWSSecretAccessKey) {
context = S3ContextFactory.createS3Context(buildS3Properties( context = S3ContextFactory.createS3Context(buildS3Properties(checkNotNull(AWSAccessKeyId,
checkNotNull(AWSAccessKeyId, "AWSAccessKeyId"), "AWSAccessKeyId"), checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey")),
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey")), createHttpModule());
createHttpModule()); }
}
@BeforeMethod(dependsOnMethods = "deleteBucket", groups = {"integration", "live"}) @BeforeMethod(dependsOnMethods = "deleteBucket", groups = { "integration", "live" })
public void setUpBucket(Method method) throws TimeoutException, ExecutionException, InterruptedException { public void setUpBucket(Method method) throws TimeoutException, ExecutionException,
bucketName = (bucketPrefix + method.getName()).toLowerCase(); InterruptedException {
createBucketAndEnsureEmpty(bucketName); bucketName = (bucketPrefix + method.getName()).toLowerCase();
} createBucketAndEnsureEmpty(bucketName);
}
@BeforeMethod(groups = {"integration", "live"}) @BeforeMethod(groups = { "integration", "live" })
@AfterMethod(groups = {"integration", "live"}) @AfterMethod(groups = { "integration", "live" })
public void deleteBucket() throws TimeoutException, ExecutionException, InterruptedException { public void deleteBucket() throws TimeoutException, ExecutionException, InterruptedException {
if (bucketName != null) if (bucketName != null)
deleteBucket(bucketName); deleteBucket(bucketName);
} }
protected boolean debugEnabled() { protected boolean debugEnabled() {
return false; return false;
} }
protected Properties buildS3Properties(String AWSAccessKeyId, protected Properties buildS3Properties(String AWSAccessKeyId, String AWSSecretAccessKey) {
String AWSSecretAccessKey) { Properties properties = new Properties(S3ContextFactory.DEFAULT_PROPERTIES);
Properties properties = new Properties( properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID, checkNotNull(AWSAccessKeyId,
S3ContextFactory.DEFAULT_PROPERTIES); "AWSAccessKeyId"));
properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID, properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, checkNotNull(
checkNotNull(AWSAccessKeyId, "AWSAccessKeyId")); AWSSecretAccessKey, "AWSSecretAccessKey"));
properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false");
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey")); properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80");
properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false"); return properties;
properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80"); }
return properties;
}
protected Module createHttpModule() { protected Module createHttpModule() {
return new JavaUrlHttpFutureCommandClientModule(); return new JavaUrlHttpFutureCommandClientModule();
} }
protected void deleteEverything() throws Exception {
try {
List<S3Bucket.Metadata> metadata = client.listOwnedBuckets().get(
10, TimeUnit.SECONDS);
for (S3Bucket.Metadata metaDatum : metadata) {
if (metaDatum.getName().startsWith(bucketPrefix.toLowerCase())) {
deleteBucket(metaDatum.getName());
}
protected void deleteEverything() throws Exception {
try {
List<S3Bucket.Metadata> metadata = client.listOwnedBuckets().get(10, TimeUnit.SECONDS);
for (S3Bucket.Metadata metaDatum : metadata) {
if (metaDatum.getName().startsWith(bucketPrefix.toLowerCase())) {
deleteBucket(metaDatum.getName());
} }
} catch (CancellationException e) {
throw e;
}
}
private void deleteBucket(String name) throws InterruptedException, ExecutionException, TimeoutException { }
if (client.bucketExists(name).get(10, TimeUnit.SECONDS)) { } catch (CancellationException e) {
List<Future<Boolean>> results = new ArrayList<Future<Boolean>>(); throw e;
}
}
S3Bucket bucket = client.listBucket(name) private void deleteBucket(String name) throws InterruptedException, ExecutionException,
.get(10, TimeUnit.SECONDS); TimeoutException {
for (S3Object.Metadata objectMeta : bucket.getContents()) { if (client.bucketExists(name).get(10, TimeUnit.SECONDS)) {
results.add(client.deleteObject(name, List<Future<Boolean>> results = new ArrayList<Future<Boolean>>();
objectMeta.getKey()));
}
Iterator<Future<Boolean>> iterator = results.iterator();
while (iterator.hasNext()) {
iterator.next().get(10, TimeUnit.SECONDS);
iterator.remove();
}
client.deleteBucketIfEmpty(name).get(10,
TimeUnit.SECONDS);
}
}
@AfterClass S3Bucket bucket = client.listBucket(name).get(10, TimeUnit.SECONDS);
protected void tearDownClient() throws Exception { for (S3Object.Metadata objectMeta : bucket.getContents()) {
deleteEverything(); results.add(client.deleteObject(name, objectMeta.getKey()));
context.close(); }
context = null; Iterator<Future<Boolean>> iterator = results.iterator();
} while (iterator.hasNext()) {
iterator.next().get(10, TimeUnit.SECONDS);
iterator.remove();
}
client.deleteBucketIfEmpty(name).get(10, TimeUnit.SECONDS);
}
}
@AfterClass
protected void tearDownClient() throws Exception {
deleteEverything();
context.close();
context = null;
}
} }

2
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3UtilsTest.java
View File

@ -44,7 +44,7 @@ import java.util.concurrent.ExecutorCompletionService;
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = "performance", sequential = true, testName = "s3.PerformanceTest") @Test(groups = "performance", sequential = true, testName = "s3.S3UtilsTest")
public class S3UtilsTest extends PerformanceTest { public class S3UtilsTest extends PerformanceTest {
@Test(dataProvider = "hmacsha1") @Test(dataProvider = "hmacsha1")

200
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java
View File

@ -23,131 +23,131 @@
*/ */
package org.jclouds.aws.s3.commands; package org.jclouds.aws.s3.commands;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.commands.options.CopyObjectOptions;
import org.jclouds.aws.s3.commands.options.GetObjectOptions;
import org.jclouds.aws.s3.commands.options.ListBucketOptions;
import org.jclouds.aws.s3.commands.options.PutBucketOptions;
import org.jclouds.aws.s3.commands.options.PutObjectOptions;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Bucket.Metadata.LocationConstraint;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap; import com.google.common.collect.Multimap;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Injector; import com.google.inject.Injector;
import com.google.inject.name.Names; import com.google.inject.name.Names;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.commands.options.*;
import org.jclouds.aws.s3.domain.S3Bucket.Metadata.LocationConstraint;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/** /**
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = {"unit"}, testName = "s3.S3CommandFactoryTest") @Test(groups = { "unit" }, testName = "s3.S3CommandFactoryTest")
public class S3CommandFactoryTest { public class S3CommandFactoryTest {
Injector injector = null; Injector injector = null;
S3CommandFactory commandFactory = null; S3CommandFactory commandFactory = null;
@BeforeMethod
void setUpInjector() {
injector = Guice.createInjector(new S3CommandsModule() {
@Override
protected void configure() {
bindConstant().annotatedWith(
Names.named("jclouds.http.address")).to("localhost");
super.configure();
}
});
commandFactory = injector.getInstance(S3CommandFactory.class);
}
@AfterMethod @BeforeTest
void tearDownInjector() { void setUpInjector() {
commandFactory = null; injector = Guice.createInjector(new S3CommandsModule() {
injector = null; @Override
} protected void configure() {
bindConstant().annotatedWith(Names.named("jclouds.http.address")).to("localhost");
super.configure();
}
});
commandFactory = injector.getInstance(S3CommandFactory.class);
}
@Test @AfterTest
void testCreateCopyObject() { void tearDownInjector() {
assert commandFactory.createCopyObject("sourcebucket", "sourceObject", commandFactory = null;
"destbucket", "destObject", CopyObjectOptions.NONE) != null; injector = null;
} }
@Test @Test
void testCreateCopyObjectOptions() { void testCreateCopyObject() {
assert commandFactory.createCopyObject("sourcebucket", "sourceObject", assert commandFactory.createCopyObject("sourcebucket", "sourceObject", "destbucket",
"destbucket", "destObject", new CopyObjectOptions()) != null; "destObject", CopyObjectOptions.NONE) != null;
} }
@Test @Test
void testCreateDeleteBucket() { void testCreateCopyObjectOptions() {
assert commandFactory.createDeleteBucket("test") != null; assert commandFactory.createCopyObject("sourcebucket", "sourceObject", "destbucket",
} "destObject", new CopyObjectOptions()) != null;
}
@Test @Test
void testCreateDeleteObject() { void testCreateDeleteBucket() {
assert commandFactory.createDeleteObject("test", "blah") != null; assert commandFactory.createDeleteBucket("test") != null;
} }
@Test @Test
void testCreateHeadBucket() { void testCreateDeleteObject() {
assert commandFactory.createHeadBucket("test") != null; assert commandFactory.createDeleteObject("test", "blah") != null;
} }
@Test @Test
void testCreatePutBucket() { void testCreateHeadBucket() {
assert commandFactory.createPutBucket("test", PutBucketOptions.NONE) != null; assert commandFactory.createHeadBucket("test") != null;
} }
@Test @Test
void testCreatePutBucketOptions() { void testCreatePutBucket() {
assert commandFactory.createPutBucket("test", PutBucketOptions.Builder assert commandFactory.createPutBucket("test", PutBucketOptions.NONE) != null;
.createIn(LocationConstraint.EU)) != null; }
}
@Test @Test
void testCreatePutObject() { void testCreatePutBucketOptions() {
S3Object.Metadata metadata = createMock(S3Object.Metadata.class); assert commandFactory.createPutBucket("test", PutBucketOptions.Builder
S3Object object = new S3Object(metadata); .createIn(LocationConstraint.EU)) != null;
expect(metadata.getSize()).andReturn(4L).atLeastOnce(); }
expect(metadata.getKey()).andReturn("rawr");
expect(metadata.getContentType()).andReturn("text/xml").atLeastOnce();
expect(metadata.getCacheControl()).andReturn("no-cache").atLeastOnce();
expect(metadata.getContentDisposition()).andReturn("disposition")
.atLeastOnce();
expect(metadata.getContentEncoding()).andReturn("encoding")
.atLeastOnce();
expect(metadata.getMd5()).andReturn("encoding".getBytes())
.atLeastOnce();
Multimap<String, String> userMdata = HashMultimap.create();
expect(metadata.getUserMetadata()).andReturn(userMdata).atLeastOnce();
replay(metadata); @Test
object.setData("<a></a>"); void testCreatePutObject() {
S3Object.Metadata metadata = createMock(S3Object.Metadata.class);
S3Object object = new S3Object(metadata);
expect(metadata.getSize()).andReturn(4L).atLeastOnce();
expect(metadata.getKey()).andReturn("rawr");
expect(metadata.getContentType()).andReturn("text/xml").atLeastOnce();
expect(metadata.getCacheControl()).andReturn("no-cache").atLeastOnce();
expect(metadata.getContentDisposition()).andReturn("disposition").atLeastOnce();
expect(metadata.getContentEncoding()).andReturn("encoding").atLeastOnce();
expect(metadata.getMd5()).andReturn("encoding".getBytes()).atLeastOnce();
Multimap<String, String> userMdata = HashMultimap.create();
expect(metadata.getUserMetadata()).andReturn(userMdata).atLeastOnce();
assert commandFactory.createPutObject("test", object, replay(metadata);
PutObjectOptions.NONE) != null; object.setData("<a></a>");
}
@Test assert commandFactory.createPutObject("test", object, PutObjectOptions.NONE) != null;
void testCreateGetObject() { }
assert commandFactory.createGetObject("test", "blah",
GetObjectOptions.NONE) != null;
}
@Test @Test
void testCreateHeadMetadata() { void testCreateGetObject() {
assert commandFactory.createHeadMetadata("test", "blah") != null; assert commandFactory.createGetObject("test", "blah", GetObjectOptions.NONE) != null;
} }
@Test @Test
void testCreateListAllMyBuckets() { void testCreateHeadMetadata() {
assert commandFactory.createGetMetadataForOwnedBuckets() != null; assert commandFactory.createHeadMetadata("test", "blah") != null;
} }
@Test @Test
void testCreateListBucket() { void testCreateListAllMyBuckets() {
assert commandFactory.createListBucket("test", ListBucketOptions.NONE) != null; assert commandFactory.createGetMetadataForOwnedBuckets() != null;
} }
@Test
void testCreateListBucket() {
assert commandFactory.createListBucket("test", ListBucketOptions.NONE) != null;
}
} }

266
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/S3ParserTest.java
View File

@ -23,8 +23,16 @@
*/ */
package org.jclouds.aws.s3.commands; package org.jclouds.aws.s3.commands;
import com.google.inject.Guice; import static org.testng.Assert.assertEquals;
import com.google.inject.Injector;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.jclouds.aws.PerformanceTest; import org.jclouds.aws.PerformanceTest;
import org.jclouds.aws.s3.domain.CanonicalUser; import org.jclouds.aws.s3.domain.CanonicalUser;
@ -38,165 +46,149 @@ import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.jclouds.http.HttpException; import org.jclouds.http.HttpException;
import org.jclouds.http.commands.callables.xml.ParseSax; import org.jclouds.http.commands.callables.xml.ParseSax;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import static org.testng.Assert.assertEquals; import org.testng.annotations.AfterTest;
import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import java.io.IOException; import com.google.inject.Guice;
import java.io.UnsupportedEncodingException; import com.google.inject.Injector;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
/** /**
* Tests parsing of S3 responses * Tests parsing of S3 responses
* *
* @author Adrian Cole * @author Adrian Cole
*/ */
@Test(groups = {"performance"}, testName = "s3.S3ParserTest") @Test(groups = { "performance" }, testName = "s3.S3ParserTest")
public class S3ParserTest extends PerformanceTest { public class S3ParserTest extends PerformanceTest {
Injector injector = null; Injector injector = null;
public static final String listAllMyBucketsResultOn200 = "<ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/callables/\"><Owner ><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID></Owner><Buckets><Bucket><Name>adrianjbosstest</Name><CreationDate>2009-03-12T02:00:07.000Z</CreationDate></Bucket><Bucket><Name>adrianjbosstest2</Name><CreationDate>2009-03-12T02:00:09.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>"; public static final String listAllMyBucketsResultOn200 = "<ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/callables/\"><Owner ><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID></Owner><Buckets><Bucket><Name>adrianjbosstest</Name><CreationDate>2009-03-12T02:00:07.000Z</CreationDate></Bucket><Bucket><Name>adrianjbosstest2</Name><CreationDate>2009-03-12T02:00:09.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>";
S3ParserFactory parserFactory = null; S3ParserFactory parserFactory = null;
@BeforeMethod @BeforeTest
protected void setUpInjector() { protected void setUpInjector() {
injector = Guice.createInjector(new S3ParserModule()); injector = Guice.createInjector(new S3ParserModule());
parserFactory = injector.getInstance(S3ParserFactory.class); parserFactory = injector.getInstance(S3ParserFactory.class);
assert parserFactory != null; assert parserFactory != null;
} }
@AfterMethod @AfterTest
protected void tearDownInjector() { protected void tearDownInjector() {
parserFactory = null; parserFactory = null;
injector = null; injector = null;
} }
@Test @Test
void testParseListAllMyBucketsSerialResponseTime() throws HttpException { void testParseListAllMyBucketsSerialResponseTime() throws HttpException {
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
runParseListAllMyBuckets(); runParseListAllMyBuckets();
} }
private List<S3Bucket.Metadata> runParseListAllMyBuckets() private List<S3Bucket.Metadata> runParseListAllMyBuckets() throws HttpException {
throws HttpException { return parserFactory.createListBucketsParser().parse(
return parserFactory.createListBucketsParser().parse( IOUtils.toInputStream(listAllMyBucketsResultOn200));
IOUtils.toInputStream(listAllMyBucketsResultOn200)); }
}
@Test @Test
void testParseListAllMyBucketsParallelResponseTime() void testParseListAllMyBucketsParallelResponseTime() throws InterruptedException,
throws InterruptedException, ExecutionException { ExecutionException {
CompletionService<List<S3Bucket.Metadata>> completer = new ExecutorCompletionService<List<S3Bucket.Metadata>>( CompletionService<List<S3Bucket.Metadata>> completer = new ExecutorCompletionService<List<S3Bucket.Metadata>>(
exec); exec);
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<List<S3Bucket.Metadata>>() { completer.submit(new Callable<List<S3Bucket.Metadata>>() {
public List<S3Bucket.Metadata> call() throws IOException, public List<S3Bucket.Metadata> call() throws IOException, SAXException, HttpException {
SAXException, HttpException { return runParseListAllMyBuckets();
return runParseListAllMyBuckets(); }
} });
}); for (int i = 0; i < LOOP_COUNT; i++)
for (int i = 0; i < LOOP_COUNT; i++) assert completer.take().get() != null;
assert completer.take().get() != null; }
}
@Test @Test
public void testCanParseListAllMyBuckets() throws HttpException { public void testCanParseListAllMyBuckets() throws HttpException {
List<S3Bucket.Metadata> s3Buckets = runParseListAllMyBuckets(); List<S3Bucket.Metadata> s3Buckets = runParseListAllMyBuckets();
S3Bucket.Metadata bucket1 = s3Buckets.get(0); S3Bucket.Metadata bucket1 = s3Buckets.get(0);
assert bucket1.getName().equals("adrianjbosstest"); assert bucket1.getName().equals("adrianjbosstest");
DateTime expectedDate1 = new DateTime("2009-03-12T02:00:07.000Z"); DateTime expectedDate1 = new DateTime("2009-03-12T02:00:07.000Z");
DateTime date1 = bucket1.getCreationDate(); DateTime date1 = bucket1.getCreationDate();
assert date1.equals(expectedDate1); assert date1.equals(expectedDate1);
S3Bucket.Metadata bucket2 = s3Buckets.get(1); S3Bucket.Metadata bucket2 = s3Buckets.get(1);
assert bucket2.getName().equals("adrianjbosstest2"); assert bucket2.getName().equals("adrianjbosstest2");
DateTime expectedDate2 = new DateTime("2009-03-12T02:00:09.000Z"); DateTime expectedDate2 = new DateTime("2009-03-12T02:00:09.000Z");
DateTime date2 = bucket2.getCreationDate(); DateTime date2 = bucket2.getCreationDate();
assert date2.equals(expectedDate2); assert date2.equals(expectedDate2);
assert s3Buckets.size() == 2; assert s3Buckets.size() == 2;
CanonicalUser owner = new CanonicalUser( CanonicalUser owner = new CanonicalUser(
"e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0"); "e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0");
assert bucket1.getOwner().equals(owner); assert bucket1.getOwner().equals(owner);
assert bucket2.getOwner().equals(owner); assert bucket2.getOwner().equals(owner);
} }
public static final String listBucketResult = "<ListBucketHandler xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>adrianjbosstest</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>3366</Key><LastModified>2009-03-12T02:00:13.000Z</LastModified><ETag>&quot;9d7bb64e8e18ee34eec06dd2cf37b766&quot;</ETag><Size>136</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketHandler>"; public static final String listBucketResult = "<ListBucketHandler xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>adrianjbosstest</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>3366</Key><LastModified>2009-03-12T02:00:13.000Z</LastModified><ETag>&quot;9d7bb64e8e18ee34eec06dd2cf37b766&quot;</ETag><Size>136</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketHandler>";
public void testCanParseListBucketResult() throws HttpException, public void testCanParseListBucketResult() throws HttpException, UnsupportedEncodingException {
UnsupportedEncodingException { S3Bucket bucket = runParseListBucketResult();
S3Bucket bucket = runParseListBucketResult(); assert !bucket.isTruncated();
assert !bucket.isTruncated(); assert bucket.getName().equals("adrianjbosstest");
assert bucket.getName().equals("adrianjbosstest"); assert bucket.getContents().size() == 1;
assert bucket.getContents().size() == 1; S3Object.Metadata object = bucket.getContents().iterator().next();
S3Object.Metadata object = bucket.getContents().iterator().next(); assert object.getKey().equals("3366");
assert object.getKey().equals("3366"); DateTime expected = new DateTime("2009-03-12T02:00:13.000Z");
DateTime expected = new DateTime("2009-03-12T02:00:13.000Z"); assert object.getLastModified().equals(expected) : String.format(
assert object.getLastModified().equals(expected) : String.format( "expected %1$s, but got %1$s", expected, object.getLastModified());
"expected %1$s, but got %1$s", expected, object assertEquals(S3Utils.toHexString(object.getMd5()), "9d7bb64e8e18ee34eec06dd2cf37b766");
.getLastModified()); assert object.getSize() == 136;
assertEquals(S3Utils.toHexString(object.getMd5()), CanonicalUser owner = new CanonicalUser(
"9d7bb64e8e18ee34eec06dd2cf37b766"); "e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0");
assert object.getSize() == 136; owner.setDisplayName("ferncam");
CanonicalUser owner = new CanonicalUser( assert object.getOwner().equals(owner);
"e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0"); assert object.getStorageClass().equals("STANDARD");
owner.setDisplayName("ferncam"); }
assert object.getOwner().equals(owner);
assert object.getStorageClass().equals("STANDARD");
}
private S3Bucket runParseListBucketResult() throws HttpException { private S3Bucket runParseListBucketResult() throws HttpException {
ParseSax<S3Bucket> parser = parserFactory.createListBucketParser(); ParseSax<S3Bucket> parser = parserFactory.createListBucketParser();
ListBucketHandler handler = (ListBucketHandler) parser.getHandler(); ListBucketHandler handler = (ListBucketHandler) parser.getHandler();
handler.setBucketName("adrianjbosstest"); handler.setBucketName("adrianjbosstest");
return parser.parse(IOUtils.toInputStream(listBucketResult)); return parser.parse(IOUtils.toInputStream(listBucketResult));
} }
public static final String successfulCopyObject200 = "<CopyObjectResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><LastModified>2009-03-19T13:23:27.000Z</LastModified><ETag>\"92836a3ea45a6984d1b4d23a747d46bb\"</ETag></CopyObjectResult>"; public static final String successfulCopyObject200 = "<CopyObjectResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><LastModified>2009-03-19T13:23:27.000Z</LastModified><ETag>\"92836a3ea45a6984d1b4d23a747d46bb\"</ETag></CopyObjectResult>";
private S3Object.Metadata runParseCopyObjectResult() throws HttpException { private S3Object.Metadata runParseCopyObjectResult() throws HttpException {
ParseSax<S3Object.Metadata> parser = parserFactory ParseSax<S3Object.Metadata> parser = parserFactory.createCopyObjectParser();
.createCopyObjectParser(); CopyObjectHandler handler = (CopyObjectHandler) parser.getHandler();
CopyObjectHandler handler = (CopyObjectHandler) parser.getHandler(); handler.setKey("adrianjbosstest");
handler.setKey("adrianjbosstest"); return parser.parse(IOUtils.toInputStream(successfulCopyObject200));
return parser.parse(IOUtils.toInputStream(successfulCopyObject200)); }
}
public void testCanParseCopyObjectResult() throws HttpException, public void testCanParseCopyObjectResult() throws HttpException, UnsupportedEncodingException {
UnsupportedEncodingException { S3Object.Metadata metadata = runParseCopyObjectResult();
S3Object.Metadata metadata = runParseCopyObjectResult(); DateTime expected = new DateTime("2009-03-19T13:23:27.000Z");
DateTime expected = new DateTime("2009-03-19T13:23:27.000Z"); assertEquals(metadata.getLastModified(), expected);
assertEquals(metadata.getLastModified(), expected); assertEquals(S3Utils.toHexString(metadata.getMd5()), "92836a3ea45a6984d1b4d23a747d46bb");
assertEquals(S3Utils.toHexString(metadata.getMd5()), assertEquals(metadata.getKey(), "adrianjbosstest");
"92836a3ea45a6984d1b4d23a747d46bb"); }
assertEquals(metadata.getKey(), "adrianjbosstest");
}
@Test @Test
void testParseListBucketResultSerialResponseTime() throws HttpException { void testParseListBucketResultSerialResponseTime() throws HttpException {
for (int i = 0; i < LOOP_COUNT; i++) for (int i = 0; i < LOOP_COUNT; i++)
runParseListBucketResult(); runParseListBucketResult();
} }
@Test @Test
void testParseListBucketResultParallelResponseTime() void testParseListBucketResultParallelResponseTime() throws InterruptedException,
throws InterruptedException, ExecutionException { ExecutionException {
CompletionService<S3Bucket> completer = new ExecutorCompletionService<S3Bucket>( CompletionService<S3Bucket> completer = new ExecutorCompletionService<S3Bucket>(exec);
exec); for (int i = 0; i < LOOP_COUNT; i++)
for (int i = 0; i < LOOP_COUNT; i++) completer.submit(new Callable<S3Bucket>() {
completer.submit(new Callable<S3Bucket>() { public S3Bucket call() throws IOException, SAXException, HttpException {
public S3Bucket call() throws IOException, SAXException, return runParseListBucketResult();
HttpException { }
return runParseListBucketResult(); });
} for (int i = 0; i < LOOP_COUNT; i++)
}); assert completer.take().get() != null;
for (int i = 0; i < LOOP_COUNT; i++) }
assert completer.take().get() != null;
}
} }

15
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/callables/ParseObjectFromHeadersAndHttpContentTest.java
View File

@ -23,16 +23,19 @@
*/ */
package org.jclouds.aws.s3.commands.callables; package org.jclouds.aws.s3.commands.callables;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import static org.testng.Assert.assertEquals;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import static org.easymock.classextension.EasyMock.*;
import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Object.Metadata; import org.jclouds.aws.s3.domain.S3Object.Metadata;
import org.jclouds.http.HttpException; import org.jclouds.http.HttpException;
import org.jclouds.http.HttpHeaders; import org.jclouds.http.HttpHeaders;
import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpResponse;
import static org.testng.Assert.assertEquals; import org.testng.annotations.AfterTest;
import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
/** /**
@ -43,13 +46,13 @@ public class ParseObjectFromHeadersAndHttpContentTest {
ParseObjectFromHeadersAndHttpContent callable; ParseObjectFromHeadersAndHttpContent callable;
ParseMetadataFromHeaders metadataParser; ParseMetadataFromHeaders metadataParser;
@BeforeMethod @BeforeTest
void setUp() { void setUp() {
metadataParser = createMock(ParseMetadataFromHeaders.class); metadataParser = createMock(ParseMetadataFromHeaders.class);
callable = new ParseObjectFromHeadersAndHttpContent(metadataParser); callable = new ParseObjectFromHeadersAndHttpContent(metadataParser);
} }
@AfterMethod @AfterTest
void tearDown() { void tearDown() {
callable = null; callable = null;
} }

121
aws/s3/core/src/test/java/org/jclouds/aws/s3/internal/BaseS3MapTest.java Normal file
View File

@ -0,0 +1,121 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.internal;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.createNiceMock;
import static org.easymock.classextension.EasyMock.replay;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.Test;
/**
*
* Tests retry logic.
*
* @author Adrian Cole
*/
@Test(groups = { "unit" }, testName = "s3.BaseS3MapTest")
public class BaseS3MapTest {
class MockBaseS3Map extends BaseS3Map<String> {
public MockBaseS3Map() {
super(createNiceMock(S3Connection.class), "bucket");
}
public Set<java.util.Map.Entry<String, String>> entrySet() {
return null;
}
public String get(Object key) {
return null;
}
public String put(String key, String value) {
return null;
}
public void putAll(Map<? extends String, ? extends String> t) {
}
public String remove(Object key) {
return null;
}
public Collection<String> values() {
return null;
}
}
public void testIfNotFoundRetryOtherwiseAddToSet() throws InterruptedException,
ExecutionException, TimeoutException {
BaseS3Map<String> map = new MockBaseS3Map();
Future<S3Object> futureObject = createMock(Future.class);
S3Object object = createNiceMock(S3Object.class);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
S3Object.NOT_FOUND);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
object);
replay(futureObject);
Set<S3Object> objects = new HashSet<S3Object>();
long time = System.currentTimeMillis();
map.ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
// should have retried once
assert System.currentTimeMillis() >= time + map.requestRetryMilliseconds;
assert objects.contains(object);
assert !objects.contains(S3Object.NOT_FOUND);
}
public void testIfNotFoundRetryOtherwiseAddToSetButNeverGetsIt() throws InterruptedException,
ExecutionException, TimeoutException {
BaseS3Map<String> map = new MockBaseS3Map();
Future<S3Object> futureObject = createMock(Future.class);
S3Object object = createNiceMock(S3Object.class);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
S3Object.NOT_FOUND).atLeastOnce();
replay(futureObject);
Set<S3Object> objects = new HashSet<S3Object>();
long time = System.currentTimeMillis();
map.ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
// should have retried thrice
assert System.currentTimeMillis() >= time + map.requestRetryMilliseconds * 3;
assert !objects.contains(object);
assert !objects.contains(S3Object.NOT_FOUND);
}
}

65
aws/s3/core/src/test/java/org/jclouds/aws/s3/xml/S3ParserFactoryTest.java
View File

@ -23,12 +23,13 @@
*/ */
package org.jclouds.aws.s3.xml; package org.jclouds.aws.s3.xml;
import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Injector; import com.google.inject.Injector;
import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/** /**
* @author Adrian Cole * @author Adrian Cole
@ -36,39 +37,39 @@ import org.testng.annotations.Test;
@Test(groups = "unit", testName = "s3.S3ParserFactoryTest") @Test(groups = "unit", testName = "s3.S3ParserFactoryTest")
public class S3ParserFactoryTest { public class S3ParserFactoryTest {
Injector injector = null; Injector injector = null;
S3ParserFactory parserFactory = null; S3ParserFactory parserFactory = null;
@BeforeMethod @BeforeTest
void setUpInjector() { void setUpInjector() {
injector = Guice.createInjector(new S3ParserModule()); injector = Guice.createInjector(new S3ParserModule());
parserFactory = injector.getInstance(S3ParserFactory.class); parserFactory = injector.getInstance(S3ParserFactory.class);
} }
@AfterMethod @AfterTest
void tearDownInjector() { void tearDownInjector() {
parserFactory = null; parserFactory = null;
injector = null; injector = null;
} }
@Test @Test
void testCreateListBucketsParser() { void testCreateListBucketsParser() {
assert parserFactory.createListBucketsParser() != null; assert parserFactory.createListBucketsParser() != null;
} }
@Test @Test
void testCreateListBucketParser() { void testCreateListBucketParser() {
assert parserFactory.createListBucketParser() != null; assert parserFactory.createListBucketParser() != null;
} }
@Test @Test
void testCreateCopyObjectParser() { void testCreateCopyObjectParser() {
assert parserFactory.createCopyObjectParser() != null; assert parserFactory.createCopyObjectParser() != null;
} }
@Test @Test
void testCreateErrorParser() { void testCreateErrorParser() {
assert parserFactory.createErrorParser() != null; assert parserFactory.createErrorParser() != null;
} }
} }

447
aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java
View File

@ -40,277 +40,252 @@ import org.jets3t.service.acl.AccessControlList;
import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3BucketLoggingStatus; import org.jets3t.service.model.S3BucketLoggingStatus;
import org.jets3t.service.model.S3Object; import org.jets3t.service.model.S3Object;
import org.jets3t.service.model.S3Owner;
import org.jets3t.service.security.AWSCredentials; import org.jets3t.service.security.AWSCredentials;
import com.google.inject.Module; import com.google.inject.Module;
/** /**
* A JetS3t S3Service implemented by JClouds * A JetS3t S3Service implemented by JClouds
* *
* @author Adrian Cole * @author Adrian Cole
* @author James Murty * @author James Murty
*/ */
public class JCloudsS3Service extends S3Service { public class JCloudsS3Service extends S3Service {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private final S3Context context; private final S3Context context;
private final S3Connection connection; private final S3Connection connection;
private final long requestTimeoutMilliseconds = 10000; private final long requestTimeoutMilliseconds = 10000;
/** /**
* Initializes a JClouds context to S3. * Initializes a JClouds context to S3.
* *
* @param awsCredentials - credentials to access S3 * @param awsCredentials
* @param modules - Module that configures a FutureHttpClient, if not specified, * - credentials to access S3
* default is URLFetchServiceClientModule * @param modules
* @throws S3ServiceException * - Module that configures a FutureHttpClient, if not specified, default is
*/ * URLFetchServiceClientModule
protected JCloudsS3Service(AWSCredentials awsCredentials, Module... modules) * @throws S3ServiceException
*/
protected JCloudsS3Service(AWSCredentials awsCredentials, Module... modules)
throws S3ServiceException { throws S3ServiceException {
super(awsCredentials); super(awsCredentials);
context = S3ContextFactory.createS3Context(awsCredentials context = S3ContextFactory.createS3Context(awsCredentials.getAccessKey(), awsCredentials
.getAccessKey(), awsCredentials.getSecretKey(), modules); .getSecretKey(), modules);
connection = context.getConnection(); connection = context.getConnection();
} }
@Override @Override
public int checkBucketStatus(final String bucketName) throws S3ServiceException { public int checkBucketStatus(final String bucketName) throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
/** /**
* {@inheritDoc} * {@inheritDoc}
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override
protected Map copyObjectImpl(String sourceBucketName, protected Map copyObjectImpl(String sourceBucketName, String sourceObjectKey,
String sourceObjectKey, String destinationBucketName, String destinationBucketName, String destinationObjectKey, AccessControlList acl,
String destinationObjectKey, AccessControlList acl, Map destinationMetadata, Calendar ifModifiedSince, Calendar ifUnmodifiedSince,
Map destinationMetadata, Calendar ifModifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags) throws S3ServiceException {
Calendar ifUnmodifiedSince, String[] ifMatchTags, // TODO Unimplemented
String[] ifNoneMatchTags) throws S3ServiceException { throw new UnsupportedOperationException();
// TODO Unimplemented }
throw new UnsupportedOperationException();
}
@Override @Override
protected S3Bucket createBucketImpl(String bucketName, String location, protected S3Bucket createBucketImpl(String bucketName, String location, AccessControlList acl)
AccessControlList acl) throws S3ServiceException
{
if (location != null)
throw new UnsupportedOperationException("Bucket location is not yet supported");
if (acl != null)
throw new UnsupportedOperationException("Bucket ACL is not yet supported");
try {
if (connection.putBucketIfNotExists(bucketName).get()) {
// Bucket created.
}
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(
"error creating bucket: " + bucketName, e);
}
return new S3Bucket(bucketName);
}
/**
* {@inheritDoc}
*
* @see S3Connection#deleteBucketIfEmpty(String)
*/
@Override
protected void deleteBucketImpl(String bucketName)
throws S3ServiceException { throws S3ServiceException {
try { if (location != null)
connection.deleteBucketIfEmpty(bucketName).get( throw new UnsupportedOperationException("Bucket location is not yet supported");
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); if (acl != null)
} catch (Exception e) { throw new UnsupportedOperationException("Bucket ACL is not yet supported");
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(
"error deleting bucket: " + bucketName, e);
}
}
/** try {
* {@inheritDoc} if (connection.putBucketIfNotExists(bucketName).get()) {
* // Bucket created.
* @see S3Connection#deleteObject(String, String) }
*/ } catch (Exception e) {
@Override Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
protected void deleteObjectImpl(String bucketName, String objectKey) throw new S3ServiceException("error creating bucket: " + bucketName, e);
}
return new S3Bucket(bucketName);
}
/**
* {@inheritDoc}
*
* @see S3Connection#deleteBucketIfEmpty(String)
*/
@Override
protected void deleteBucketImpl(String bucketName) throws S3ServiceException {
try {
connection.deleteBucketIfEmpty(bucketName).get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS);
} catch (Exception e) {
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error deleting bucket: " + bucketName, e);
}
}
/**
* {@inheritDoc}
*
* @see S3Connection#deleteObject(String, String)
*/
@Override
protected void deleteObjectImpl(String bucketName, String objectKey) throws S3ServiceException {
try {
connection.deleteObject(bucketName, objectKey).get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS);
} catch (Exception e) {
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format("error deleting object: %1$s:%2$s", bucketName,
objectKey), e);
}
}
@Override
protected AccessControlList getBucketAclImpl(String bucketName) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected String getBucketLocationImpl(String bucketName) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3BucketLoggingStatus getBucketLoggingStatusImpl(String bucketName)
throws S3ServiceException { throws S3ServiceException {
try { // TODO Unimplemented
connection.deleteObject(bucketName, objectKey).get( throw new UnsupportedOperationException();
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS); }
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format(
"error deleting object: %1$s:%2$s", bucketName, objectKey), e);
}
}
@Override @Override
protected AccessControlList getBucketAclImpl(String bucketName) protected AccessControlList getObjectAclImpl(String bucketName, String objectKey)
throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected String getBucketLocationImpl(String bucketName) protected S3Object getObjectDetailsImpl(String bucketName, String objectKey,
Calendar ifModifiedSince, Calendar ifUnmodifiedSince, String[] ifMatchTags,
String[] ifNoneMatchTags) throws S3ServiceException {
try {
if (ifModifiedSince != null)
throw new IllegalArgumentException("ifModifiedSince");
if (ifUnmodifiedSince != null)
throw new IllegalArgumentException("ifUnmodifiedSince");
if (ifMatchTags != null)
throw new IllegalArgumentException("ifMatchTags");
if (ifNoneMatchTags != null)
throw new IllegalArgumentException("ifNoneMatchTags");
return Util.convertObjectHead(connection.headObject(bucketName, objectKey).get());
} catch (Exception e) {
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format("error retrieving object head: %1$s:%2$s",
bucketName, objectKey), e);
}
}
@Override
protected S3Object getObjectImpl(String bucketName, String objectKey, Calendar ifModifiedSince,
Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags,
Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException {
try {
GetObjectOptions options = Util.convertOptions(ifModifiedSince, ifUnmodifiedSince,
ifMatchTags, ifNoneMatchTags);
return Util.convertObject(connection.getObject(bucketName, objectKey, options).get());
} catch (Exception e) {
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format("error retrieving object: %1$s:%2$s",
bucketName, objectKey), e);
}
}
@Override
public boolean isBucketAccessible(String bucketName) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected boolean isRequesterPaysBucketImpl(String bucketName) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3Bucket[] listAllBucketsImpl() throws S3ServiceException {
try {
List<org.jclouds.aws.s3.domain.S3Bucket.Metadata> jcBucketList = connection
.listOwnedBuckets().get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
return Util.convertBuckets(jcBucketList);
} catch (Exception e) {
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error listing buckets", e);
}
}
@Override
protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, String prefix,
String delimiter, long maxListingLength, String priorLastKey, boolean completeListing)
throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override @Override
protected S3BucketLoggingStatus getBucketLoggingStatusImpl(String bucketName) protected S3Object[] listObjectsImpl(String bucketName, String prefix, String delimiter,
long maxListingLength) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected void putBucketAclImpl(String bucketName, AccessControlList acl)
throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
}
@Override }
protected AccessControlList getObjectAclImpl(String bucketName,
String objectKey) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override @Override
protected S3Object getObjectDetailsImpl(String bucketName, protected void putObjectAclImpl(String bucketName, String objectKey, AccessControlList acl)
String objectKey, Calendar ifModifiedSince,
Calendar ifUnmodifiedSince, String[] ifMatchTags,
String[] ifNoneMatchTags) throws S3ServiceException
{
try {
if (ifModifiedSince != null)
throw new IllegalArgumentException("ifModifiedSince");
if (ifUnmodifiedSince != null)
throw new IllegalArgumentException("ifUnmodifiedSince");
if (ifMatchTags != null)
throw new IllegalArgumentException("ifMatchTags");
if (ifNoneMatchTags != null)
throw new IllegalArgumentException("ifNoneMatchTags");
return Util.convertObjectHead(
connection.headObject(bucketName, objectKey).get());
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format(
"error retrieving object head: %1$s:%2$s", bucketName, objectKey), e);
}
}
@Override
protected S3Object getObjectImpl(String bucketName, String objectKey,
Calendar ifModifiedSince, Calendar ifUnmodifiedSince,
String[] ifMatchTags, String[] ifNoneMatchTags,
Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException
{
try {
GetObjectOptions options = Util.convertOptions(
ifModifiedSince, ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags);
return Util.convertObject(
connection.getObject(bucketName, objectKey, options).get());
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format(
"error retrieving object: %1$s:%2$s", bucketName, objectKey), e);
}
}
@Override
public boolean isBucketAccessible(String bucketName)
throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
}
@Override }
protected boolean isRequesterPaysBucketImpl(String bucketName)
@Override
protected S3Object putObjectImpl(String bucketName, S3Object object) throws S3ServiceException {
// TODO Unimplemented
return null;
}
@Override
protected void setBucketLoggingStatusImpl(String bucketName, S3BucketLoggingStatus status)
throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
}
@Override }
protected S3Bucket[] listAllBucketsImpl() throws S3ServiceException {
try {
List<org.jclouds.aws.s3.domain.S3Bucket.Metadata> jcBucketList = connection
.listOwnedBuckets().get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS);
return Util.convertBuckets(jcBucketList);
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error listing buckets", e);
}
}
@Override @Override
protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, protected void setRequesterPaysBucketImpl(String bucketName, boolean requesterPays)
String prefix, String delimiter, long maxListingLength,
String priorLastKey, boolean completeListing)
throws S3ServiceException { throws S3ServiceException {
// TODO Unimplemented // TODO Unimplemented
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
}
@Override }
protected S3Object[] listObjectsImpl(String bucketName, String prefix,
String delimiter, long maxListingLength) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected void putBucketAclImpl(String bucketName, AccessControlList acl)
throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected void putObjectAclImpl(String bucketName, String objectKey,
AccessControlList acl) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3Object putObjectImpl(String bucketName, S3Object object)
throws S3ServiceException {
// TODO Unimplemented
return null;
}
@Override
protected void setBucketLoggingStatusImpl(String bucketName,
S3BucketLoggingStatus status) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected void setRequesterPaysBucketImpl(String bucketName,
boolean requesterPays) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3Owner getAccountOwnerImpl() throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
} }

27
aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java
View File

@ -23,6 +23,19 @@
*/ */
package org.jclouds.aws.s3.jets3t; package org.jclouds.aws.s3.jets3t;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3IntegrationTest; import org.jclouds.aws.s3.S3IntegrationTest;
import org.jclouds.aws.s3.config.StubS3ConnectionModule; import org.jclouds.aws.s3.config.StubS3ConnectionModule;
@ -31,26 +44,12 @@ import org.jets3t.service.S3ServiceException;
import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object; import org.jets3t.service.model.S3Object;
import org.jets3t.service.security.AWSCredentials; import org.jets3t.service.security.AWSCredentials;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.fail;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap; import com.google.common.collect.Multimap;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/** /**
* Tests to cover JCloudsS3Service * Tests to cover JCloudsS3Service
* *

184
aws/s3/perftest/pom.xml
View File

@ -1,103 +1,99 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!-- <!--
$HeadURL$ $HeadURL$ $Revision$ $Date$ Copyright (C) 2009 Adrian Cole
$Revision$ <adrian@jclouds.org>
$Date$
Copyright (C) 2009 Adrian Cole <adrian@jclouds.org> ====================================================================
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to you under the Apache License, Version
2.0 (the "License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
==================================================================== http://www.apache.org/licenses/LICENSE-2.0.html Unless required by
Licensed to the Apache Software Foundation (ASF) under one applicable law or agreed to in writing, software distributed under the
or more contributor license agreements. See the NOTICE file License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
distributed with this work for additional information CONDITIONS OF ANY KIND, either express or implied. See the License for
regarding copyright ownership. The ASF licenses this file the specific language governing permissions and limitations under the
to you under the Apache License, Version 2.0 (the License.
"License"); you may not use this file except in compliance ====================================================================
with the License. You may obtain a copy of the License at -->
<!--
http://www.apache.org/licenses/LICENSE-2.0.html Note that the code that these performance tests are evaluated against
exist in src/main. The code in that location was copied straight from
Unless required by applicable law or agreed to in writing, Amazon's website. There have been small modifications to make unit
software distributed under the License is distributed on an testing possible. That code exists with the following license: This
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY software code is made available "AS IS" without warranties of any
KIND, either express or implied. See the License for the kind. You may copy, display, modify and redistribute the software code
specific language governing permissions and limitations either by itself or as incorporated into your code; provided that you
under the License. do not remove any proprietary notices. Your use of this software code
==================================================================== is at your own risk and you waive any claim against Amazon Digital
--> Services, Inc. or its affiliates with respect to your use of this
<!-- software code. (c) 2006 Amazon Digital Services, Inc. or its
Note that the code that these performance tests are evaluated against exist in src/main. The code in that affiliates.
location was copied straight from Amazon's website. There have been small modifications to make unit testing -->
possible. That code exists with the following license:
This software code is made available "AS IS" without warranties of any
kind. You may copy, display, modify and redistribute the software
code either by itself or as incorporated into your code; provided that
you do not remove any proprietary notices. Your use of this software
code is at your own risk and you waive any claim against Amazon
Digital Services, Inc. or its affiliates with respect to your use of
this software code. (c) 2006 Amazon Digital Services, Inc. or its
affiliates.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent> <parent>
<artifactId>jclouds-s3-project</artifactId> <artifactId>jclouds-s3-project</artifactId>
<groupId>org.jclouds</groupId> <groupId>org.jclouds</groupId>
<version>1.0-SNAPSHOT</version> <version>1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<artifactId>jclouds-s3-perftest</artifactId> <artifactId>jclouds-s3-perftest</artifactId>
<name>jclouds Performance test verses Amazon samples implementation</name> <name>jclouds Performance test verses Amazon samples implementation</name>
<packaging>jar</packaging> <packaging>jar</packaging>
<description>Performance test verses Amazon samples implementation</description> <description>Performance test verses Amazon samples implementation</description>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>${project.groupId}</groupId> <groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3</artifactId> <artifactId>jclouds-s3</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>${project.groupId}</groupId> <groupId>${project.groupId}</groupId>
<artifactId>jclouds-s3</artifactId> <artifactId>jclouds-s3</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
<type>test-jar</type> <type>test-jar</type>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>xstream</groupId> <groupId>xstream</groupId>
<artifactId>xstream</artifactId> <artifactId>xstream</artifactId>
<version>1.2</version> <version>1.2</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>${project.groupId}</groupId> <groupId>${project.groupId}</groupId>
<artifactId>jclouds-httpnio</artifactId> <artifactId>jclouds-httpnio</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>net.java.dev.jets3t</groupId> <groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId> <artifactId>jets3t</artifactId>
<version>0.7.1</version> <version>0.7.1</version>
</dependency> </dependency>
</dependencies> </dependencies>
<scm> <scm>
<connection>scm:svn:http://jclouds.googlecode.com/svn/trunk/s3core/perftest</connection> <connection>scm:svn:http://jclouds.googlecode.com/svn/trunk/s3core/perftest</connection>
<developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk/s3core/perftest</developerConnection> <developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk/s3core/perftest</developerConnection>
<url>http://jclouds.googlecode.com/svn/trunk/s3core/perftest</url> <url>http://jclouds.googlecode.com/svn/trunk/s3core/perftest</url>
</scm> </scm>
<build>
<repositories>
<!-- For Amazon S3 artifacts -->
<repository>
<name>jets3t</name>
<id>jets3t</id>
<url>http://jets3t.s3.amazonaws.com/maven2</url>
</repository>
</repositories>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<parallel>classes</parallel>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>

131
aws/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformanceLiveTest.java
View File

@ -1,131 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.jclouds.aws.s3.reference.S3Constants;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import java.io.File;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
/**
* Runs operations that amazon s3 sample code is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformanceLiveTest", groups = {"live"})
public class AmazonPerformanceLiveTest extends BasePerformance {
private AWSAuthConnection amzClient;
@Override
@BeforeTest
@Parameters({S3Constants.PROPERTY_AWS_ACCESSKEYID,
S3Constants.PROPERTY_AWS_SECRETACCESSKEY})
protected void setUpCredentials(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey, ITestContext context) throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey,
false);
}
@Override
@Test(enabled = false)
public void testPutFileSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
com.amazon.s3.S3Object object = new com.amazon.s3.S3Object(data, null);
Map<String, List<String>> headers = new TreeMap<String, List<String>>();
headers
.put("Content-Type", Arrays
.asList(new String[]{contentType}));
return amzClient.put(bucket, key, object, headers).connection
.getResponseMessage() != null;
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
}

94
aws/s3/perftest/src/test/java/com/amazon/s3/BaseJCloudsPerformance.java
View File

@ -1,94 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.TimeUnit;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public abstract class BaseJCloudsPerformance extends BasePerformance {
// boolean get
// (
// int id) throws Exception {
// S3Bucket s3Bucket = new S3Bucket();
// s3Bucket.setName(bucketPrefix + "-jclouds-puts");
// org.jclouds.aws.s3.domain.S3Object object = new
// org.jclouds.aws.s3.domain.S3Object();
// object.setKey(id + "");
// //object.setContentType("text/plain");
// object.setContentType("application/octetstream");
// //object.setData("this is a test");
// object.setData(test);
// return clientProvider.getObject(s3Bucket,
// object.getKey()).get(120,TimeUnit.SECONDS) !=
// org.jclouds.aws.s3.domain.S3Object.NOT_FOUND;
// }
@Override
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
object.getMetadata().setSize(data.available());
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
}

252
aws/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java
View File

@ -1,252 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import com.google.inject.Provider;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.testng.ITestContext;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test(groups = {"live"})
public abstract class BasePerformance extends S3IntegrationTest {
protected boolean debugEnabled() {
return false;
}
protected static int LOOP_COUNT = 100;
protected ExecutorService exec;
protected final String BUCKET_BYTES = bucketPrefix + "-bytes";
protected final String BUCKET_INPUTSTREAM = bucketPrefix + "-inputstream";
protected final String BUCKET_STRING = bucketPrefix + "-string";
protected final String BUCKET_FILE = bucketPrefix + "-file";
protected final String[] BUCKETS = {BUCKET_BYTES, BUCKET_INPUTSTREAM,
BUCKET_STRING, BUCKET_FILE};
protected PutBytesCallable putBytesCallable;
protected PutFileCallable putFileCallable;
protected PutInputStreamCallable putInputStreamCallable;
protected PutStringCallable putStringCallable;
protected CompletionService<Boolean> completer;
@BeforeTest
protected void setUpCallables() {
putBytesCallable = new PutBytesCallable();
putFileCallable = new PutFileCallable();
putInputStreamCallable = new PutInputStreamCallable();
putStringCallable = new PutStringCallable();
exec = Executors.newCachedThreadPool();
completer = new ExecutorCompletionService<Boolean>(exec);
}
@Override
@BeforeTest
protected void setUpClient(ITestContext context) throws Exception {
super.setUpClient(context);
for (String bucket : BUCKETS) {
client.putBucketIfNotExists(bucket).get(10, TimeUnit.SECONDS);
}
}
@AfterTest
protected void tearDownExecutor() throws Exception {
exec.shutdownNow();
exec = null;
}
@Test(enabled = true)
public void testPutBytesSerial() throws Exception {
doSerial(putBytesCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutBytesParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putBytesCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutFileSerial() throws Exception {
doSerial(putFileCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutFileParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putFileCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutInputStreamSerial() throws Exception {
doSerial(putInputStreamCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putInputStreamCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutStringSerial() throws Exception {
doSerial(putStringCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutStringParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putStringCallable, LOOP_COUNT);
}
private void doSerial(Provider<Callable<Boolean>> provider, int loopCount)
throws Exception, ExecutionException {
for (int i = 0; i < loopCount; i++)
assert provider.get().call();
}
private void doParallel(Provider<Callable<Boolean>> provider, int loopCount)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < loopCount; i++)
completer.submit(provider.get());
for (int i = 0; i < loopCount; i++)
assert completer.take().get(10, TimeUnit.SECONDS);
}
class PutBytesCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected byte[] test = new byte[1024 * 2];
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putByteArray(BUCKET_BYTES, key.getAndIncrement()
+ "", test, "application/octetstring");
}
};
}
}
class PutFileCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml");
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putFile(BUCKET_FILE, key.getAndIncrement() + "",
file, "text/xml");
}
};
}
}
class PutInputStreamCallable extends PutBytesCallable {
final AtomicInteger key = new AtomicInteger(0);
@Override
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putInputStream(BUCKET_INPUTSTREAM, key
.getAndIncrement()
+ "", new ByteArrayInputStream(test),
"application/octetstring");
}
};
}
}
class PutStringCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected String testString = "hello world!";
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putString(BUCKET_STRING, key.getAndIncrement() + "",
testString, "text/plain");
}
};
}
}
protected abstract boolean putByteArray(String bucket, String key,
byte[] data, String contentType) throws Exception;
protected abstract boolean putFile(String bucket, String key, File data,
String contentType) throws Exception;
protected abstract boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception;
protected abstract boolean putString(String bucket, String key,
String data, String contentType) throws Exception;
// private class BucketDeleter implements Callable<Boolean> {
// private BucketDeleter(S3Bucket bucket) {
// this.bucket = bucket;
// }
//
// private S3Bucket bucket;
//
// @Override
// public String toString() {
// return "BucketDeleter{" + "bucket=" + bucket + '}';
// }
//
// public Boolean call() throws Exception {
// bucket =
// clientProvider.get(10,TimeUnit.SECONDS).getBucket(bucket).get(10,TimeUnit.SECONDS);
// List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
// for (org.jclouds.aws.s3.domain.S3Object object : bucket
// .getContents()) {
// deletes.add(clientProvider.get(10,TimeUnit.SECONDS).deleteObject(bucket,
// object.getKey()));
// }
// for (Future<Boolean> isdeleted : deletes)
// assert isdeleted.get(10,TimeUnit.SECONDS) :
// String.format("failed to delete %1$ss",
// isdeleted);
// return
// clientProvider.get(10,TimeUnit.SECONDS).deleteBucket(bucket).get(10,TimeUnit.SECONDS);
// }
// }
}

437
aws/s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java
View File

@ -23,6 +23,9 @@
*/ */
package com.amazon.s3; package com.amazon.s3;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import java.util.Date; import java.util.Date;
import java.util.Locale; import java.util.Locale;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
@ -41,9 +44,6 @@ import org.testng.annotations.Test;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Injector; import com.google.inject.Injector;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
/* /*
* TODO: Scrap any non-DateService references (eg Joda & Amazon) if/when * TODO: Scrap any non-DateService references (eg Joda & Amazon) if/when
* we confirm that the DateService is fast enough. * we confirm that the DateService is fast enough.
@ -57,253 +57,228 @@ import static org.testng.Assert.assertTrue;
*/ */
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest") @Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest")
public class DateServiceTest extends PerformanceTest { public class DateServiceTest extends PerformanceTest {
Injector i = Guice.createInjector(); Injector i = Guice.createInjector();
DateService dateService = i.getInstance(DateService.class); DateService dateService = i.getInstance(DateService.class);
private TestData[] testData; private TestData[] testData;
class TestData {
public final String iso8601DateString;
public final String rfc822DateString;
public final Date date;
TestData(String iso8601, String rfc822, Date date) {
this.iso8601DateString = iso8601;
this.rfc822DateString = rfc822;
this.date = date;
}
}
private long startTime;
public DateServiceTest() { class TestData {
// Constant time test values, each TestData item must contain matching times! public final String iso8601DateString;
testData = new TestData[] { public final String rfc822DateString;
new TestData("2009-03-12T02:00:07.000Z", "Thu, 12 Mar 2009 02:00:07 GMT", new Date(1236823207000l)), public final Date date;
new TestData("2009-03-14T04:00:07.000Z", "Sat, 14 Mar 2009 04:00:07 GMT", new Date(1237003207000l)),
new TestData("2009-03-16T06:00:07.000Z", "Mon, 16 Mar 2009 06:00:07 GMT", new Date(1237183207000l)),
new TestData("2009-03-18T08:00:07.000Z", "Wed, 18 Mar 2009 08:00:07 GMT", new Date(1237363207000l)),
new TestData("2009-03-20T10:00:07.000Z", "Fri, 20 Mar 2009 10:00:07 GMT", new Date(1237543207000l))
};
}
// Joda items for performance comparisons
private DateTimeFormatter headerDateFormat = DateTimeFormat
.forPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.US);
private DateTime jodaParseIso8601(String toParse) { TestData(String iso8601, String rfc822, Date date) {
return new DateTime(toParse); this.iso8601DateString = iso8601;
} this.rfc822DateString = rfc822;
this.date = date;
}
}
private DateTime jodaParseRfc822(String toParse) { private long startTime;
return headerDateFormat.parseDateTime(toParse);
}
private String timestampAsHeaderString() { public DateServiceTest() {
return toHeaderString(new DateTime()); // Constant time test values, each TestData item must contain matching times!
} testData = new TestData[] {
new TestData("2009-03-12T02:00:07.000Z", "Thu, 12 Mar 2009 02:00:07 GMT", new Date(
1236823207000l)),
new TestData("2009-03-14T04:00:07.000Z", "Sat, 14 Mar 2009 04:00:07 GMT", new Date(
1237003207000l)),
new TestData("2009-03-16T06:00:07.000Z", "Mon, 16 Mar 2009 06:00:07 GMT", new Date(
1237183207000l)),
new TestData("2009-03-18T08:00:07.000Z", "Wed, 18 Mar 2009 08:00:07 GMT", new Date(
1237363207000l)),
new TestData("2009-03-20T10:00:07.000Z", "Fri, 20 Mar 2009 10:00:07 GMT", new Date(
1237543207000l)) };
}
private String toHeaderString(DateTime date) { // Joda items for performance comparisons
return headerDateFormat.print(date.withZone(DateTimeZone.UTC)); private DateTimeFormatter headerDateFormat = DateTimeFormat.forPattern(
} "EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.US);
private DateTime jodaParseIso8601(String toParse) {
private void startClock() { return new DateTime(toParse);
startTime = System.currentTimeMillis(); }
}
private void printElapsedClockTime(String testName) {
System.out.println(testName + " took " +
(System.currentTimeMillis() - startTime) + "ms for "
+ LOOP_COUNT+ " loops");
}
@Test
public void testIso8601DateParse() throws ExecutionException,
InterruptedException
{
DateTime dsDate = dateService.iso8601DateParse(testData[0].iso8601DateString);
assertEquals(dsDate.toDate(), testData[0].date);
}
@Test private DateTime jodaParseRfc822(String toParse) {
public void testRfc822DateParse() throws ExecutionException, return headerDateFormat.parseDateTime(toParse);
InterruptedException }
{
DateTime dsDate = dateService.rfc822DateParse(testData[0].rfc822DateString);
assertEquals(dsDate.toDate(), testData[0].date);
}
@Test private String timestampAsHeaderString() {
public void testIso8601DateFormat() throws ExecutionException, return toHeaderString(new DateTime());
InterruptedException }
{
String dsString = dateService.iso8601DateFormat(testData[0].date);
assertEquals(dsString, testData[0].iso8601DateString);
}
@Test private String toHeaderString(DateTime date) {
public void testRfc822DateFormat() throws ExecutionException, return headerDateFormat.print(date.withZone(DateTimeZone.UTC));
InterruptedException }
{
String dsString = dateService.rfc822DateFormat(testData[0].date);
assertEquals(dsString, testData[0].rfc822DateString);
}
@Test private void startClock() {
void testIso8601DateFormatResponseTime() throws ExecutionException, startTime = System.currentTimeMillis();
InterruptedException { }
for (int i = 0; i < LOOP_COUNT; i++)
dateService.iso8601DateFormat();
}
@Test private void printElapsedClockTime(String testName) {
void testRfc822DateFormatResponseTime() throws ExecutionException, System.out.println(testName + " took " + (System.currentTimeMillis() - startTime) + "ms for "
InterruptedException { + LOOP_COUNT + " loops");
for (int i = 0; i < LOOP_COUNT; i++) }
dateService.rfc822DateFormat();
}
@Test
void testFormatIso8601DateInParallel() throws InterruptedException,
ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() {
public Boolean call() throws ExecutionException,
InterruptedException
{
String dsString = dateService.iso8601DateFormat(myData.date);
/*
* Comment-in the assert below to test thread safety.
* Comment it out to test performance
*/
assertEquals(dsString, myData.iso8601DateString);
return true;
}
});
}
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testFormatIso8601DateInParallel");
}
@Test @Test
void testFormatAmazonDateInParallel() throws InterruptedException, public void testIso8601DateParse() throws ExecutionException, InterruptedException {
ExecutionException { DateTime dsDate = dateService.iso8601DateParse(testData[0].iso8601DateString);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>( assertEquals(dsDate.toDate(), testData[0].date);
exec); }
startClock();
for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<Boolean>() {
public Boolean call() {
AWSAuthConnection.httpDate();
return true;
}
});
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testFormatAmazonDateInParallel");
}
@Test @Test
void testFormatJodaDateInParallel() throws InterruptedException, public void testRfc822DateParse() throws ExecutionException, InterruptedException {
ExecutionException { DateTime dsDate = dateService.rfc822DateParse(testData[0].rfc822DateString);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>( assertEquals(dsDate.toDate(), testData[0].date);
exec); }
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() {
public Boolean call() {
String jodaString = toHeaderString(new DateTime(myData.date));
assertEquals(jodaString, myData.rfc822DateString);
return true;
}
});
}
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testFormatJodaDateInParallel");
}
@Test @Test
void testIso8601ParseDateSerialResponseTime() throws ExecutionException, public void testIso8601DateFormat() throws ExecutionException, InterruptedException {
InterruptedException { String dsString = dateService.iso8601DateFormat(testData[0].date);
for (int i = 0; i < LOOP_COUNT; i++) assertEquals(dsString, testData[0].iso8601DateString);
dateService.iso8601DateParse(testData[0].iso8601DateString); }
}
@Test @Test
void testAmazonParseDateSerialResponseTime() { public void testRfc822DateFormat() throws ExecutionException, InterruptedException {
for (int i = 0; i < LOOP_COUNT; i++) String dsString = dateService.rfc822DateFormat(testData[0].date);
AWSAuthConnection.httpDate(); assertEquals(dsString, testData[0].rfc822DateString);
} }
@Test @Test
void testParseIso8601DateInParallel() throws InterruptedException, void testIso8601DateFormatResponseTime() throws ExecutionException, InterruptedException {
ExecutionException { for (int i = 0; i < LOOP_COUNT; i++)
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>( dateService.iso8601DateFormat();
exec); }
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() {
public Boolean call() throws ExecutionException,
InterruptedException {
DateTime dsDate = dateService.iso8601DateParse(myData.iso8601DateString);
assertEquals(dsDate.toDate(), myData.date);
return true;
}
});
}
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testParseIso8601DateInParallel");
}
@Test @Test
void testParseAmazonDateInParallel() throws InterruptedException, void testRfc822DateFormatResponseTime() throws ExecutionException, InterruptedException {
ExecutionException { for (int i = 0; i < LOOP_COUNT; i++)
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>( dateService.rfc822DateFormat();
exec); }
startClock();
for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<Boolean>() {
public Boolean call() {
AWSAuthConnection.httpDate();
return true;
}
});
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testParseAmazonDateInParallel");
}
@Test @Test
void testParseJodaDateInParallel() throws InterruptedException, void testFormatIso8601DateInParallel() throws InterruptedException, ExecutionException {
ExecutionException { CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>( startClock();
exec); for (int i = 0; i < LOOP_COUNT; i++) {
startClock(); final TestData myData = testData[i % testData.length];
for (int i = 0; i < LOOP_COUNT; i++) { completer.submit(new Callable<Boolean>() {
final TestData myData = testData[i % testData.length]; public Boolean call() throws ExecutionException, InterruptedException {
completer.submit(new Callable<Boolean>() { String dsString = dateService.iso8601DateFormat(myData.date);
public Boolean call() { /*
Date jodaDate = jodaParseIso8601(myData.iso8601DateString).toDate(); * Comment-in the assert below to test thread safety. Comment it out to test
assertEquals(jodaDate, myData.date); * performance
return true; */
} assertEquals(dsString, myData.iso8601DateString);
}); return true;
} }
for (int i = 0; i < LOOP_COUNT; i++) });
assertTrue(completer.take().get()); }
printElapsedClockTime("testParseJodaDateInParallel"); for (int i = 0; i < LOOP_COUNT; i++)
} assertTrue(completer.take().get());
printElapsedClockTime("testFormatIso8601DateInParallel");
}
@Test
void testFormatAmazonDateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<Boolean>() {
public Boolean call() {
AWSAuthConnection.httpDate();
return true;
}
});
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testFormatAmazonDateInParallel");
}
@Test
void testFormatJodaDateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() {
public Boolean call() {
String jodaString = toHeaderString(new DateTime(myData.date));
assertEquals(jodaString, myData.rfc822DateString);
return true;
}
});
}
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testFormatJodaDateInParallel");
}
@Test
void testIso8601ParseDateSerialResponseTime() throws ExecutionException, InterruptedException {
for (int i = 0; i < LOOP_COUNT; i++)
dateService.iso8601DateParse(testData[0].iso8601DateString);
}
@Test
void testAmazonParseDateSerialResponseTime() {
for (int i = 0; i < LOOP_COUNT; i++)
AWSAuthConnection.httpDate();
}
@Test
void testParseIso8601DateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() {
public Boolean call() throws ExecutionException, InterruptedException {
DateTime dsDate = dateService.iso8601DateParse(myData.iso8601DateString);
assertEquals(dsDate.toDate(), myData.date);
return true;
}
});
}
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testParseIso8601DateInParallel");
}
@Test
void testParseAmazonDateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<Boolean>() {
public Boolean call() {
AWSAuthConnection.httpDate();
return true;
}
});
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testParseAmazonDateInParallel");
}
@Test
void testParseJodaDateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() {
public Boolean call() {
Date jodaDate = jodaParseIso8601(myData.iso8601DateString).toDate();
assertEquals(jodaDate, myData.date);
return true;
}
});
}
for (int i = 0; i < LOOP_COUNT; i++)
assertTrue(completer.take().get());
printElapsedClockTime("testParseJodaDateInParallel");
}
} }

130
aws/s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformanceLiveTest.java
View File

@ -1,130 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.jets3t.service.S3Service;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.security.AWSCredentials;
import org.testng.ITestContext;
import org.testng.annotations.Test;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.ExecutionException;
/**
* Runs operations that jets3t is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformanceLiveTest", groups = {"live"})
public class Jets3tPerformanceLiveTest extends BasePerformance {
private S3Service jetClient;
@Override
protected void setUpCredentials(String AWSAccessKeyId, String AWSSecretAccessKey, ITestContext context)
throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
jetClient = new RestS3Service(new AWSCredentials(AWSAccessKeyId,
AWSSecretAccessKey));
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(
key);
object.setContentType(contentType);
object.setDataInputFile(data);
object.setContentLength(data.length());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(
key);
object.setContentType(contentType);
object.setDataInputStream(data);
object.setContentLength(data.available());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
}

73
aws/s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java
View File

@ -1,73 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.testng.annotations.Test;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
/**
* Compares performance of encryption operations.
*
* @author Adrian Cole
*
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest")
public class S3UtilsTest extends org.jclouds.aws.s3.S3UtilsTest {
@Test(dataProvider = "hmacsha1")
void testAmazonSampleDigestSerialResponseTime(byte[] key, String message, String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException {
for (int i = 0; i < 10000; i++)
testAmazonSampleDigest(key, message, base64Digest);
}
@Test(dataProvider = "hmacsha1")
public void testAmazonSampleDigest(byte[] key, String message, String base64Digest) {
String encoded = Utils.encode(new String(key), message, false);
assert encoded.equals(base64Digest);
}
@Test(dataProvider = "hmacsha1")
void testAmazonSampleDigestParallelResponseTime(final byte[] key, final String message, final String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException, InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
for (int i = 0; i < 10000; i++)
completer.submit(new Callable<Boolean>() {
public Boolean call() {
try {
testAmazonSampleDigest(key, message, base64Digest);
return true;
} catch (Exception e) {
return false;
}
}
});
for (int i = 0; i < 10000; i++) assert completer.take().get();
}
}

124
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/AmazonPerformanceLiveTest.java Normal file
View File

@ -0,0 +1,124 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.File;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import org.jclouds.aws.s3.reference.S3Constants;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import com.amazon.s3.AWSAuthConnection;
/**
* Runs operations that amazon s3 sample code is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformanceLiveTest", groups = { "live" })
public class AmazonPerformanceLiveTest extends BasePerformance {
private AWSAuthConnection amzClient;
@Override
@BeforeTest
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpCredentials(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey, ITestContext context) throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey, false);
}
@Override
@Test(enabled = false)
public void testPutFileSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
com.amazon.s3.S3Object object = new com.amazon.s3.S3Object(data, null);
Map<String, List<String>> headers = new TreeMap<String, List<String>>();
headers.put("Content-Type", Arrays.asList(new String[] { contentType }));
return amzClient.put(bucket, key, object, headers).connection.getResponseMessage() != null;
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
}

90
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/BaseJCloudsPerformance.java Normal file
View File

@ -0,0 +1,90 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.TimeUnit;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public abstract class BaseJCloudsPerformance extends BasePerformance {
// boolean get
// (
// int id) throws Exception {
// S3Bucket s3Bucket = new S3Bucket();
// s3Bucket.setName(bucketPrefix + "-jclouds-puts");
// org.jclouds.aws.s3.domain.S3Object object = new
// org.jclouds.aws.s3.domain.S3Object();
// object.setKey(id + "");
// //object.setContentType("text/plain");
// object.setContentType("application/octetstream");
// //object.setData("this is a test");
// object.setData(test);
// return clientProvider.getObject(s3Bucket,
// object.getKey()).get(120,TimeUnit.SECONDS) !=
// org.jclouds.aws.s3.domain.S3Object.NOT_FOUND;
// }
@Override
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
object.getMetadata().setSize(data.available());
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
}

226
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/BasePerformance.java Normal file
View File

@ -0,0 +1,226 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.inject.Provider;
/**
* Tests relative performance of S3 functions.
*
* @author Adrian Cole
*/
@Test(groups = { "live" })
public abstract class BasePerformance extends S3IntegrationTest {
@Override
protected boolean debugEnabled() {
return false;
}
protected static int LOOP_COUNT = 100;
protected ExecutorService exec;
protected CompletionService<Boolean> completer;
@BeforeTest
protected void setUpCallables() {
exec = Executors.newCachedThreadPool();
completer = new ExecutorCompletionService<Boolean>(exec);
}
@AfterTest
protected void tearDownExecutor() throws Exception {
exec.shutdownNow();
exec = null;
}
@Test(enabled = true)
public void testPutBytesSerial() throws Exception {
doSerial(new PutBytesCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutBytesParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutBytesCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutFileSerial() throws Exception {
doSerial(new PutFileCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutFileParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutFileCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutInputStreamSerial() throws Exception {
doSerial(new PutInputStreamCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutInputStreamCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutStringSerial() throws Exception {
doSerial(new PutStringCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutStringParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutStringCallable(this.bucketName), LOOP_COUNT);
}
private void doSerial(Provider<Callable<Boolean>> provider, int loopCount) throws Exception,
ExecutionException {
for (int i = 0; i < loopCount; i++)
assert provider.get().call();
}
private void doParallel(Provider<Callable<Boolean>> provider, int loopCount)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < loopCount; i++)
completer.submit(provider.get());
for (int i = 0; i < loopCount; i++)
assert completer.take().get(10, TimeUnit.SECONDS);
}
class PutBytesCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected byte[] test = new byte[1024 * 2];
private final String bucketName;
public PutBytesCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
String bucketName2 = bucketName;
return putByteArray(bucketName2, key.getAndIncrement() + "", test,
"application/octetstring");
}
};
}
}
class PutFileCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml");
private final String bucketName;
public PutFileCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putFile(bucketName, key.getAndIncrement() + "", file, "text/xml");
}
};
}
}
class PutInputStreamCallable extends PutBytesCallable {
final AtomicInteger key = new AtomicInteger(0);
private final String bucketName;
public PutInputStreamCallable(String bucketName) {
super(bucketName);
this.bucketName = bucketName;
}
@Override
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putInputStream(bucketName, key.getAndIncrement() + "",
new ByteArrayInputStream(test), "application/octetstring");
}
};
}
}
class PutStringCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected String testString = "hello world!";
private final String bucketName;
public PutStringCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putString(bucketName, key.getAndIncrement() + "", testString, "text/plain");
}
};
}
}
protected abstract boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception;
protected abstract boolean putFile(String bucket, String key, File data, String contentType)
throws Exception;
protected abstract boolean putInputStream(String bucket, String key, InputStream data,
String contentType) throws Exception;
protected abstract boolean putString(String bucket, String key, String data, String contentType)
throws Exception;
}

6
aws/s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformanceLiveTest.java → aws/s3/perftest/src/test/java/org/jclouds/aws/s3/JCloudsNioPerformanceLiveTest.java
View File

@ -21,14 +21,14 @@
* under the License. * under the License.
* ==================================================================== * ====================================================================
*/ */
package com.amazon.s3; package org.jclouds.aws.s3;
import java.util.Properties;
import com.google.inject.Module;
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule; import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.util.Properties; import com.google.inject.Module;
@Test(sequential = true, testName = "s3.JCloudsNioPerformanceLiveTest", groups = {"live"}) @Test(sequential = true, testName = "s3.JCloudsNioPerformanceLiveTest", groups = {"live"})
public class JCloudsNioPerformanceLiveTest extends BaseJCloudsPerformance { public class JCloudsNioPerformanceLiveTest extends BaseJCloudsPerformance {

3
aws/s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformanceLiveTest.java → aws/s3/perftest/src/test/java/org/jclouds/aws/s3/JCloudsPerformanceLiveTest.java
View File

@ -21,8 +21,7 @@
* under the License. * under the License.
* ==================================================================== * ====================================================================
*/ */
package com.amazon.s3; package org.jclouds.aws.s3;
import org.testng.annotations.Test; import org.testng.annotations.Test;
/** /**

127
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/Jets3tPerformanceLiveTest.java Normal file
View File

@ -0,0 +1,127 @@
package org.jclouds.aws.s3;
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.ExecutionException;
import org.jclouds.aws.s3.reference.S3Constants;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.security.AWSCredentials;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
/**
* Runs operations that jets3t is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformanceLiveTest", groups = { "live" })
public class Jets3tPerformanceLiveTest extends BasePerformance {
private S3Service jetClient;
@BeforeTest
public void setUpJetS3t(ITestContext testContext) throws S3ServiceException {
String AWSAccessKeyId = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID);
String AWSSecretAccessKey = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
jetClient = new RestS3Service(new AWSCredentials(AWSAccessKeyId, AWSSecretAccessKey));
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
object.setContentType(contentType);
object.setDataInputFile(data);
object.setContentLength(data.length());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
object.setContentType(contentType);
object.setDataInputStream(data);
object.setContentLength(data.available());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
}

101
core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java
View File

@ -35,66 +35,55 @@ import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.jclouds.http.HttpFutureCommand; import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpResponse;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@Test @Test(groups = { "unit" })
public class ReturnStringIf200Test { public class ReturnStringIf200Test {
private HttpFutureCommand.ResponseCallable<String> callable = null; @Test
public void testExceptionWhenNoContentOn200() throws ExecutionException, InterruptedException,
TimeoutException, IOException {
HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
expect(response.getContent()).andReturn(null);
replay(response);
callable.setResponse(response);
try {
callable.call();
} catch (Exception e) {
assert e.getMessage().equals("no content");
}
verify(response);
}
@BeforeMethod @Test
void setUp() { public void testExceptionWhenIOExceptionOn200() throws ExecutionException, InterruptedException,
callable = new ReturnStringIf200(); TimeoutException, IOException {
} HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
RuntimeException exception = new RuntimeException("bad");
expect(response.getContent()).andThrow(exception);
replay(response);
callable.setResponse(response);
try {
callable.call();
} catch (Exception e) {
assert e.equals(exception);
}
verify(response);
}
@AfterMethod @Test
void tearDown() { public void testResponseOk() throws Exception {
callable = null; HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
} HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
@Test expect(response.getContent()).andReturn(IOUtils.toInputStream("hello"));
public void testExceptionWhenNoContentOn200() throws ExecutionException, replay(response);
InterruptedException, TimeoutException, IOException { callable.setResponse(response);
HttpResponse response = createMock(HttpResponse.class); assert "hello".equals(callable.call());
expect(response.getStatusCode()).andReturn(200).atLeastOnce(); verify(response);
expect(response.getContent()).andReturn(null); }
replay(response);
callable.setResponse(response);
try {
callable.call();
} catch (Exception e) {
assert e.getMessage().equals("no content");
}
verify(response);
}
@Test
public void testExceptionWhenIOExceptionOn200() throws ExecutionException,
InterruptedException, TimeoutException, IOException {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
RuntimeException exception = new RuntimeException("bad");
expect(response.getContent()).andThrow(exception);
replay(response);
callable.setResponse(response);
try {
callable.call();
} catch (Exception e) {
assert e.equals(exception);
}
verify(response);
}
@Test
public void testResponseOk() throws Exception {
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
expect(response.getContent()).andReturn(IOUtils.toInputStream("hello"));
replay(response);
callable.setResponse(response);
assert "hello".equals(callable.call());
verify(response);
}
} }

76
core/src/test/java/org/jclouds/test/testng/UnitTestStatusListener.java Normal file
View File

@ -0,0 +1,76 @@
package org.jclouds.test.testng;
import java.util.concurrent.atomic.AtomicInteger;
import org.testng.IClass;
import org.testng.ITestContext;
import org.testng.ITestListener;
import org.testng.ITestResult;
/**
* adapted from the following class:
*
* @see org.UnitTestStatusListener.test.testng.UnitTestTestNGListener
* @author Adrian Cole
* @author dpospisi@redhat.com
* @author Mircea.Markus@jboss.com
*/
public class UnitTestStatusListener implements ITestListener {
/**
* Holds test classes actually running in all threads.
*/
private ThreadLocal<IClass> threadTestClass = new ThreadLocal<IClass>();
private ThreadLocal<Long> threadTestStart = new ThreadLocal<Long>();
private AtomicInteger failed = new AtomicInteger(0);
private AtomicInteger succeded = new AtomicInteger(0);
private AtomicInteger skipped = new AtomicInteger(0);
public void onTestStart(ITestResult res) {
System.out.println("Starting test " + getTestDesc(res));
threadTestClass.set(res.getTestClass());
threadTestStart.set(System.currentTimeMillis());
}
synchronized public void onTestSuccess(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " succeeded: "
+ (System.currentTimeMillis() - threadTestStart.get()) + "ms");
succeded.incrementAndGet();
printStatus();
}
synchronized public void onTestFailure(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " failed.");
failed.incrementAndGet();
printStatus();
}
synchronized public void onTestSkipped(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " skipped.");
skipped.incrementAndGet();
printStatus();
}
public void onTestFailedButWithinSuccessPercentage(ITestResult arg0) {
}
public void onStart(ITestContext arg0) {
}
public void onFinish(ITestContext arg0) {
}
private String getThreadId() {
return "[" + Thread.currentThread().getName() + "]";
}
private String getTestDesc(ITestResult res) {
return res.getMethod().getMethodName() + "(" + res.getTestClass().getName() + ")";
}
private void printStatus() {
System.out.println("Test suite progress: tests succeeded: " + succeded.get() + ", failed: "
+ failed.get() + ", skipped: " + skipped.get() + ".");
}
}

16
project/pom.xml
View File

@ -169,7 +169,9 @@
<goal>test</goal> <goal>test</goal>
</goals> </goals>
<configuration> <configuration>
<!-- note that the groups/excluded groups don't work due to some problem <parallel>tests</parallel>
<threadCount>5</threadCount>
<!-- note that the groups/excluded groups don't work due to some problem
in surefire or testng. instead, we have to exclude via file path in surefire or testng. instead, we have to exclude via file path
<groups>integration</groups> <groups>integration</groups>
<excludedGroups>unit,performance,live</excludedGroups> --> <excludedGroups>unit,performance,live</excludedGroups> -->
@ -183,14 +185,22 @@
</execution> </execution>
</executions> </executions>
<configuration> <configuration>
<parallel>methods</parallel>
<threadCount>5</threadCount>
<!-- note that the groups/excluded groups don't work due to some problem <!-- note that the groups/excluded groups don't work due to some problem
in surefire or testng. instead, we have to exclude via file path in surefire or testng. instead, we have to exclude via file path
<groups>unit,performance</groups> <groups>unit,performance</groups>
<excludedGroups>integration,live</excludedGroups> --> <excludedGroups>integration,live</excludedGroups> -->
<excludes> <excludes>
<exclude>**/*IntegrationTest.java</exclude> <exclude>**/*IntegrationTest.java</exclude>
<exclude>**/*LiveTest.java</exclude> <exclude>**/*LiveTest.java</exclude>
</excludes> </excludes>
<properties>
<property>
<name>listener</name>
<value>org.jclouds.test.testng.UnitTestStatusListener</value>
</property>
</properties>
</configuration> </configuration>
</plugin> </plugin>
<!-- Make sure we generate src jars too --> <!-- Make sure we generate src jars too -->