Apache/jclouds
1
0
Fork 0
mirror of https://github.com/apache/jclouds.git synced 2025-02-16 06:56:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

made tests parallel and fixed some transient errors in BaseS3Map

Browse Source 
git-svn-id: http://jclouds.googlecode.com/svn/trunk@834 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-05-22 15:35:57 +00:00
parent 6b9c2431d6
commit f8da659695
29 changed files with 2068 additions and 2025 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
aws/s3/core/src/main/java/org/jclouds/aws/s3/internal/BaseS3Map.java
View File

@ -24,9 +24,19 @@
package org.jclouds.aws.s3.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.name.Named;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.S3Map;
import org.jclouds.aws.s3.domain.S3Bucket;
@ -34,19 +44,16 @@ import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.util.Utils;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.name.Named;
/**
* Implements core Map functionality with an {@link S3Connection}
* <p/>
* All commands will wait a maximum of ${jclouds.s3.map.timeout} milliseconds to
* complete before throwing an exception.
* All commands will wait a maximum of ${jclouds.s3.map.timeout} milliseconds to complete before
* throwing an exception.
*
* @author Adrian Cole
* @param <V>
@ -64,6 +71,13 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
@Named(S3Constants.PROPERTY_S3_MAP_TIMEOUT)
protected long requestTimeoutMilliseconds = 10000;
/**
* time to pause before retrying a transient failure
*/
@Inject(optional = true)
@Named(S3Constants.PROPERTY_S3_MAP_RETRY)
protected long requestRetryMilliseconds = 10;
@Inject
public BaseS3Map(S3Connection connection, @Assisted String bucket) {
this.connection = checkNotNull(connection, "connection");
@ -83,14 +97,13 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
Set<S3Object.Metadata> contents = bucket.getContents();
return contents.size();
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting size of bucketName"
+ bucket, e);
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting size of bucketName" + bucket, e);
}
}
protected boolean containsMd5(byte[] md5) throws InterruptedException,
ExecutionException, TimeoutException {
protected boolean containsMd5(byte[] md5) throws InterruptedException, ExecutionException,
TimeoutException {
for (S3Object.Metadata metadata : refreshBucket().getContents()) {
if (Arrays.equals(md5, metadata.getMd5()))
return true;
@ -98,9 +111,8 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
return false;
}
protected byte[] getMd5(Object value) throws IOException,
FileNotFoundException, InterruptedException, ExecutionException,
TimeoutException {
protected byte[] getMd5(Object value) throws IOException, FileNotFoundException,
InterruptedException, ExecutionException, TimeoutException {
S3Object object = null;
if (value instanceof S3Object) {
object = (S3Object) value;
@ -124,37 +136,47 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
futureObjects.add(connection.getObject(bucket, key));
}
for (Future<S3Object> futureObject : futureObjects) {
S3Object object = null;
try {
object = futureObject.get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS);
ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error getting value from bucket %1$s", bucket), e);
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format("Error getting value from bucket %1$s",
bucket), e);
}
if (object != S3Object.NOT_FOUND)
objects.add(object);
}
return objects;
}
@VisibleForTesting
void ifNotFoundRetryOtherwiseAddToSet(Future<S3Object> futureObject, Set<S3Object> objects)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < 3; i++) {
S3Object object = futureObject.get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
if (object != S3Object.NOT_FOUND) {
objects.add(object);
break;
} else {
Thread.sleep(requestRetryMilliseconds);
}
}
}
/**
* {@inheritDoc}
* <p/>
* Note that if value is an instance of InputStream, it will be read and
* closed following this method. To reuse data from InputStreams, pass
* {@link java.io.InputStream}s inside {@link S3Object}s
* Note that if value is an instance of InputStream, it will be read and closed following this
* method. To reuse data from InputStreams, pass {@link java.io.InputStream}s inside
* {@link S3Object}s
*/
public boolean containsValue(Object value) {
try {
byte[] md5 = getMd5(value);
return containsMd5(md5);
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for ETAG of value: [%2$s] in bucketName:%1$s",
bucket, value), e);
"Error searching for ETAG of value: [%2$s] in bucketName:%1$s", bucket, value), e);
}
}
@ -177,20 +199,19 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
deletes.add(connection.deleteObject(bucket, key));
}
for (Future<Boolean> isdeleted : deletes)
if (!isdeleted.get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS)) {
if (!isdeleted.get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)) {
throw new S3RuntimeException("failed to delete entry");
}
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error clearing bucketName" + bucket, e);
}
}
protected S3Bucket refreshBucket() throws InterruptedException,
ExecutionException, TimeoutException {
S3Bucket currentBucket = connection.listBucket(bucket).get(
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
protected S3Bucket refreshBucket() throws InterruptedException, ExecutionException,
TimeoutException {
S3Bucket currentBucket = connection.listBucket(bucket).get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS);
if (currentBucket == S3Bucket.NOT_FOUND)
throw new S3RuntimeException("bucketName not found: " + bucket);
else
@ -204,20 +225,19 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
keys.add(object.getKey());
return keys;
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting keys in bucketName: "
+ bucket, e);
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting keys in bucketName: " + bucket, e);
}
}
public boolean containsKey(Object key) {
try {
return connection.headObject(bucket, key.toString()).get(
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS) != S3Object.Metadata.NOT_FOUND;
return connection.headObject(bucket, key.toString()).get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS) != S3Object.Metadata.NOT_FOUND;
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format(
"Error searching for %1$s:%2$s", bucket, key), e);
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException(String.format("Error searching for %1$s:%2$s", bucket, key),
e);
}
}
@ -229,7 +249,7 @@ public abstract class BaseS3Map<V> implements S3Map<String, V> {
try {
return refreshBucket();
} catch (Exception e) {
Utils.<S3RuntimeException>rethrowIfRuntimeOrSameType(e);
Utils.<S3RuntimeException> rethrowIfRuntimeOrSameType(e);
throw new S3RuntimeException("Error getting bucketName" + bucket, e);
}
}

7
aws/s3/core/src/main/java/org/jclouds/aws/s3/reference/S3Constants.java
View File

@ -36,9 +36,12 @@ public interface S3Constants extends HttpConstants, PoolConstants, S3Headers {
public static final String PROPERTY_AWS_SECRETACCESSKEY = "jclouds.aws.secretaccesskey";
public static final String PROPERTY_AWS_ACCESSKEYID = "jclouds.aws.accesskeyid";
/**
* longest time a single Map operation can take before throwing an
* exception.
* longest time a single Map operation can take before throwing an exception.
*/
public static final String PROPERTY_S3_MAP_TIMEOUT = "jclouds.s3.map.timeout";
/**
* time to pause before retrying a transient failure
*/
public static final String PROPERTY_S3_MAP_RETRY = "jclouds.s3.map.retry";
}

14
aws/s3/core/src/test/java/org/jclouds/aws/PerformanceTest.java
View File

@ -23,29 +23,29 @@
*/
package org.jclouds.aws;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test(groups = "performance", sequential = true, testName = "s3.PerformanceTest")
@Test(groups = "performance")
public class PerformanceTest {
protected static int LOOP_COUNT = 1000;
protected ExecutorService exec;
@BeforeMethod
@BeforeTest
public void setupExecutorService() {
exec = Executors.newCachedThreadPool();
}
@AfterMethod
@AfterTest
public void teardownExecutorService() {
exec.shutdownNow();
exec = null;

49
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3ConnectionIntegrationTest.java
View File

@ -24,20 +24,19 @@
package org.jclouds.aws.s3;
import static com.google.common.base.Preconditions.checkNotNull;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.util.S3Utils;
import static org.testng.Assert.assertEquals;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.util.S3Utils;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
/**
* Tests connection by listing all the buckets and their size
*
@ -46,37 +45,30 @@ import java.util.concurrent.TimeUnit;
@Test(testName = "s3.S3ConnectionIntegrationTest")
public class S3ConnectionIntegrationTest extends S3IntegrationTest {
@Test(groups = {"integration"})
@Test(groups = { "integration" })
void testListBuckets() throws Exception {
List<S3Bucket.Metadata> myBuckets = client.listOwnedBuckets().get(10,
TimeUnit.SECONDS);
for (S3Bucket.Metadata bucket : myBuckets) {
context.createInputStreamMap(bucket.getName()).size();
}
client.listOwnedBuckets().get(10, TimeUnit.SECONDS);
}
private static final String sysHttpStreamUrl = System
.getProperty("jclouds.s3.httpstream.url");
private static final String sysHttpStreamMd5 = System
.getProperty("jclouds.s3.httpstream.md5");
private static final String sysHttpStreamUrl = System.getProperty("jclouds.s3.httpstream.url");
private static final String sysHttpStreamMd5 = System.getProperty("jclouds.s3.httpstream.md5");
@Test(groups = {"integration"})
@Parameters({"jclouds.s3.httpstream.url", "jclouds.s3.httpstream.md5"})
public void testCopyUrl(@Optional String httpStreamUrl,
@Optional String httpStreamMd5) throws Exception {
httpStreamUrl = checkNotNull(httpStreamUrl != null ? httpStreamUrl
: sysHttpStreamUrl, "httpStreamUrl");
@Test(groups = { "integration" })
@Parameters( { "jclouds.s3.httpstream.url", "jclouds.s3.httpstream.md5" })
public void testCopyUrl(@Optional String httpStreamUrl, @Optional String httpStreamMd5)
throws Exception {
httpStreamUrl = checkNotNull(httpStreamUrl != null ? httpStreamUrl : sysHttpStreamUrl,
"httpStreamUrl");
httpStreamMd5 = checkNotNull(httpStreamMd5 != null ? httpStreamMd5
: sysHttpStreamMd5, "httpStreamMd5");
httpStreamMd5 = checkNotNull(httpStreamMd5 != null ? httpStreamMd5 : sysHttpStreamMd5,
"httpStreamMd5");
String bucketName = bucketPrefix + "tcu";
createBucketAndEnsureEmpty(bucketName);
String key = "hello";
URL url = new URL(httpStreamUrl);
byte[] md5 = S3Utils
.fromHexString(httpStreamMd5);
byte[] md5 = S3Utils.fromHexString(httpStreamMd5);
URLConnection connection = url.openConnection();
int length = connection.getContentLength();
@ -87,8 +79,7 @@ public class S3ConnectionIntegrationTest extends S3IntegrationTest {
object.getMetadata().setMd5(md5);
object.getMetadata().setSize(length);
byte[] newMd5 = client.putObject(bucketName, object).get(30,
TimeUnit.SECONDS);
byte[] newMd5 = client.putObject(bucketName, object).get(30, TimeUnit.SECONDS);
assertEquals(newMd5, md5);
}
}

146
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java
View File

@ -24,7 +24,27 @@
package org.jclouds.aws.s3;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.inject.Module;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.ConsoleHandler;
import java.util.logging.Formatter;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import org.jclouds.aws.s3.config.StubS3ConnectionModule;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
@ -32,16 +52,15 @@ import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.aws.s3.util.S3Utils;
import org.jclouds.http.HttpConstants;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import static org.testng.Assert.assertEquals;
import org.testng.ITestContext;
import org.testng.annotations.*;
import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.*;
import java.util.logging.*;
import java.util.logging.Formatter;
import com.google.inject.Module;
public class S3IntegrationTest {
protected static final String TEST_STRING = "<apples><apple name=\"fuji\"></apple> </apples>";
@ -50,16 +69,16 @@ public class S3IntegrationTest {
protected byte[] badMd5;
protected String bucketName;
protected void createBucketAndEnsureEmpty(String sourceBucket)
throws InterruptedException, ExecutionException, TimeoutException {
protected void createBucketAndEnsureEmpty(String sourceBucket) throws InterruptedException,
ExecutionException, TimeoutException {
client.deleteBucketIfEmpty(sourceBucket).get(10, TimeUnit.SECONDS);
client.putBucketIfNotExists(sourceBucket).get(10, TimeUnit.SECONDS);
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS)
.getContents().size(), 0);
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS).getContents().size(),
0, "bucket " + sourceBucket + "wasn't empty");
}
protected void addObjectToBucket(String sourceBucket, String key)
throws InterruptedException, ExecutionException, TimeoutException,
IOException {
protected void addObjectToBucket(String sourceBucket, String key) throws InterruptedException,
ExecutionException, TimeoutException, IOException {
S3Object sourceObject = new S3Object(key);
sourceObject.getMetadata().setContentType("text/xml");
sourceObject.setData(TEST_STRING);
@ -67,24 +86,21 @@ public class S3IntegrationTest {
}
protected void addObjectToBucket(String sourceBucket, S3Object object)
throws InterruptedException, ExecutionException, TimeoutException,
IOException {
throws InterruptedException, ExecutionException, TimeoutException, IOException {
client.putObject(sourceBucket, object).get(10, TimeUnit.SECONDS);
}
protected S3Object validateContent(String sourceBucket, String key)
throws InterruptedException, ExecutionException, TimeoutException,
IOException {
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS)
.getContents().size(), 1);
S3Object newObject = client.getObject(sourceBucket, key).get(10,
TimeUnit.SECONDS);
protected S3Object validateContent(String sourceBucket, String key) throws InterruptedException,
ExecutionException, TimeoutException, IOException {
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS).getContents().size(),
1);
S3Object newObject = client.getObject(sourceBucket, key).get(10, TimeUnit.SECONDS);
assert newObject != S3Object.NOT_FOUND;
assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING);
return newObject;
}
@BeforeClass(groups = {"integration", "live"})
@BeforeClass(groups = { "integration", "live" })
void enableDebug() {
if (debugEnabled()) {
Handler HANDLER = new ConsoleHandler() {
@ -94,14 +110,10 @@ public class S3IntegrationTest {
@Override
public String format(LogRecord record) {
return String.format(
"[%tT %-7s] [%-7s] [%s]: %s %s\n",
new Date(record.getMillis()), record
.getLevel(), Thread.currentThread()
.getName(), record.getLoggerName(),
record.getMessage(),
record.getThrown() == null ? "" : record
.getThrown());
return String.format("[%tT %-7s] [%-7s] [%s]: %s %s\n", new Date(record
.getMillis()), record.getLevel(), Thread.currentThread().getName(),
record.getLoggerName(), record.getMessage(),
record.getThrown() == null ? "" : record.getThrown());
}
});
}
@ -115,34 +127,33 @@ public class S3IntegrationTest {
protected S3Connection client;
protected S3Context context = null;
protected String bucketPrefix = (System.getProperty("user.name") + "." + this
.getClass().getSimpleName()).toLowerCase();
protected String bucketPrefix = (System.getProperty("user.name") + "." + this.getClass()
.getSimpleName()).toLowerCase();
private static final String sysAWSAccessKeyId = System
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
private static final String sysAWSSecretAccessKey = System
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
@BeforeClass(inheritGroups = false, groups = {"integration", "live"})
@Parameters({S3Constants.PROPERTY_AWS_ACCESSKEYID,
S3Constants.PROPERTY_AWS_SECRETACCESSKEY})
@BeforeClass(inheritGroups = false, groups = { "integration", "live" })
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpCredentials(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey, ITestContext testContext) throws Exception {
AWSAccessKeyId = AWSAccessKeyId != null ? AWSAccessKeyId
: sysAWSAccessKeyId;
AWSSecretAccessKey = AWSSecretAccessKey != null ? AWSSecretAccessKey
: sysAWSSecretAccessKey;
AWSAccessKeyId = AWSAccessKeyId != null ? AWSAccessKeyId : sysAWSAccessKeyId;
AWSSecretAccessKey = AWSSecretAccessKey != null ? AWSSecretAccessKey : sysAWSSecretAccessKey;
if (AWSAccessKeyId != null)
testContext.setAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID, AWSAccessKeyId);
if (AWSSecretAccessKey != null)
testContext.setAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, AWSSecretAccessKey);
}
@BeforeClass(dependsOnMethods = {"setUpCredentials"}, groups = {"integration", "live"})
@BeforeClass(dependsOnMethods = { "setUpCredentials" }, groups = { "integration", "live" })
protected void setUpClient(ITestContext testContext) throws Exception {
if (testContext.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID) != null) {
String AWSAccessKeyId = (String) testContext.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID);
String AWSSecretAccessKey = (String) testContext.getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
String AWSAccessKeyId = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID);
String AWSSecretAccessKey = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
createLiveS3Context(AWSAccessKeyId, AWSSecretAccessKey);
} else {
createStubS3Context();
@ -161,20 +172,20 @@ public class S3IntegrationTest {
}
protected void createLiveS3Context(String AWSAccessKeyId, String AWSSecretAccessKey) {
context = S3ContextFactory.createS3Context(buildS3Properties(
checkNotNull(AWSAccessKeyId, "AWSAccessKeyId"),
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey")),
context = S3ContextFactory.createS3Context(buildS3Properties(checkNotNull(AWSAccessKeyId,
"AWSAccessKeyId"), checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey")),
createHttpModule());
}
@BeforeMethod(dependsOnMethods = "deleteBucket", groups = {"integration", "live"})
public void setUpBucket(Method method) throws TimeoutException, ExecutionException, InterruptedException {
@BeforeMethod(dependsOnMethods = "deleteBucket", groups = { "integration", "live" })
public void setUpBucket(Method method) throws TimeoutException, ExecutionException,
InterruptedException {
bucketName = (bucketPrefix + method.getName()).toLowerCase();
createBucketAndEnsureEmpty(bucketName);
}
@BeforeMethod(groups = {"integration", "live"})
@AfterMethod(groups = {"integration", "live"})
@BeforeMethod(groups = { "integration", "live" })
@AfterMethod(groups = { "integration", "live" })
public void deleteBucket() throws TimeoutException, ExecutionException, InterruptedException {
if (bucketName != null)
deleteBucket(bucketName);
@ -184,14 +195,12 @@ public class S3IntegrationTest {
return false;
}
protected Properties buildS3Properties(String AWSAccessKeyId,
String AWSSecretAccessKey) {
Properties properties = new Properties(
S3ContextFactory.DEFAULT_PROPERTIES);
properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID,
checkNotNull(AWSAccessKeyId, "AWSAccessKeyId"));
properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY,
checkNotNull(AWSSecretAccessKey, "AWSSecretAccessKey"));
protected Properties buildS3Properties(String AWSAccessKeyId, String AWSSecretAccessKey) {
Properties properties = new Properties(S3ContextFactory.DEFAULT_PROPERTIES);
properties.setProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID, checkNotNull(AWSAccessKeyId,
"AWSAccessKeyId"));
properties.setProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY, checkNotNull(
AWSSecretAccessKey, "AWSSecretAccessKey"));
properties.setProperty(HttpConstants.PROPERTY_HTTP_SECURE, "false");
properties.setProperty(HttpConstants.PROPERTY_HTTP_PORT, "80");
return properties;
@ -203,8 +212,7 @@ public class S3IntegrationTest {
protected void deleteEverything() throws Exception {
try {
List<S3Bucket.Metadata> metadata = client.listOwnedBuckets().get(
10, TimeUnit.SECONDS);
List<S3Bucket.Metadata> metadata = client.listOwnedBuckets().get(10, TimeUnit.SECONDS);
for (S3Bucket.Metadata metaDatum : metadata) {
if (metaDatum.getName().startsWith(bucketPrefix.toLowerCase())) {
deleteBucket(metaDatum.getName());
@ -216,23 +224,21 @@ public class S3IntegrationTest {
}
}
private void deleteBucket(String name) throws InterruptedException, ExecutionException, TimeoutException {
private void deleteBucket(String name) throws InterruptedException, ExecutionException,
TimeoutException {
if (client.bucketExists(name).get(10, TimeUnit.SECONDS)) {
List<Future<Boolean>> results = new ArrayList<Future<Boolean>>();
S3Bucket bucket = client.listBucket(name)
.get(10, TimeUnit.SECONDS);
S3Bucket bucket = client.listBucket(name).get(10, TimeUnit.SECONDS);
for (S3Object.Metadata objectMeta : bucket.getContents()) {
results.add(client.deleteObject(name,
objectMeta.getKey()));
results.add(client.deleteObject(name, objectMeta.getKey()));
}
Iterator<Future<Boolean>> iterator = results.iterator();
while (iterator.hasNext()) {
iterator.next().get(10, TimeUnit.SECONDS);
iterator.remove();
}
client.deleteBucketIfEmpty(name).get(10,
TimeUnit.SECONDS);
client.deleteBucketIfEmpty(name).get(10, TimeUnit.SECONDS);
}
}

2
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3UtilsTest.java
View File

@ -44,7 +44,7 @@ import java.util.concurrent.ExecutorCompletionService;
*
* @author Adrian Cole
*/
@Test(groups = "performance", sequential = true, testName = "s3.PerformanceTest")
@Test(groups = "performance", sequential = true, testName = "s3.S3UtilsTest")
public class S3UtilsTest extends PerformanceTest {
@Test(dataProvider = "hmacsha1")

58
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/S3CommandFactoryTest.java
View File

@ -23,45 +23,50 @@
*/
package org.jclouds.aws.s3.commands;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.commands.options.CopyObjectOptions;
import org.jclouds.aws.s3.commands.options.GetObjectOptions;
import org.jclouds.aws.s3.commands.options.ListBucketOptions;
import org.jclouds.aws.s3.commands.options.PutBucketOptions;
import org.jclouds.aws.s3.commands.options.PutObjectOptions;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Bucket.Metadata.LocationConstraint;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.name.Names;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import org.jclouds.aws.s3.commands.config.S3CommandsModule;
import org.jclouds.aws.s3.commands.options.*;
import org.jclouds.aws.s3.domain.S3Bucket.Metadata.LocationConstraint;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/**
* @author Adrian Cole
*/
@Test(groups = {"unit"}, testName = "s3.S3CommandFactoryTest")
@Test(groups = { "unit" }, testName = "s3.S3CommandFactoryTest")
public class S3CommandFactoryTest {
Injector injector = null;
S3CommandFactory commandFactory = null;
@BeforeMethod
@BeforeTest
void setUpInjector() {
injector = Guice.createInjector(new S3CommandsModule() {
@Override
protected void configure() {
bindConstant().annotatedWith(
Names.named("jclouds.http.address")).to("localhost");
bindConstant().annotatedWith(Names.named("jclouds.http.address")).to("localhost");
super.configure();
}
});
commandFactory = injector.getInstance(S3CommandFactory.class);
}
@AfterMethod
@AfterTest
void tearDownInjector() {
commandFactory = null;
injector = null;
@ -69,14 +74,14 @@ public class S3CommandFactoryTest {
@Test
void testCreateCopyObject() {
assert commandFactory.createCopyObject("sourcebucket", "sourceObject",
"destbucket", "destObject", CopyObjectOptions.NONE) != null;
assert commandFactory.createCopyObject("sourcebucket", "sourceObject", "destbucket",
"destObject", CopyObjectOptions.NONE) != null;
}
@Test
void testCreateCopyObjectOptions() {
assert commandFactory.createCopyObject("sourcebucket", "sourceObject",
"destbucket", "destObject", new CopyObjectOptions()) != null;
assert commandFactory.createCopyObject("sourcebucket", "sourceObject", "destbucket",
"destObject", new CopyObjectOptions()) != null;
}
@Test
@ -113,26 +118,21 @@ public class S3CommandFactoryTest {
expect(metadata.getKey()).andReturn("rawr");
expect(metadata.getContentType()).andReturn("text/xml").atLeastOnce();
expect(metadata.getCacheControl()).andReturn("no-cache").atLeastOnce();
expect(metadata.getContentDisposition()).andReturn("disposition")
.atLeastOnce();
expect(metadata.getContentEncoding()).andReturn("encoding")
.atLeastOnce();
expect(metadata.getMd5()).andReturn("encoding".getBytes())
.atLeastOnce();
expect(metadata.getContentDisposition()).andReturn("disposition").atLeastOnce();
expect(metadata.getContentEncoding()).andReturn("encoding").atLeastOnce();
expect(metadata.getMd5()).andReturn("encoding".getBytes()).atLeastOnce();
Multimap<String, String> userMdata = HashMultimap.create();
expect(metadata.getUserMetadata()).andReturn(userMdata).atLeastOnce();
replay(metadata);
object.setData("<a></a>");
assert commandFactory.createPutObject("test", object,
PutObjectOptions.NONE) != null;
assert commandFactory.createPutObject("test", object, PutObjectOptions.NONE) != null;
}
@Test
void testCreateGetObject() {
assert commandFactory.createGetObject("test", "blah",
GetObjectOptions.NONE) != null;
assert commandFactory.createGetObject("test", "blah", GetObjectOptions.NONE) != null;
}
@Test

70
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/S3ParserTest.java
View File

@ -23,8 +23,16 @@
*/
package org.jclouds.aws.s3.commands;
import com.google.inject.Guice;
import com.google.inject.Injector;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.PerformanceTest;
import org.jclouds.aws.s3.domain.CanonicalUser;
@ -38,26 +46,20 @@ import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.jclouds.http.HttpException;
import org.jclouds.http.commands.callables.xml.ParseSax;
import org.joda.time.DateTime;
import static org.testng.Assert.assertEquals;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import org.xml.sax.SAXException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import com.google.inject.Guice;
import com.google.inject.Injector;
/**
* Tests parsing of S3 responses
*
* @author Adrian Cole
*/
@Test(groups = {"performance"}, testName = "s3.S3ParserTest")
@Test(groups = { "performance" }, testName = "s3.S3ParserTest")
public class S3ParserTest extends PerformanceTest {
Injector injector = null;
@ -65,14 +67,14 @@ public class S3ParserTest extends PerformanceTest {
S3ParserFactory parserFactory = null;
@BeforeMethod
@BeforeTest
protected void setUpInjector() {
injector = Guice.createInjector(new S3ParserModule());
parserFactory = injector.getInstance(S3ParserFactory.class);
assert parserFactory != null;
}
@AfterMethod
@AfterTest
protected void tearDownInjector() {
parserFactory = null;
injector = null;
@ -84,21 +86,19 @@ public class S3ParserTest extends PerformanceTest {
runParseListAllMyBuckets();
}
private List<S3Bucket.Metadata> runParseListAllMyBuckets()
throws HttpException {
private List<S3Bucket.Metadata> runParseListAllMyBuckets() throws HttpException {
return parserFactory.createListBucketsParser().parse(
IOUtils.toInputStream(listAllMyBucketsResultOn200));
}
@Test
void testParseListAllMyBucketsParallelResponseTime()
throws InterruptedException, ExecutionException {
void testParseListAllMyBucketsParallelResponseTime() throws InterruptedException,
ExecutionException {
CompletionService<List<S3Bucket.Metadata>> completer = new ExecutorCompletionService<List<S3Bucket.Metadata>>(
exec);
for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<List<S3Bucket.Metadata>>() {
public List<S3Bucket.Metadata> call() throws IOException,
SAXException, HttpException {
public List<S3Bucket.Metadata> call() throws IOException, SAXException, HttpException {
return runParseListAllMyBuckets();
}
});
@ -128,8 +128,7 @@ public class S3ParserTest extends PerformanceTest {
public static final String listBucketResult = "<ListBucketHandler xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>adrianjbosstest</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>3366</Key><LastModified>2009-03-12T02:00:13.000Z</LastModified><ETag>&quot;9d7bb64e8e18ee34eec06dd2cf37b766&quot;</ETag><Size>136</Size><Owner><ID>e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0</ID><DisplayName>ferncam</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketHandler>";
public void testCanParseListBucketResult() throws HttpException,
UnsupportedEncodingException {
public void testCanParseListBucketResult() throws HttpException, UnsupportedEncodingException {
S3Bucket bucket = runParseListBucketResult();
assert !bucket.isTruncated();
assert bucket.getName().equals("adrianjbosstest");
@ -138,10 +137,8 @@ public class S3ParserTest extends PerformanceTest {
assert object.getKey().equals("3366");
DateTime expected = new DateTime("2009-03-12T02:00:13.000Z");
assert object.getLastModified().equals(expected) : String.format(
"expected %1$s, but got %1$s", expected, object
.getLastModified());
assertEquals(S3Utils.toHexString(object.getMd5()),
"9d7bb64e8e18ee34eec06dd2cf37b766");
"expected %1$s, but got %1$s", expected, object.getLastModified());
assertEquals(S3Utils.toHexString(object.getMd5()), "9d7bb64e8e18ee34eec06dd2cf37b766");
assert object.getSize() == 136;
CanonicalUser owner = new CanonicalUser(
"e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0");
@ -160,20 +157,17 @@ public class S3ParserTest extends PerformanceTest {
public static final String successfulCopyObject200 = "<CopyObjectResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><LastModified>2009-03-19T13:23:27.000Z</LastModified><ETag>\"92836a3ea45a6984d1b4d23a747d46bb\"</ETag></CopyObjectResult>";
private S3Object.Metadata runParseCopyObjectResult() throws HttpException {
ParseSax<S3Object.Metadata> parser = parserFactory
.createCopyObjectParser();
ParseSax<S3Object.Metadata> parser = parserFactory.createCopyObjectParser();
CopyObjectHandler handler = (CopyObjectHandler) parser.getHandler();
handler.setKey("adrianjbosstest");
return parser.parse(IOUtils.toInputStream(successfulCopyObject200));
}
public void testCanParseCopyObjectResult() throws HttpException,
UnsupportedEncodingException {
public void testCanParseCopyObjectResult() throws HttpException, UnsupportedEncodingException {
S3Object.Metadata metadata = runParseCopyObjectResult();
DateTime expected = new DateTime("2009-03-19T13:23:27.000Z");
assertEquals(metadata.getLastModified(), expected);
assertEquals(S3Utils.toHexString(metadata.getMd5()),
"92836a3ea45a6984d1b4d23a747d46bb");
assertEquals(S3Utils.toHexString(metadata.getMd5()), "92836a3ea45a6984d1b4d23a747d46bb");
assertEquals(metadata.getKey(), "adrianjbosstest");
}
@ -184,14 +178,12 @@ public class S3ParserTest extends PerformanceTest {
}
@Test
void testParseListBucketResultParallelResponseTime()
throws InterruptedException, ExecutionException {
CompletionService<S3Bucket> completer = new ExecutorCompletionService<S3Bucket>(
exec);
void testParseListBucketResultParallelResponseTime() throws InterruptedException,
ExecutionException {
CompletionService<S3Bucket> completer = new ExecutorCompletionService<S3Bucket>(exec);
for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<S3Bucket>() {
public S3Bucket call() throws IOException, SAXException,
HttpException {
public S3Bucket call() throws IOException, SAXException, HttpException {
return runParseListBucketResult();
}
});

15
aws/s3/core/src/test/java/org/jclouds/aws/s3/commands/callables/ParseObjectFromHeadersAndHttpContentTest.java
View File

@ -23,16 +23,19 @@
*/
package org.jclouds.aws.s3.commands.callables;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.replay;
import static org.testng.Assert.assertEquals;
import org.apache.commons.io.IOUtils;
import static org.easymock.classextension.EasyMock.*;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Object.Metadata;
import org.jclouds.http.HttpException;
import org.jclouds.http.HttpHeaders;
import org.jclouds.http.HttpResponse;
import static org.testng.Assert.assertEquals;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
/**
@ -43,13 +46,13 @@ public class ParseObjectFromHeadersAndHttpContentTest {
ParseObjectFromHeadersAndHttpContent callable;
ParseMetadataFromHeaders metadataParser;
@BeforeMethod
@BeforeTest
void setUp() {
metadataParser = createMock(ParseMetadataFromHeaders.class);
callable = new ParseObjectFromHeadersAndHttpContent(metadataParser);
}
@AfterMethod
@AfterTest
void tearDown() {
callable = null;
}

121
aws/s3/core/src/test/java/org/jclouds/aws/s3/internal/BaseS3MapTest.java Normal file
View File

@ -0,0 +1,121 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3.internal;
import static org.easymock.EasyMock.expect;
import static org.easymock.classextension.EasyMock.createMock;
import static org.easymock.classextension.EasyMock.createNiceMock;
import static org.easymock.classextension.EasyMock.replay;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.jclouds.aws.s3.S3Connection;
import org.jclouds.aws.s3.domain.S3Object;
import org.testng.annotations.Test;
/**
*
* Tests retry logic.
*
* @author Adrian Cole
*/
@Test(groups = { "unit" }, testName = "s3.BaseS3MapTest")
public class BaseS3MapTest {
class MockBaseS3Map extends BaseS3Map<String> {
public MockBaseS3Map() {
super(createNiceMock(S3Connection.class), "bucket");
}
public Set<java.util.Map.Entry<String, String>> entrySet() {
return null;
}
public String get(Object key) {
return null;
}
public String put(String key, String value) {
return null;
}
public void putAll(Map<? extends String, ? extends String> t) {
}
public String remove(Object key) {
return null;
}
public Collection<String> values() {
return null;
}
}
public void testIfNotFoundRetryOtherwiseAddToSet() throws InterruptedException,
ExecutionException, TimeoutException {
BaseS3Map<String> map = new MockBaseS3Map();
Future<S3Object> futureObject = createMock(Future.class);
S3Object object = createNiceMock(S3Object.class);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
S3Object.NOT_FOUND);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
object);
replay(futureObject);
Set<S3Object> objects = new HashSet<S3Object>();
long time = System.currentTimeMillis();
map.ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
// should have retried once
assert System.currentTimeMillis() >= time + map.requestRetryMilliseconds;
assert objects.contains(object);
assert !objects.contains(S3Object.NOT_FOUND);
}
public void testIfNotFoundRetryOtherwiseAddToSetButNeverGetsIt() throws InterruptedException,
ExecutionException, TimeoutException {
BaseS3Map<String> map = new MockBaseS3Map();
Future<S3Object> futureObject = createMock(Future.class);
S3Object object = createNiceMock(S3Object.class);
expect(futureObject.get(map.requestTimeoutMilliseconds, TimeUnit.MILLISECONDS)).andReturn(
S3Object.NOT_FOUND).atLeastOnce();
replay(futureObject);
Set<S3Object> objects = new HashSet<S3Object>();
long time = System.currentTimeMillis();
map.ifNotFoundRetryOtherwiseAddToSet(futureObject, objects);
// should have retried thrice
assert System.currentTimeMillis() >= time + map.requestRetryMilliseconds * 3;
assert !objects.contains(object);
assert !objects.contains(S3Object.NOT_FOUND);
}
}

13
aws/s3/core/src/test/java/org/jclouds/aws/s3/xml/S3ParserFactoryTest.java
View File

@ -23,12 +23,13 @@
*/
package org.jclouds.aws.s3.xml;
import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.inject.Guice;
import com.google.inject.Injector;
import org.jclouds.aws.s3.xml.config.S3ParserModule;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/**
* @author Adrian Cole
@ -39,13 +40,13 @@ public class S3ParserFactoryTest {
Injector injector = null;
S3ParserFactory parserFactory = null;
@BeforeMethod
@BeforeTest
void setUpInjector() {
injector = Guice.createInjector(new S3ParserModule());
parserFactory = injector.getInstance(S3ParserFactory.class);
}
@AfterMethod
@AfterTest
void tearDownInjector() {
parserFactory = null;
injector = null;

147
aws/s3/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java
View File

@ -40,7 +40,6 @@ import org.jets3t.service.acl.AccessControlList;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3BucketLoggingStatus;
import org.jets3t.service.model.S3Object;
import org.jets3t.service.model.S3Owner;
import org.jets3t.service.security.AWSCredentials;
import com.google.inject.Module;
@ -63,16 +62,18 @@ public class JCloudsS3Service extends S3Service {
/**
* Initializes a JClouds context to S3.
*
* @param awsCredentials - credentials to access S3
* @param modules - Module that configures a FutureHttpClient, if not specified,
* default is URLFetchServiceClientModule
* @param awsCredentials
* - credentials to access S3
* @param modules
* - Module that configures a FutureHttpClient, if not specified, default is
* URLFetchServiceClientModule
* @throws S3ServiceException
*/
protected JCloudsS3Service(AWSCredentials awsCredentials, Module... modules)
throws S3ServiceException {
super(awsCredentials);
context = S3ContextFactory.createS3Context(awsCredentials
.getAccessKey(), awsCredentials.getSecretKey(), modules);
context = S3ContextFactory.createS3Context(awsCredentials.getAccessKey(), awsCredentials
.getSecretKey(), modules);
connection = context.getConnection();
}
@ -87,20 +88,17 @@ public class JCloudsS3Service extends S3Service {
*/
@SuppressWarnings("unchecked")
@Override
protected Map copyObjectImpl(String sourceBucketName,
String sourceObjectKey, String destinationBucketName,
String destinationObjectKey, AccessControlList acl,
Map destinationMetadata, Calendar ifModifiedSince,
Calendar ifUnmodifiedSince, String[] ifMatchTags,
String[] ifNoneMatchTags) throws S3ServiceException {
protected Map copyObjectImpl(String sourceBucketName, String sourceObjectKey,
String destinationBucketName, String destinationObjectKey, AccessControlList acl,
Map destinationMetadata, Calendar ifModifiedSince, Calendar ifUnmodifiedSince,
String[] ifMatchTags, String[] ifNoneMatchTags) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3Bucket createBucketImpl(String bucketName, String location,
AccessControlList acl) throws S3ServiceException
{
protected S3Bucket createBucketImpl(String bucketName, String location, AccessControlList acl)
throws S3ServiceException {
if (location != null)
throw new UnsupportedOperationException("Bucket location is not yet supported");
if (acl != null)
@ -111,9 +109,8 @@ public class JCloudsS3Service extends S3Service {
// Bucket created.
}
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(
"error creating bucket: " + bucketName, e);
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error creating bucket: " + bucketName, e);
}
return new S3Bucket(bucketName);
}
@ -124,15 +121,13 @@ public class JCloudsS3Service extends S3Service {
* @see S3Connection#deleteBucketIfEmpty(String)
*/
@Override
protected void deleteBucketImpl(String bucketName)
throws S3ServiceException {
protected void deleteBucketImpl(String bucketName) throws S3ServiceException {
try {
connection.deleteBucketIfEmpty(bucketName).get(
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
connection.deleteBucketIfEmpty(bucketName).get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS);
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(
"error deleting bucket: " + bucketName, e);
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error deleting bucket: " + bucketName, e);
}
}
@ -142,28 +137,25 @@ public class JCloudsS3Service extends S3Service {
* @see S3Connection#deleteObject(String, String)
*/
@Override
protected void deleteObjectImpl(String bucketName, String objectKey)
throws S3ServiceException {
protected void deleteObjectImpl(String bucketName, String objectKey) throws S3ServiceException {
try {
connection.deleteObject(bucketName, objectKey).get(
requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
connection.deleteObject(bucketName, objectKey).get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS);
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format(
"error deleting object: %1$s:%2$s", bucketName, objectKey), e);
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format("error deleting object: %1$s:%2$s", bucketName,
objectKey), e);
}
}
@Override
protected AccessControlList getBucketAclImpl(String bucketName)
throws S3ServiceException {
protected AccessControlList getBucketAclImpl(String bucketName) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected String getBucketLocationImpl(String bucketName)
throws S3ServiceException {
protected String getBucketLocationImpl(String bucketName) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@ -176,18 +168,16 @@ public class JCloudsS3Service extends S3Service {
}
@Override
protected AccessControlList getObjectAclImpl(String bucketName,
String objectKey) throws S3ServiceException {
protected AccessControlList getObjectAclImpl(String bucketName, String objectKey)
throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3Object getObjectDetailsImpl(String bucketName,
String objectKey, Calendar ifModifiedSince,
Calendar ifUnmodifiedSince, String[] ifMatchTags,
String[] ifNoneMatchTags) throws S3ServiceException
{
protected S3Object getObjectDetailsImpl(String bucketName, String objectKey,
Calendar ifModifiedSince, Calendar ifUnmodifiedSince, String[] ifMatchTags,
String[] ifNoneMatchTags) throws S3ServiceException {
try {
if (ifModifiedSince != null)
throw new IllegalArgumentException("ifModifiedSince");
@ -198,43 +188,37 @@ public class JCloudsS3Service extends S3Service {
if (ifNoneMatchTags != null)
throw new IllegalArgumentException("ifNoneMatchTags");
return Util.convertObjectHead(
connection.headObject(bucketName, objectKey).get());
return Util.convertObjectHead(connection.headObject(bucketName, objectKey).get());
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format(
"error retrieving object head: %1$s:%2$s", bucketName, objectKey), e);
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format("error retrieving object head: %1$s:%2$s",
bucketName, objectKey), e);
}
}
@Override
protected S3Object getObjectImpl(String bucketName, String objectKey,
Calendar ifModifiedSince, Calendar ifUnmodifiedSince,
String[] ifMatchTags, String[] ifNoneMatchTags,
Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException
{
protected S3Object getObjectImpl(String bucketName, String objectKey, Calendar ifModifiedSince,
Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags,
Long byteRangeStart, Long byteRangeEnd) throws S3ServiceException {
try {
GetObjectOptions options = Util.convertOptions(
ifModifiedSince, ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags);
return Util.convertObject(
connection.getObject(bucketName, objectKey, options).get());
GetObjectOptions options = Util.convertOptions(ifModifiedSince, ifUnmodifiedSince,
ifMatchTags, ifNoneMatchTags);
return Util.convertObject(connection.getObject(bucketName, objectKey, options).get());
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format(
"error retrieving object: %1$s:%2$s", bucketName, objectKey), e);
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException(String.format("error retrieving object: %1$s:%2$s",
bucketName, objectKey), e);
}
}
@Override
public boolean isBucketAccessible(String bucketName)
throws S3ServiceException {
public boolean isBucketAccessible(String bucketName) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected boolean isRequesterPaysBucketImpl(String bucketName)
throws S3ServiceException {
protected boolean isRequesterPaysBucketImpl(String bucketName) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@ -243,27 +227,25 @@ public class JCloudsS3Service extends S3Service {
protected S3Bucket[] listAllBucketsImpl() throws S3ServiceException {
try {
List<org.jclouds.aws.s3.domain.S3Bucket.Metadata> jcBucketList = connection
.listOwnedBuckets().get(requestTimeoutMilliseconds,
TimeUnit.MILLISECONDS);
.listOwnedBuckets().get(requestTimeoutMilliseconds, TimeUnit.MILLISECONDS);
return Util.convertBuckets(jcBucketList);
} catch (Exception e) {
Utils.<S3ServiceException>rethrowIfRuntimeOrSameType(e);
Utils.<S3ServiceException> rethrowIfRuntimeOrSameType(e);
throw new S3ServiceException("error listing buckets", e);
}
}
@Override
protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName,
String prefix, String delimiter, long maxListingLength,
String priorLastKey, boolean completeListing)
protected S3ObjectsChunk listObjectsChunkedImpl(String bucketName, String prefix,
String delimiter, long maxListingLength, String priorLastKey, boolean completeListing)
throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3Object[] listObjectsImpl(String bucketName, String prefix,
String delimiter, long maxListingLength) throws S3ServiceException {
protected S3Object[] listObjectsImpl(String bucketName, String prefix, String delimiter,
long maxListingLength) throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@ -277,40 +259,33 @@ public class JCloudsS3Service extends S3Service {
}
@Override
protected void putObjectAclImpl(String bucketName, String objectKey,
AccessControlList acl) throws S3ServiceException {
protected void putObjectAclImpl(String bucketName, String objectKey, AccessControlList acl)
throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3Object putObjectImpl(String bucketName, S3Object object)
throws S3ServiceException {
protected S3Object putObjectImpl(String bucketName, S3Object object) throws S3ServiceException {
// TODO Unimplemented
return null;
}
@Override
protected void setBucketLoggingStatusImpl(String bucketName,
S3BucketLoggingStatus status) throws S3ServiceException {
protected void setBucketLoggingStatusImpl(String bucketName, S3BucketLoggingStatus status)
throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected void setRequesterPaysBucketImpl(String bucketName,
boolean requesterPays) throws S3ServiceException {
protected void setRequesterPaysBucketImpl(String bucketName, boolean requesterPays)
throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
@Override
protected S3Owner getAccountOwnerImpl() throws S3ServiceException {
// TODO Unimplemented
throw new UnsupportedOperationException();
}
}

27
aws/s3/extensions/jets3t/src/test/java/org/jclouds/aws/s3/jets3t/JCloudsS3ServiceIntegrationTest.java
View File

@ -23,6 +23,19 @@
*/
package org.jclouds.aws.s3.jets3t;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.jclouds.aws.s3.config.StubS3ConnectionModule;
@ -31,26 +44,12 @@ import org.jets3t.service.S3ServiceException;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;
import org.jets3t.service.security.AWSCredentials;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.fail;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* Tests to cover JCloudsS3Service
*

78
aws/s3/perftest/pom.xml
View File

@ -1,42 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
$HeadURL$
$Revision$
$Date$
Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
<!--
$HeadURL$ $Revision$ $Date$ Copyright (C) 2009 Adrian Cole
<adrian@jclouds.org>
====================================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to you under the Apache License, Version
2.0 (the "License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0.html
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
http://www.apache.org/licenses/LICENSE-2.0.html Unless required by
applicable law or agreed to in writing, software distributed under the
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the
License.
====================================================================
-->
<!--
Note that the code that these performance tests are evaluated against exist in src/main. The code in that
location was copied straight from Amazon's website. There have been small modifications to make unit testing
possible. That code exists with the following license:
This software code is made available "AS IS" without warranties of any
kind. You may copy, display, modify and redistribute the software
code either by itself or as incorporated into your code; provided that
you do not remove any proprietary notices. Your use of this software
code is at your own risk and you waive any claim against Amazon
Digital Services, Inc. or its affiliates with respect to your use of
this software code. (c) 2006 Amazon Digital Services, Inc. or its
<!--
Note that the code that these performance tests are evaluated against
exist in src/main. The code in that location was copied straight from
Amazon's website. There have been small modifications to make unit
testing possible. That code exists with the following license: This
software code is made available "AS IS" without warranties of any
kind. You may copy, display, modify and redistribute the software code
either by itself or as incorporated into your code; provided that you
do not remove any proprietary notices. Your use of this software code
is at your own risk and you waive any claim against Amazon Digital
Services, Inc. or its affiliates with respect to your use of this
software code. (c) 2006 Amazon Digital Services, Inc. or its
affiliates.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@ -90,14 +84,16 @@
<developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk/s3core/perftest</developerConnection>
<url>http://jclouds.googlecode.com/svn/trunk/s3core/perftest</url>
</scm>
<build>
<repositories>
<!-- For Amazon S3 artifacts -->
<repository>
<name>jets3t</name>
<id>jets3t</id>
<url>http://jets3t.s3.amazonaws.com/maven2</url>
</repository>
</repositories>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<parallel>classes</parallel>
</configuration>
</plugin>
</plugins>
</build>
</project>

131
aws/s3/perftest/src/test/java/com/amazon/s3/AmazonPerformanceLiveTest.java
View File

@ -1,131 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.jclouds.aws.s3.reference.S3Constants;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import java.io.File;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
/**
* Runs operations that amazon s3 sample code is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformanceLiveTest", groups = {"live"})
public class AmazonPerformanceLiveTest extends BasePerformance {
private AWSAuthConnection amzClient;
@Override
@BeforeTest
@Parameters({S3Constants.PROPERTY_AWS_ACCESSKEYID,
S3Constants.PROPERTY_AWS_SECRETACCESSKEY})
protected void setUpCredentials(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey, ITestContext context) throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey,
false);
}
@Override
@Test(enabled = false)
public void testPutFileSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
com.amazon.s3.S3Object object = new com.amazon.s3.S3Object(data, null);
Map<String, List<String>> headers = new TreeMap<String, List<String>>();
headers
.put("Content-Type", Arrays
.asList(new String[]{contentType}));
return amzClient.put(bucket, key, object, headers).connection
.getResponseMessage() != null;
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
}

94
aws/s3/perftest/src/test/java/com/amazon/s3/BaseJCloudsPerformance.java
View File

@ -1,94 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.TimeUnit;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public abstract class BaseJCloudsPerformance extends BasePerformance {
// boolean get
// (
// int id) throws Exception {
// S3Bucket s3Bucket = new S3Bucket();
// s3Bucket.setName(bucketPrefix + "-jclouds-puts");
// org.jclouds.aws.s3.domain.S3Object object = new
// org.jclouds.aws.s3.domain.S3Object();
// object.setKey(id + "");
// //object.setContentType("text/plain");
// object.setContentType("application/octetstream");
// //object.setData("this is a test");
// object.setData(test);
// return clientProvider.getObject(s3Bucket,
// object.getKey()).get(120,TimeUnit.SECONDS) !=
// org.jclouds.aws.s3.domain.S3Object.NOT_FOUND;
// }
@Override
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
object.getMetadata().setSize(data.available());
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(
key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
}

252
aws/s3/perftest/src/test/java/com/amazon/s3/BasePerformance.java
View File

@ -1,252 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import com.google.inject.Provider;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.testng.ITestContext;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
@Test(groups = {"live"})
public abstract class BasePerformance extends S3IntegrationTest {
protected boolean debugEnabled() {
return false;
}
protected static int LOOP_COUNT = 100;
protected ExecutorService exec;
protected final String BUCKET_BYTES = bucketPrefix + "-bytes";
protected final String BUCKET_INPUTSTREAM = bucketPrefix + "-inputstream";
protected final String BUCKET_STRING = bucketPrefix + "-string";
protected final String BUCKET_FILE = bucketPrefix + "-file";
protected final String[] BUCKETS = {BUCKET_BYTES, BUCKET_INPUTSTREAM,
BUCKET_STRING, BUCKET_FILE};
protected PutBytesCallable putBytesCallable;
protected PutFileCallable putFileCallable;
protected PutInputStreamCallable putInputStreamCallable;
protected PutStringCallable putStringCallable;
protected CompletionService<Boolean> completer;
@BeforeTest
protected void setUpCallables() {
putBytesCallable = new PutBytesCallable();
putFileCallable = new PutFileCallable();
putInputStreamCallable = new PutInputStreamCallable();
putStringCallable = new PutStringCallable();
exec = Executors.newCachedThreadPool();
completer = new ExecutorCompletionService<Boolean>(exec);
}
@Override
@BeforeTest
protected void setUpClient(ITestContext context) throws Exception {
super.setUpClient(context);
for (String bucket : BUCKETS) {
client.putBucketIfNotExists(bucket).get(10, TimeUnit.SECONDS);
}
}
@AfterTest
protected void tearDownExecutor() throws Exception {
exec.shutdownNow();
exec = null;
}
@Test(enabled = true)
public void testPutBytesSerial() throws Exception {
doSerial(putBytesCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutBytesParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putBytesCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutFileSerial() throws Exception {
doSerial(putFileCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutFileParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putFileCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutInputStreamSerial() throws Exception {
doSerial(putInputStreamCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putInputStreamCallable, LOOP_COUNT);
}
@Test(enabled = true)
public void testPutStringSerial() throws Exception {
doSerial(putStringCallable, LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutStringParallel() throws InterruptedException,
ExecutionException, TimeoutException {
doParallel(putStringCallable, LOOP_COUNT);
}
private void doSerial(Provider<Callable<Boolean>> provider, int loopCount)
throws Exception, ExecutionException {
for (int i = 0; i < loopCount; i++)
assert provider.get().call();
}
private void doParallel(Provider<Callable<Boolean>> provider, int loopCount)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < loopCount; i++)
completer.submit(provider.get());
for (int i = 0; i < loopCount; i++)
assert completer.take().get(10, TimeUnit.SECONDS);
}
class PutBytesCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected byte[] test = new byte[1024 * 2];
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putByteArray(BUCKET_BYTES, key.getAndIncrement()
+ "", test, "application/octetstring");
}
};
}
}
class PutFileCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml");
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putFile(BUCKET_FILE, key.getAndIncrement() + "",
file, "text/xml");
}
};
}
}
class PutInputStreamCallable extends PutBytesCallable {
final AtomicInteger key = new AtomicInteger(0);
@Override
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putInputStream(BUCKET_INPUTSTREAM, key
.getAndIncrement()
+ "", new ByteArrayInputStream(test),
"application/octetstring");
}
};
}
}
class PutStringCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected String testString = "hello world!";
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putString(BUCKET_STRING, key.getAndIncrement() + "",
testString, "text/plain");
}
};
}
}
protected abstract boolean putByteArray(String bucket, String key,
byte[] data, String contentType) throws Exception;
protected abstract boolean putFile(String bucket, String key, File data,
String contentType) throws Exception;
protected abstract boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception;
protected abstract boolean putString(String bucket, String key,
String data, String contentType) throws Exception;
// private class BucketDeleter implements Callable<Boolean> {
// private BucketDeleter(S3Bucket bucket) {
// this.bucket = bucket;
// }
//
// private S3Bucket bucket;
//
// @Override
// public String toString() {
// return "BucketDeleter{" + "bucket=" + bucket + '}';
// }
//
// public Boolean call() throws Exception {
// bucket =
// clientProvider.get(10,TimeUnit.SECONDS).getBucket(bucket).get(10,TimeUnit.SECONDS);
// List<Future<Boolean>> deletes = new ArrayList<Future<Boolean>>();
// for (org.jclouds.aws.s3.domain.S3Object object : bucket
// .getContents()) {
// deletes.add(clientProvider.get(10,TimeUnit.SECONDS).deleteObject(bucket,
// object.getKey()));
// }
// for (Future<Boolean> isdeleted : deletes)
// assert isdeleted.get(10,TimeUnit.SECONDS) :
// String.format("failed to delete %1$ss",
// isdeleted);
// return
// clientProvider.get(10,TimeUnit.SECONDS).deleteBucket(bucket).get(10,TimeUnit.SECONDS);
// }
// }
}

107
aws/s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java
View File

@ -23,6 +23,9 @@
*/
package com.amazon.s3;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import java.util.Date;
import java.util.Locale;
import java.util.concurrent.Callable;
@ -41,9 +44,6 @@ import org.testng.annotations.Test;
import com.google.inject.Guice;
import com.google.inject.Injector;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
/*
* TODO: Scrap any non-DateService references (eg Joda & Amazon) if/when
* we confirm that the DateService is fast enough.
@ -62,6 +62,7 @@ public class DateServiceTest extends PerformanceTest {
DateService dateService = i.getInstance(DateService.class);
private TestData[] testData;
class TestData {
public final String iso8601DateString;
public final String rfc822DateString;
@ -76,22 +77,24 @@ public class DateServiceTest extends PerformanceTest {
private long startTime;
public DateServiceTest() {
// Constant time test values, each TestData item must contain matching times!
testData = new TestData[] {
new TestData("2009-03-12T02:00:07.000Z", "Thu, 12 Mar 2009 02:00:07 GMT", new Date(1236823207000l)),
new TestData("2009-03-14T04:00:07.000Z", "Sat, 14 Mar 2009 04:00:07 GMT", new Date(1237003207000l)),
new TestData("2009-03-16T06:00:07.000Z", "Mon, 16 Mar 2009 06:00:07 GMT", new Date(1237183207000l)),
new TestData("2009-03-18T08:00:07.000Z", "Wed, 18 Mar 2009 08:00:07 GMT", new Date(1237363207000l)),
new TestData("2009-03-20T10:00:07.000Z", "Fri, 20 Mar 2009 10:00:07 GMT", new Date(1237543207000l))
};
new TestData("2009-03-12T02:00:07.000Z", "Thu, 12 Mar 2009 02:00:07 GMT", new Date(
1236823207000l)),
new TestData("2009-03-14T04:00:07.000Z", "Sat, 14 Mar 2009 04:00:07 GMT", new Date(
1237003207000l)),
new TestData("2009-03-16T06:00:07.000Z", "Mon, 16 Mar 2009 06:00:07 GMT", new Date(
1237183207000l)),
new TestData("2009-03-18T08:00:07.000Z", "Wed, 18 Mar 2009 08:00:07 GMT", new Date(
1237363207000l)),
new TestData("2009-03-20T10:00:07.000Z", "Fri, 20 Mar 2009 10:00:07 GMT", new Date(
1237543207000l)) };
}
// Joda items for performance comparisons
private DateTimeFormatter headerDateFormat = DateTimeFormat
.forPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.US);
private DateTimeFormatter headerDateFormat = DateTimeFormat.forPattern(
"EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.US);
private DateTime jodaParseIso8601(String toParse) {
return new DateTime(toParse);
@ -109,79 +112,63 @@ public class DateServiceTest extends PerformanceTest {
return headerDateFormat.print(date.withZone(DateTimeZone.UTC));
}
private void startClock() {
startTime = System.currentTimeMillis();
}
private void printElapsedClockTime(String testName) {
System.out.println(testName + " took " +
(System.currentTimeMillis() - startTime) + "ms for "
+ LOOP_COUNT+ " loops");
System.out.println(testName + " took " + (System.currentTimeMillis() - startTime) + "ms for "
+ LOOP_COUNT + " loops");
}
@Test
public void testIso8601DateParse() throws ExecutionException,
InterruptedException
{
public void testIso8601DateParse() throws ExecutionException, InterruptedException {
DateTime dsDate = dateService.iso8601DateParse(testData[0].iso8601DateString);
assertEquals(dsDate.toDate(), testData[0].date);
}
@Test
public void testRfc822DateParse() throws ExecutionException,
InterruptedException
{
public void testRfc822DateParse() throws ExecutionException, InterruptedException {
DateTime dsDate = dateService.rfc822DateParse(testData[0].rfc822DateString);
assertEquals(dsDate.toDate(), testData[0].date);
}
@Test
public void testIso8601DateFormat() throws ExecutionException,
InterruptedException
{
public void testIso8601DateFormat() throws ExecutionException, InterruptedException {
String dsString = dateService.iso8601DateFormat(testData[0].date);
assertEquals(dsString, testData[0].iso8601DateString);
}
@Test
public void testRfc822DateFormat() throws ExecutionException,
InterruptedException
{
public void testRfc822DateFormat() throws ExecutionException, InterruptedException {
String dsString = dateService.rfc822DateFormat(testData[0].date);
assertEquals(dsString, testData[0].rfc822DateString);
}
@Test
void testIso8601DateFormatResponseTime() throws ExecutionException,
InterruptedException {
void testIso8601DateFormatResponseTime() throws ExecutionException, InterruptedException {
for (int i = 0; i < LOOP_COUNT; i++)
dateService.iso8601DateFormat();
}
@Test
void testRfc822DateFormatResponseTime() throws ExecutionException,
InterruptedException {
void testRfc822DateFormatResponseTime() throws ExecutionException, InterruptedException {
for (int i = 0; i < LOOP_COUNT; i++)
dateService.rfc822DateFormat();
}
@Test
void testFormatIso8601DateInParallel() throws InterruptedException,
ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
void testFormatIso8601DateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() {
public Boolean call() throws ExecutionException,
InterruptedException
{
public Boolean call() throws ExecutionException, InterruptedException {
String dsString = dateService.iso8601DateFormat(myData.date);
/*
* Comment-in the assert below to test thread safety.
* Comment it out to test performance
* Comment-in the assert below to test thread safety. Comment it out to test
* performance
*/
assertEquals(dsString, myData.iso8601DateString);
return true;
@ -194,10 +181,8 @@ public class DateServiceTest extends PerformanceTest {
}
@Test
void testFormatAmazonDateInParallel() throws InterruptedException,
ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
void testFormatAmazonDateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<Boolean>() {
@ -212,10 +197,8 @@ public class DateServiceTest extends PerformanceTest {
}
@Test
void testFormatJodaDateInParallel() throws InterruptedException,
ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
void testFormatJodaDateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
@ -233,8 +216,7 @@ public class DateServiceTest extends PerformanceTest {
}
@Test
void testIso8601ParseDateSerialResponseTime() throws ExecutionException,
InterruptedException {
void testIso8601ParseDateSerialResponseTime() throws ExecutionException, InterruptedException {
for (int i = 0; i < LOOP_COUNT; i++)
dateService.iso8601DateParse(testData[0].iso8601DateString);
}
@ -246,16 +228,13 @@ public class DateServiceTest extends PerformanceTest {
}
@Test
void testParseIso8601DateInParallel() throws InterruptedException,
ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
void testParseIso8601DateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];
completer.submit(new Callable<Boolean>() {
public Boolean call() throws ExecutionException,
InterruptedException {
public Boolean call() throws ExecutionException, InterruptedException {
DateTime dsDate = dateService.iso8601DateParse(myData.iso8601DateString);
assertEquals(dsDate.toDate(), myData.date);
return true;
@ -268,10 +247,8 @@ public class DateServiceTest extends PerformanceTest {
}
@Test
void testParseAmazonDateInParallel() throws InterruptedException,
ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
void testParseAmazonDateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++)
completer.submit(new Callable<Boolean>() {
@ -286,10 +263,8 @@ public class DateServiceTest extends PerformanceTest {
}
@Test
void testParseJodaDateInParallel() throws InterruptedException,
ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(
exec);
void testParseJodaDateInParallel() throws InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
startClock();
for (int i = 0; i < LOOP_COUNT; i++) {
final TestData myData = testData[i % testData.length];

130
aws/s3/perftest/src/test/java/com/amazon/s3/Jets3tPerformanceLiveTest.java
View File

@ -1,130 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.jets3t.service.S3Service;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.security.AWSCredentials;
import org.testng.ITestContext;
import org.testng.annotations.Test;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.ExecutionException;
/**
* Runs operations that jets3t is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformanceLiveTest", groups = {"live"})
public class Jets3tPerformanceLiveTest extends BasePerformance {
private S3Service jetClient;
@Override
protected void setUpCredentials(String AWSAccessKeyId, String AWSSecretAccessKey, ITestContext context)
throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
jetClient = new RestS3Service(new AWSCredentials(AWSAccessKeyId,
AWSSecretAccessKey));
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException,
ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
protected boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putFile(String bucket, String key, File data,
String contentType) throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(
key);
object.setContentType(contentType);
object.setDataInputFile(data);
object.setContentLength(data.length());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putInputStream(String bucket, String key,
InputStream data, String contentType) throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(
key);
object.setContentType(contentType);
object.setDataInputStream(data);
object.setContentLength(data.available());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putString(String bucket, String key, String data,
String contentType) throws Exception {
throw new UnsupportedOperationException();
}
}

73
aws/s3/perftest/src/test/java/com/amazon/s3/S3UtilsTest.java
View File

@ -1,73 +0,0 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package com.amazon.s3;
import org.testng.annotations.Test;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
/**
* Compares performance of encryption operations.
*
* @author Adrian Cole
*
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3UtilsTest")
public class S3UtilsTest extends org.jclouds.aws.s3.S3UtilsTest {
@Test(dataProvider = "hmacsha1")
void testAmazonSampleDigestSerialResponseTime(byte[] key, String message, String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException {
for (int i = 0; i < 10000; i++)
testAmazonSampleDigest(key, message, base64Digest);
}
@Test(dataProvider = "hmacsha1")
public void testAmazonSampleDigest(byte[] key, String message, String base64Digest) {
String encoded = Utils.encode(new String(key), message, false);
assert encoded.equals(base64Digest);
}
@Test(dataProvider = "hmacsha1")
void testAmazonSampleDigestParallelResponseTime(final byte[] key, final String message, final String base64Digest) throws NoSuchProviderException, NoSuchAlgorithmException, InvalidKeyException, InterruptedException, ExecutionException {
CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
for (int i = 0; i < 10000; i++)
completer.submit(new Callable<Boolean>() {
public Boolean call() {
try {
testAmazonSampleDigest(key, message, base64Digest);
return true;
} catch (Exception e) {
return false;
}
}
});
for (int i = 0; i < 10000; i++) assert completer.take().get();
}
}

124
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/AmazonPerformanceLiveTest.java Normal file
View File

@ -0,0 +1,124 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.File;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import org.jclouds.aws.s3.reference.S3Constants;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import com.amazon.s3.AWSAuthConnection;
/**
* Runs operations that amazon s3 sample code is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.AmazonPerformanceLiveTest", groups = { "live" })
public class AmazonPerformanceLiveTest extends BasePerformance {
private AWSAuthConnection amzClient;
@Override
@BeforeTest
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpCredentials(@Optional String AWSAccessKeyId,
@Optional String AWSSecretAccessKey, ITestContext context) throws Exception {
super.setUpCredentials(AWSAccessKeyId, AWSSecretAccessKey, context);
amzClient = new AWSAuthConnection(AWSAccessKeyId, AWSSecretAccessKey, false);
}
@Override
@Test(enabled = false)
public void testPutFileSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
com.amazon.s3.S3Object object = new com.amazon.s3.S3Object(data, null);
Map<String, List<String>> headers = new TreeMap<String, List<String>>();
headers.put("Content-Type", Arrays.asList(new String[] { contentType }));
return amzClient.put(bucket, key, object, headers).connection.getResponseMessage() != null;
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
}

90
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/BaseJCloudsPerformance.java Normal file
View File

@ -0,0 +1,90 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.TimeUnit;
/**
* // TODO: Adrian: Document this!
*
* @author Adrian Cole
*/
public abstract class BaseJCloudsPerformance extends BasePerformance {
// boolean get
// (
// int id) throws Exception {
// S3Bucket s3Bucket = new S3Bucket();
// s3Bucket.setName(bucketPrefix + "-jclouds-puts");
// org.jclouds.aws.s3.domain.S3Object object = new
// org.jclouds.aws.s3.domain.S3Object();
// object.setKey(id + "");
// //object.setContentType("text/plain");
// object.setContentType("application/octetstream");
// //object.setData("this is a test");
// object.setData(test);
// return clientProvider.getObject(s3Bucket,
// object.getKey()).get(120,TimeUnit.SECONDS) !=
// org.jclouds.aws.s3.domain.S3Object.NOT_FOUND;
// }
@Override
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
object.getMetadata().setSize(data.available());
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
org.jclouds.aws.s3.domain.S3Object object = new org.jclouds.aws.s3.domain.S3Object(key);
object.getMetadata().setContentType(contentType);
object.setData(data);
return client.putObject(bucket, object).get(120, TimeUnit.SECONDS) != null;
}
}

226
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/BasePerformance.java Normal file
View File

@ -0,0 +1,226 @@
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
package org.jclouds.aws.s3;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import org.jclouds.aws.s3.S3IntegrationTest;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.inject.Provider;
/**
* Tests relative performance of S3 functions.
*
* @author Adrian Cole
*/
@Test(groups = { "live" })
public abstract class BasePerformance extends S3IntegrationTest {
@Override
protected boolean debugEnabled() {
return false;
}
protected static int LOOP_COUNT = 100;
protected ExecutorService exec;
protected CompletionService<Boolean> completer;
@BeforeTest
protected void setUpCallables() {
exec = Executors.newCachedThreadPool();
completer = new ExecutorCompletionService<Boolean>(exec);
}
@AfterTest
protected void tearDownExecutor() throws Exception {
exec.shutdownNow();
exec = null;
}
@Test(enabled = true)
public void testPutBytesSerial() throws Exception {
doSerial(new PutBytesCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutBytesParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutBytesCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutFileSerial() throws Exception {
doSerial(new PutFileCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutFileParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutFileCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutInputStreamSerial() throws Exception {
doSerial(new PutInputStreamCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutInputStreamCallable(this.bucketName), LOOP_COUNT);
}
@Test(enabled = true)
public void testPutStringSerial() throws Exception {
doSerial(new PutStringCallable(this.bucketName), LOOP_COUNT / 10);
}
@Test(enabled = true)
public void testPutStringParallel() throws InterruptedException, ExecutionException,
TimeoutException {
doParallel(new PutStringCallable(this.bucketName), LOOP_COUNT);
}
private void doSerial(Provider<Callable<Boolean>> provider, int loopCount) throws Exception,
ExecutionException {
for (int i = 0; i < loopCount; i++)
assert provider.get().call();
}
private void doParallel(Provider<Callable<Boolean>> provider, int loopCount)
throws InterruptedException, ExecutionException, TimeoutException {
for (int i = 0; i < loopCount; i++)
completer.submit(provider.get());
for (int i = 0; i < loopCount; i++)
assert completer.take().get(10, TimeUnit.SECONDS);
}
class PutBytesCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected byte[] test = new byte[1024 * 2];
private final String bucketName;
public PutBytesCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
String bucketName2 = bucketName;
return putByteArray(bucketName2, key.getAndIncrement() + "", test,
"application/octetstring");
}
};
}
}
class PutFileCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml");
private final String bucketName;
public PutFileCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putFile(bucketName, key.getAndIncrement() + "", file, "text/xml");
}
};
}
}
class PutInputStreamCallable extends PutBytesCallable {
final AtomicInteger key = new AtomicInteger(0);
private final String bucketName;
public PutInputStreamCallable(String bucketName) {
super(bucketName);
this.bucketName = bucketName;
}
@Override
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putInputStream(bucketName, key.getAndIncrement() + "",
new ByteArrayInputStream(test), "application/octetstring");
}
};
}
}
class PutStringCallable implements Provider<Callable<Boolean>> {
final AtomicInteger key = new AtomicInteger(0);
protected String testString = "hello world!";
private final String bucketName;
public PutStringCallable(String bucketName) {
this.bucketName = bucketName;
}
public Callable<Boolean> get() {
return new Callable<Boolean>() {
public Boolean call() throws Exception {
return putString(bucketName, key.getAndIncrement() + "", testString, "text/plain");
}
};
}
}
protected abstract boolean putByteArray(String bucket, String key, byte[] data,
String contentType) throws Exception;
protected abstract boolean putFile(String bucket, String key, File data, String contentType)
throws Exception;
protected abstract boolean putInputStream(String bucket, String key, InputStream data,
String contentType) throws Exception;
protected abstract boolean putString(String bucket, String key, String data, String contentType)
throws Exception;
}

6
aws/s3/perftest/src/test/java/com/amazon/s3/JCloudsNioPerformanceLiveTest.java → aws/s3/perftest/src/test/java/org/jclouds/aws/s3/JCloudsNioPerformanceLiveTest.java
View File

@ -21,14 +21,14 @@
* under the License.
* ====================================================================
*/
package com.amazon.s3;
package org.jclouds.aws.s3;
import java.util.Properties;
import com.google.inject.Module;
import org.jclouds.http.httpnio.config.HttpNioConnectionPoolClientModule;
import org.testng.annotations.Test;
import java.util.Properties;
import com.google.inject.Module;
@Test(sequential = true, testName = "s3.JCloudsNioPerformanceLiveTest", groups = {"live"})
public class JCloudsNioPerformanceLiveTest extends BaseJCloudsPerformance {

3
aws/s3/perftest/src/test/java/com/amazon/s3/JCloudsPerformanceLiveTest.java → aws/s3/perftest/src/test/java/org/jclouds/aws/s3/JCloudsPerformanceLiveTest.java
View File

@ -21,8 +21,7 @@
* under the License.
* ====================================================================
*/
package com.amazon.s3;
package org.jclouds.aws.s3;
import org.testng.annotations.Test;
/**

127
aws/s3/perftest/src/test/java/org/jclouds/aws/s3/Jets3tPerformanceLiveTest.java Normal file
View File

@ -0,0 +1,127 @@
package org.jclouds.aws.s3;
/**
*
* Copyright (C) 2009 Adrian Cole <adrian@jclouds.org>
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
import java.io.File;
import java.io.InputStream;
import java.util.concurrent.ExecutionException;
import org.jclouds.aws.s3.reference.S3Constants;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.security.AWSCredentials;
import org.testng.ITestContext;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
/**
* Runs operations that jets3t is capable of performing.
*
* @author Adrian Cole
*/
@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.Jets3tPerformanceLiveTest", groups = { "live" })
public class Jets3tPerformanceLiveTest extends BasePerformance {
private S3Service jetClient;
@BeforeTest
public void setUpJetS3t(ITestContext testContext) throws S3ServiceException {
String AWSAccessKeyId = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_ACCESSKEYID);
String AWSSecretAccessKey = (String) testContext
.getAttribute(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
jetClient = new RestS3Service(new AWSCredentials(AWSAccessKeyId, AWSSecretAccessKey));
}
@Override
@Test(enabled = false)
public void testPutStringSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesSerial() throws Exception {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutStringParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutBytesParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
public void testPutFileParallel() throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException();
}
@Override
@Test(enabled = false)
protected boolean putByteArray(String bucket, String key, byte[] data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected boolean putFile(String bucket, String key, File data, String contentType)
throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
object.setContentType(contentType);
object.setDataInputFile(data);
object.setContentLength(data.length());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
object.setContentType(contentType);
object.setDataInputStream(data);
object.setContentLength(data.available());
return jetClient.putObject(bucket, object) != null;
}
@Override
protected boolean putString(String bucket, String key, String data, String contentType)
throws Exception {
throw new UnsupportedOperationException();
}
}

27
core/src/test/java/org/jclouds/http/commands/callables/ReturnStringIf200Test.java
View File

@ -35,28 +35,15 @@ import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils;
import org.jclouds.http.HttpFutureCommand;
import org.jclouds.http.HttpResponse;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
@Test(groups = { "unit" })
public class ReturnStringIf200Test {
private HttpFutureCommand.ResponseCallable<String> callable = null;
@BeforeMethod
void setUp() {
callable = new ReturnStringIf200();
}
@AfterMethod
void tearDown() {
callable = null;
}
@Test
public void testExceptionWhenNoContentOn200() throws ExecutionException,
InterruptedException, TimeoutException, IOException {
public void testExceptionWhenNoContentOn200() throws ExecutionException, InterruptedException,
TimeoutException, IOException {
HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
expect(response.getContent()).andReturn(null);
@ -71,8 +58,9 @@ public class ReturnStringIf200Test {
}
@Test
public void testExceptionWhenIOExceptionOn200() throws ExecutionException,
InterruptedException, TimeoutException, IOException {
public void testExceptionWhenIOExceptionOn200() throws ExecutionException, InterruptedException,
TimeoutException, IOException {
HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
RuntimeException exception = new RuntimeException("bad");
@ -89,6 +77,7 @@ public class ReturnStringIf200Test {
@Test
public void testResponseOk() throws Exception {
HttpFutureCommand.ResponseCallable<String> callable = new ReturnStringIf200();
HttpResponse response = createMock(HttpResponse.class);
expect(response.getStatusCode()).andReturn(200).atLeastOnce();
expect(response.getContent()).andReturn(IOUtils.toInputStream("hello"));

76
core/src/test/java/org/jclouds/test/testng/UnitTestStatusListener.java Normal file
View File

@ -0,0 +1,76 @@
package org.jclouds.test.testng;
import java.util.concurrent.atomic.AtomicInteger;
import org.testng.IClass;
import org.testng.ITestContext;
import org.testng.ITestListener;
import org.testng.ITestResult;
/**
* adapted from the following class:
*
* @see org.UnitTestStatusListener.test.testng.UnitTestTestNGListener
* @author Adrian Cole
* @author dpospisi@redhat.com
* @author Mircea.Markus@jboss.com
*/
public class UnitTestStatusListener implements ITestListener {
/**
* Holds test classes actually running in all threads.
*/
private ThreadLocal<IClass> threadTestClass = new ThreadLocal<IClass>();
private ThreadLocal<Long> threadTestStart = new ThreadLocal<Long>();
private AtomicInteger failed = new AtomicInteger(0);
private AtomicInteger succeded = new AtomicInteger(0);
private AtomicInteger skipped = new AtomicInteger(0);
public void onTestStart(ITestResult res) {
System.out.println("Starting test " + getTestDesc(res));
threadTestClass.set(res.getTestClass());
threadTestStart.set(System.currentTimeMillis());
}
synchronized public void onTestSuccess(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " succeeded: "
+ (System.currentTimeMillis() - threadTestStart.get()) + "ms");
succeded.incrementAndGet();
printStatus();
}
synchronized public void onTestFailure(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " failed.");
failed.incrementAndGet();
printStatus();
}
synchronized public void onTestSkipped(ITestResult arg0) {
System.out.println(getThreadId() + " Test " + getTestDesc(arg0) + " skipped.");
skipped.incrementAndGet();
printStatus();
}
public void onTestFailedButWithinSuccessPercentage(ITestResult arg0) {
}
public void onStart(ITestContext arg0) {
}
public void onFinish(ITestContext arg0) {
}
private String getThreadId() {
return "[" + Thread.currentThread().getName() + "]";
}
private String getTestDesc(ITestResult res) {
return res.getMethod().getMethodName() + "(" + res.getTestClass().getName() + ")";
}
private void printStatus() {
System.out.println("Test suite progress: tests succeeded: " + succeded.get() + ", failed: "
+ failed.get() + ", skipped: " + skipped.get() + ".");
}
}

10
project/pom.xml
View File

@ -169,6 +169,8 @@
<goal>test</goal>
</goals>
<configuration>
<parallel>tests</parallel>
<threadCount>5</threadCount>
<!-- note that the groups/excluded groups don't work due to some problem
in surefire or testng. instead, we have to exclude via file path
<groups>integration</groups>
@ -183,6 +185,8 @@
</execution>
</executions>
<configuration>
<parallel>methods</parallel>
<threadCount>5</threadCount>
<!-- note that the groups/excluded groups don't work due to some problem
in surefire or testng. instead, we have to exclude via file path
<groups>unit,performance</groups>
@ -191,6 +195,12 @@
<exclude>**/*IntegrationTest.java</exclude>
<exclude>**/*LiveTest.java</exclude>
</excludes>
<properties>
<property>
<name>listener</name>
<value>org.jclouds.test.testng.UnitTestStatusListener</value>
</property>
</properties>
</configuration>
</plugin>
<!-- Make sure we generate src jars too -->