Issue 64: cleaned up integration test code

git-svn-id: http://jclouds.googlecode.com/svn/trunk@1482 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-06-28 17:20:23 +00:00
parent 3b72ff31df
commit 074fefc7db
4 changed files with 104 additions and 182 deletions

View File

@ -27,27 +27,22 @@ import static org.jclouds.aws.s3.commands.options.PutBucketOptions.Builder.creat
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.util.Date;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.ConsoleHandler;
import java.util.logging.Formatter;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import org.jclouds.aws.s3.config.StubS3ConnectionModule;
import org.jclouds.aws.s3.domain.S3Bucket;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.domain.S3Bucket.Metadata;
import org.jclouds.aws.s3.domain.S3Bucket.Metadata.LocationConstraint;
import org.jclouds.aws.s3.internal.StubS3Connection;
import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.aws.s3.util.S3Utils;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
@ -66,6 +61,25 @@ import com.google.inject.Module;
public class S3IntegrationTest {
protected static final String TEST_STRING = "<apples><apple name=\"fuji\"></apple> </apples>";
public static long INCONSISTENCY_WINDOW = 1000;
protected static final String sysAWSAccessKeyId = System
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
protected static final String sysAWSSecretAccessKey = System
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
protected static int bucketCount = 20;
protected static volatile int bucketIndex = 0;
protected byte[] goodMd5;
protected byte[] badMd5;
protected S3Connection client;
protected S3Context context = null;
protected boolean SANITY_CHECK_RETURNED_BUCKET_NAME = false;
private String bucketPrefix = System.getProperty("user.name") + ".s3int";
/**
* two test groups integration and live.
*/
private static final BlockingQueue<String> bucketNames = new ArrayBlockingQueue<String>(
bucketCount);
/**
* Due to eventual consistency, bucket commands may not return correctly immediately. Hence, we
@ -86,27 +100,10 @@ public class S3IntegrationTest {
throw error;
}
protected byte[] goodMd5;
protected byte[] badMd5;
protected void createBucketAndEnsureEmpty(String bucketName) throws InterruptedException,
ExecutionException, TimeoutException {
client.putBucketIfNotExists(bucketName).get(10, TimeUnit.SECONDS);
emptyBucket(bucketName);
assertEventuallyBucketEmpty(bucketName);
}
protected void assertEventuallyBucketEmpty(final String bucketName) throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
try {
assertEquals(client.listBucket(bucketName).get(10, TimeUnit.SECONDS).getContents()
.size(), 0, "bucket " + bucketName + "wasn't empty");
} catch (Exception e) {
Utils.<RuntimeException> rethrowIfRuntimeOrSameType(e);
}
}
});
}
protected void addObjectToBucket(String sourceBucket, String key) throws InterruptedException,
@ -145,38 +142,6 @@ public class S3IntegrationTest {
});
}
@BeforeGroups(groups = { "integration", "live" })
protected void enableDebug() {
if (debugEnabled()) {
Handler HANDLER = new ConsoleHandler() {
{
setLevel(Level.ALL);
setFormatter(new Formatter() {
@Override
public String format(LogRecord record) {
return String.format("[%tT %-7s] [%-7s] [%s]: %s %s\n", new Date(record
.getMillis()), record.getLevel(), Thread.currentThread().getName(),
record.getLoggerName(), record.getMessage(),
record.getThrown() == null ? "" : record.getThrown());
}
});
}
};
Logger guiceLogger = Logger.getLogger("org.jclouds");
guiceLogger.addHandler(HANDLER);
guiceLogger.setLevel(Level.ALL);
}
}
protected S3Connection client;
protected S3Context context = null;
protected static final String sysAWSAccessKeyId = System
.getProperty(S3Constants.PROPERTY_AWS_ACCESSKEYID);
protected static final String sysAWSSecretAccessKey = System
.getProperty(S3Constants.PROPERTY_AWS_SECRETACCESSKEY);
@BeforeGroups(groups = { "integration", "live" })
@Parameters( { S3Constants.PROPERTY_AWS_ACCESSKEYID, S3Constants.PROPERTY_AWS_SECRETACCESSKEY })
protected void setUpCredentials(@Optional String AWSAccessKeyId,
@ -202,9 +167,6 @@ public class S3IntegrationTest {
}
client = context.getConnection();
assert client != null;
SANITY_CHECK_RETURNED_BUCKET_NAME = (client instanceof StubS3Connection);
goodMd5 = S3Utils.md5(TEST_STRING);
badMd5 = S3Utils.md5("alf");
}
@ -212,6 +174,7 @@ public class S3IntegrationTest {
protected void createStubS3Context() {
context = S3ContextFactory.createContext("stub", "stub").withHttpAddress("stub").withModule(
new StubS3ConnectionModule()).build();
SANITY_CHECK_RETURNED_BUCKET_NAME = true;
}
protected void createLiveS3Context(String AWSAccessKeyId, String AWSSecretAccessKey) {
@ -221,13 +184,8 @@ public class S3IntegrationTest {
public String getBucketName() throws InterruptedException, ExecutionException, TimeoutException {
String bucketName = bucketNames.poll(30, TimeUnit.SECONDS);
// retrying as inside EC2 it may take longer to reflect the contents of a bucket.
try {
emptyBucket(bucketName);
} catch (AssertionError e) {
emptyBucket(bucketName);
}
assert bucketName != null : "unable to get a bucket for the test";
emptyBucket(bucketName);
return bucketName;
}
@ -244,7 +202,6 @@ public class S3IntegrationTest {
ExecutionException, TimeoutException {
if (bucketName != null) {
bucketNames.add(bucketName);
/*
* Ensure that any returned bucket name actually exists on the server. Return of a
* non-existent bucket introduces subtle testing bugs, where later unrelated tests will
@ -280,17 +237,6 @@ public class S3IntegrationTest {
}
}
protected static int bucketCount = 20;
protected static volatile int bucketIndex = 0;
protected boolean SANITY_CHECK_RETURNED_BUCKET_NAME = false;
/**
* two test groups integration and live.
*/
private static final BlockingQueue<String> bucketNames = new ArrayBlockingQueue<String>(
bucketCount);
/**
* There are a lot of retries here mainly from experience running inside amazon EC2.
*/
@ -314,10 +260,6 @@ public class S3IntegrationTest {
}
}
protected boolean debugEnabled() {
return false;
}
protected S3ContextFactory buildS3ContextFactory(String AWSAccessKeyId, String AWSSecretAccessKey) {
return S3ContextFactory.createContext(AWSAccessKeyId, AWSSecretAccessKey).withSaxDebug()
.withHttpSecure(false).withHttpPort(80);
@ -327,8 +269,6 @@ public class S3IntegrationTest {
return new JavaUrlHttpFutureCommandClientModule();
}
private String bucketPrefix = System.getProperty("user.name") + ".s3int";
protected void deleteEverything() throws Exception {
try {
List<S3Bucket.Metadata> metadata = client.listOwnedBuckets().get(10, TimeUnit.SECONDS);
@ -355,8 +295,13 @@ public class S3IntegrationTest {
assertEventually(new Runnable() {
public void run() {
try {
context.createInputStreamMap(name).clear();
assertEventuallyBucketEmpty(name);
Map<String, InputStream> map = context.createInputStreamMap(name);
Set<String> keys = map.keySet();
if (keys.size() > 0) {
map.clear();
assertEquals(map.size(), 0, String.format(
"deleting %s, we still have %s left", keys, map.keySet()));
}
} catch (Exception e) {
Utils.<RuntimeException> rethrowIfRuntimeOrSameType(e);
}

View File

@ -71,11 +71,6 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
AWSCredentials credentials;
S3Service service;
@Override
protected boolean debugEnabled() {
return true;
}
/**
* overridden only to get access to the amazon credentials used for jets3t initialization.
*/
@ -99,8 +94,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
@Test
public void testCreateBucketImpl() throws S3ServiceException, InterruptedException,
ExecutionException, TimeoutException
{
ExecutionException, TimeoutException {
String bucketName = getScratchBucketName();
try {
S3Bucket bucket = service.createBucket(new S3Bucket(bucketName));
@ -126,8 +120,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
@Test(dependsOnMethods = "testCreateBucketImpl")
public void testDeleteObjectImpl() throws InterruptedException, ExecutionException,
TimeoutException, S3ServiceException, IOException
{
TimeoutException, S3ServiceException, IOException {
String bucketName = getBucketName();
try {
String objectKey = "key-testDeleteObjectImpl";
@ -148,8 +141,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
@Test(dependsOnMethods = "testCreateBucketImpl")
public void testGetObjectDetailsImpl() throws InterruptedException, ExecutionException,
TimeoutException, S3ServiceException, IOException
{
TimeoutException, S3ServiceException, IOException {
String bucketName = getBucketName();
try {
String objectKey = "key-testGetObjectDetailsImpl";
@ -159,8 +151,8 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
org.jclouds.aws.s3.domain.S3Object s3Object = new org.jclouds.aws.s3.domain.S3Object(
objectKey, objectValue);
s3Object.getMetadata().getUserMetadata().put(S3Constants.USER_METADATA_PREFIX + metadataName,
metadataValue);
s3Object.getMetadata().getUserMetadata().put(
S3Constants.USER_METADATA_PREFIX + metadataName, metadataValue);
addObjectToBucket(bucketName, s3Object);
S3Object objectDetails = service.getObjectDetails(new S3Bucket(bucketName), objectKey);
@ -176,8 +168,7 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
@Test(dependsOnMethods = "testCreateBucketImpl")
public void testGetObjectImpl() throws InterruptedException, ExecutionException,
TimeoutException, S3ServiceException, IOException
{
TimeoutException, S3ServiceException, IOException {
String bucketName = getBucketName();
try {
String objectKey = "key-testGetObjectImpl";
@ -187,8 +178,8 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
org.jclouds.aws.s3.domain.S3Object s3Object = new org.jclouds.aws.s3.domain.S3Object(
objectKey, objectValue);
s3Object.getMetadata().getUserMetadata().put(S3Constants.USER_METADATA_PREFIX + metadataName,
metadataValue);
s3Object.getMetadata().getUserMetadata().put(
S3Constants.USER_METADATA_PREFIX + metadataName, metadataValue);
addObjectToBucket(bucketName, s3Object);
S3Object object = service.getObject(new S3Bucket(bucketName), objectKey);
@ -207,15 +198,14 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
@Test(dependsOnMethods = "testCreateBucketImpl")
public void testListAllBucketsImpl() throws InterruptedException, ExecutionException,
TimeoutException, S3ServiceException
{
TimeoutException, S3ServiceException {
String bucketName = getBucketName();
try {
// Ensure there is at least 1 bucket in S3 account to list and compare.
S3Bucket[] jsBuckets = service.listAllBuckets();
List<org.jclouds.aws.s3.domain.S3Bucket.Metadata> jcBuckets = client.listOwnedBuckets().get(
10, TimeUnit.SECONDS);
List<org.jclouds.aws.s3.domain.S3Bucket.Metadata> jcBuckets = client.listOwnedBuckets()
.get(10, TimeUnit.SECONDS);
assert jsBuckets.length == jcBuckets.size();
@ -402,8 +392,8 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
jcObject = client.getObject(bucketName, objectKey).get(10, TimeUnit.SECONDS);
assertEquals(Iterables.getLast(jcObject.getMetadata().getUserMetadata().get(
S3Constants.USER_METADATA_PREFIX + "my-metadata-1")), "value-1");
assertEquals(jsResultObject.getMetadata(S3Constants.USER_METADATA_PREFIX + "my-metadata-1"),
"value-1");
assertEquals(jsResultObject
.getMetadata(S3Constants.USER_METADATA_PREFIX + "my-metadata-1"), "value-1");
// Upload object with public-read ACL
requestObject = new S3Object(objectKey);
@ -458,8 +448,8 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
// Copy with metadata and ACL retained
destinationObject = new S3Object(destinationObjectKey);
copyResult = service.copyObject(bucketName, sourceObjectKey, bucketName, destinationObject,
false);
copyResult = service.copyObject(bucketName, sourceObjectKey, bucketName,
destinationObject, false);
jcDestinationObject = client.getObject(bucketName, destinationObject.getKey()).get(10,
TimeUnit.SECONDS);
assertEquals(jcDestinationObject.getKey(), destinationObjectKey);
@ -473,8 +463,8 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
destinationObject = new S3Object(destinationObjectKey);
destinationObject.addMetadata(S3Constants.USER_METADATA_PREFIX + metadataName,
destinationMetadataValue);
copyResult = service.copyObject(bucketName, sourceObjectKey, bucketName, destinationObject,
true);
copyResult = service.copyObject(bucketName, sourceObjectKey, bucketName,
destinationObject, true);
jcDestinationObject = client.getObject(bucketName, destinationObject.getKey()).get(10,
TimeUnit.SECONDS);
assertEquals(Iterators.getLast(jcDestinationObject.getMetadata().getUserMetadata().get(
@ -485,8 +475,8 @@ public class JCloudsS3ServiceIntegrationTest extends S3IntegrationTest {
// Copy with ACL modified
destinationObject = new S3Object(destinationObjectKey);
destinationObject.setAcl(AccessControlList.REST_CANNED_PUBLIC_READ);
copyResult = service.copyObject(bucketName, sourceObjectKey, bucketName, destinationObject,
false);
copyResult = service.copyObject(bucketName, sourceObjectKey, bucketName,
destinationObject, false);
jcDestinationObject = client.getObject(bucketName, destinationObject.getKey()).get(10,
TimeUnit.SECONDS);
// TODO: Test destination ACL is changed (ie public-read)

View File

@ -30,19 +30,13 @@ import org.jclouds.aws.s3.suncloud.xml.config.SunCloudS3ParserModule;
import org.testng.annotations.Test;
/**
* This performs the same test as {@link S3ConnectionLiveTest}, except using Sun Cloud
* Storage.
* This performs the same test as {@link S3ConnectionLiveTest}, except using Sun Cloud Storage.
*
* @author Adrian Cole
*/
@Test(enabled = false, groups = { "live" }, testName = "s3.suncloud.SunCloudS3ConnectionLiveTest")
public class SunCloudS3ConnectionLiveTest extends S3ConnectionLiveTest {
@Override
protected boolean debugEnabled() {
return true;
}
@Override
protected S3ContextFactory buildS3ContextFactory(String AWSAccessKeyId, String AWSSecretAccessKey) {
return S3ContextFactory.createContext(AWSAccessKeyId, AWSSecretAccessKey).withModules(

View File

@ -49,16 +49,9 @@ import com.google.inject.Provider;
*/
@Test(groups = { "live" })
public abstract class BasePerformance extends S3IntegrationTest {
@Override
protected boolean debugEnabled() {
return false;
}
protected int timeoutSeconds = 10;
protected int loopCount = 100;
protected ExecutorService exec;
protected CompletionService<Boolean> completer;
@BeforeTest
@ -74,13 +67,13 @@ public abstract class BasePerformance extends S3IntegrationTest {
exec = null;
}
@Test(enabled = true)
@Test
public void testPutBytesSerialEU() throws Exception {
String euBucketName = createScratchBucketInEU();
doSerial(new PutBytesCallable(euBucketName), loopCount / 10);
}
@Test(enabled = true)
@Test
public void testPutBytesParallelEU() throws InterruptedException, ExecutionException,
TimeoutException {
String euBucketName = createScratchBucketInEU();
@ -91,7 +84,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
}
}
@Test(enabled = true)
@Test
public void testPutBytesSerial() throws Exception {
String bucketName = getBucketName();
try {
@ -101,7 +94,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
}
}
@Test(enabled = true)
@Test
public void testPutBytesParallel() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getBucketName();
@ -112,7 +105,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
}
}
@Test(enabled = true)
@Test
public void testPutFileSerial() throws Exception {
String bucketName = getBucketName();
try {
@ -122,7 +115,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
}
}
@Test(enabled = true)
@Test
public void testPutFileParallel() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getBucketName();
@ -133,7 +126,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
}
}
@Test(enabled = true)
@Test
public void testPutInputStreamSerial() throws Exception {
String bucketName = getBucketName();
try {
@ -143,7 +136,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
}
}
@Test(enabled = true)
@Test
public void testPutInputStreamParallel() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getBucketName();
@ -154,7 +147,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
}
}
@Test(enabled = true)
@Test
public void testPutStringSerial() throws Exception {
String bucketName = getBucketName();
try {
@ -164,7 +157,7 @@ public abstract class BasePerformance extends S3IntegrationTest {
}
}
@Test(enabled = true)
@Test
public void testPutStringParallel() throws InterruptedException, ExecutionException,
TimeoutException {
String bucketName = getBucketName();