Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 64: convert base integration test and all map tests to use eventually consistent bucket operation assertions 

git-svn-id: http://jclouds.googlecode.com/svn/trunk@1439 3d8758e0-26b5-11de-8745-db77d3ebf521
This commit is contained in:
adrian.f.cole 2009-06-17 22:53:51 +00:00
parent b6f129eaa4
commit b858ec002e
4 changed files with 406 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
aws/s3/core/src/test/java/org/jclouds/aws/s3/BaseS3MapIntegrationTest.java
View File

@ -47,11 +47,11 @@ import com.google.common.collect.ImmutableSet;
@Test
public abstract class BaseS3MapIntegrationTest<T> extends S3IntegrationTest {
public abstract void testPutAll();
public abstract void testPutAll() throws InterruptedException;
public abstract void testEntrySet() throws IOException;
public abstract void testEntrySet() throws IOException, InterruptedException;
public abstract void testValues() throws IOException;
public abstract void testValues() throws IOException, InterruptedException;
protected BaseS3Map<T> map;
protected Map<String, String> fiveStrings = ImmutableMap.of("one", "apple", "two", "bear",
@ -94,54 +94,119 @@ public abstract class BaseS3MapIntegrationTest<T> extends S3IntegrationTest {
protected abstract BaseS3Map<T> createMap(S3Context context, String bucket);
@Test(groups = { "integration", "live" })
public void testClear() {
public void testClear() throws InterruptedException {
map.clear();
assertEquals(map.size(), 0);
assertEventuallyMapSize(0);
putString("one", "apple");
assertEquals(map.size(), 1);
assertEventuallyMapSize(1);
map.clear();
assertEquals(map.size(), 0);
assertEventuallyMapSize(0);
}
@Test(groups = { "integration", "live" })
public abstract void testRemove() throws IOException;
public abstract void testRemove() throws IOException, InterruptedException;
@Test(groups = { "integration", "live" })
public void testKeySet() {
assertEquals(map.keySet().size(), 0);
public void testKeySet() throws InterruptedException {
assertEventuallyKeySize(0);
putString("one", "two");
assertEquals(map.keySet(), ImmutableSet.of("one"));
assertEventuallyKeySize(1);
assertEventuallyKeySetEquals(ImmutableSet.of("one"));
}
protected void assertEventuallyKeySetEquals(final Object toEqual) throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
assertEquals(map.keySet(), toEqual);
}
});
}
protected void assertEventuallyKeySize(final int size) throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
assertEquals(map.keySet().size(), size);
}
});
}
@Test(groups = { "integration", "live" })
public void testContainsKey() {
assert !map.containsKey("one");
public void testContainsKey() throws InterruptedException {
assertEventuallyDoesntContainKey();
putString("one", "apple");
assertEventuallyContainsKey();
}
protected void assertEventuallyContainsKey() throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
assert map.containsKey("one");
}
});
}
protected void assertEventuallyDoesntContainKey() throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
assert !map.containsKey("one");
}
});
}
@Test(groups = { "integration", "live" })
public void testIsEmpty() {
assert map.isEmpty();
public void testIsEmpty() throws InterruptedException {
assertEventuallyEmpty();
putString("one", "apple");
assertEventuallyNotEmpty();
}
protected void assertEventuallyNotEmpty() throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
assert !map.isEmpty();
}
});
}
protected void assertEventuallyEmpty() throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
assert map.isEmpty();
}
});
}
abstract protected void putString(String key, String value);
protected void fourLeftRemovingOne() {
protected void fourLeftRemovingOne() throws InterruptedException {
map.remove("one");
assertEquals(map.size(), 4);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(ImmutableSet.of("two",
"three", "four", "five")));
assertEventuallyMapSize(4);
assertEventuallyKeySetEquals(new TreeSet<String>(ImmutableSet.of("two", "three", "four",
"five")));
}
protected void assertEventuallyMapSize(final int size) throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
assertEquals(map.size(), size);
}
});
}
@Test(groups = { "integration", "live" })
public abstract void testPut() throws IOException;
public abstract void testPut() throws IOException, InterruptedException;
@Test(groups = { "integration", "live" })
public void testGetBucket() {
public void testGetBucket() throws InterruptedException {
assertEventuallyBucketNameCorrect();
}
protected void assertEventuallyBucketNameCorrect() throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
assertEquals(map.getBucket().getName(), bucketName);
}
});
}
}

70
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3InputStreamMapIntegrationTest.java
View File

@ -23,19 +23,20 @@
*/
package org.jclouds.aws.s3;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.internal.BaseS3Map;
import org.jclouds.util.Utils;
import static org.testng.Assert.assertEquals;
import org.testng.annotations.Test;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.Map.Entry;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.internal.BaseS3Map;
import org.jclouds.util.Utils;
import org.testng.annotations.Test;
/**
* Tests to cover @{link LiveS3ObjectMap}
@ -67,7 +68,7 @@ public class S3InputStreamMapIntegrationTest extends BaseS3MapIntegrationTest<In
}
@Test(groups = { "integration", "live" })
public void testRemove() throws IOException {
public void testRemove() throws IOException, InterruptedException {
putString("one", "two");
InputStream old = map.remove("one");
assertEquals(Utils.toStringAndClose(old), "two");
@ -75,21 +76,20 @@ public class S3InputStreamMapIntegrationTest extends BaseS3MapIntegrationTest<In
assert old == null;
old = map.get("one");
assert old == null;
assertEquals(map.keySet().size(), 0);
assertEventuallyKeySize(0);
}
@Override
@Test(groups = { "integration", "live" })
public void testEntrySet() throws IOException {
public void testEntrySet() throws IOException, InterruptedException {
map.putAllStrings(this.fiveStrings);
Set<Entry<String, InputStream>> entries = map.entrySet();
assertEquals(entries.size(), 5);
for (Entry<String, InputStream> entry : entries) {
assertEquals(IOUtils.toString(entry.getValue()), fiveStrings
.get(entry.getKey()));
assertEquals(IOUtils.toString(entry.getValue()), fiveStrings.get(entry.getKey()));
entry.setValue(IOUtils.toInputStream(""));
}
assertEquals(map.size(), 5);
assertEventuallyMapSize(5);
for (InputStream value : map.values()) {
assertEquals(IOUtils.toString(value), "");
}
@ -121,65 +121,61 @@ public class S3InputStreamMapIntegrationTest extends BaseS3MapIntegrationTest<In
@Override
@Test(groups = { "integration", "live" })
public void testPutAll() {
public void testPutAll() throws InterruptedException {
map.putAll(this.fiveInputs);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveInputs.keySet()));
assertEventuallyMapSize(5);
assertEventuallyKeySetEquals(new TreeSet<String>(fiveInputs.keySet()));
fourLeftRemovingOne();
}
@Test(groups = { "integration", "live" })
public void testPutAllBytes() {
public void testPutAllBytes() throws InterruptedException {
map.putAllBytes(this.fiveBytes);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveBytes.keySet()));
assertEventuallyMapSize(5);
assertEventuallyKeySetEquals(new TreeSet<String>(fiveBytes.keySet()));
fourLeftRemovingOne();
}
@Test(groups = { "integration", "live" })
public void testPutAllFiles() {
public void testPutAllFiles() throws InterruptedException {
map.putAllFiles(this.fiveFiles);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveFiles.keySet()));
assertEventuallyMapSize(5);
assertEventuallyKeySetEquals(new TreeSet<String>(fiveFiles.keySet()));
fourLeftRemovingOne();
}
@Test(groups = { "integration", "live" })
public void testPutAllStrings() {
public void testPutAllStrings() throws InterruptedException {
map.putAllStrings(this.fiveStrings);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveStrings.keySet()));
assertEventuallyMapSize(5);
assertEventuallyKeySetEquals(new TreeSet<String>(fiveStrings.keySet()));
fourLeftRemovingOne();
}
@Test(groups = { "integration", "live" })
public void testPutString() throws IOException {
public void testPutString() throws IOException, InterruptedException {
InputStream old = map.putString("one", "apple");
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putString("one", "bear");
getOneReturnsBearAndOldValueIsApple(apple);
}
void getOneReturnsAppleAndOldValueIsNull(InputStream old)
throws IOException {
void getOneReturnsAppleAndOldValueIsNull(InputStream old) throws IOException,
InterruptedException {
assert old == null;
assertEquals(Utils.toStringAndClose(map.get("one")), "apple");
assertEquals(map.size(), 1);
assertEventuallyMapSize(1);
}
void getOneReturnsBearAndOldValueIsApple(InputStream oldValue)
throws IOException {
void getOneReturnsBearAndOldValueIsApple(InputStream oldValue) throws IOException,
InterruptedException {
assertEquals(Utils.toStringAndClose(map.get("one")), "bear");
assertEquals(Utils.toStringAndClose(oldValue), "apple");
assertEquals(map.size(), 1);
assertEventuallyMapSize(1);
}
@Test(groups = { "integration", "live" })
public void testPutFile() throws IOException {
public void testPutFile() throws IOException, InterruptedException {
InputStream old = map.putFile("one", fiveFiles.get("one"));
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putFile("one", fiveFiles.get("two"));
@ -187,7 +183,7 @@ public class S3InputStreamMapIntegrationTest extends BaseS3MapIntegrationTest<In
}
@Test(groups = { "integration", "live" })
public void testPutBytes() throws IOException {
public void testPutBytes() throws IOException, InterruptedException {
InputStream old = map.putBytes("one", "apple".getBytes());
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.putBytes("one", "bear".getBytes());
@ -195,7 +191,7 @@ public class S3InputStreamMapIntegrationTest extends BaseS3MapIntegrationTest<In
}
@Test(groups = { "integration", "live" })
public void testPut() throws IOException {
public void testPut() throws IOException, InterruptedException {
InputStream old = map.put("one", IOUtils.toInputStream("apple"));
getOneReturnsAppleAndOldValueIsNull(old);
InputStream apple = map.put("one", IOUtils.toInputStream("bear"));

56
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3IntegrationTest.java
View File

@ -49,6 +49,7 @@ import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.reference.S3Constants;
import org.jclouds.aws.s3.util.S3Utils;
import org.jclouds.http.config.JavaUrlHttpFutureCommandClientModule;
import org.jclouds.util.Utils;
import org.testng.ITestContext;
import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
@ -61,6 +62,28 @@ import com.google.inject.Module;
public class S3IntegrationTest {
protected static final String TEST_STRING = "<apples><apple name=\"fuji\"></apple> </apples>";
public static long INCONSISTENCY_WINDOW = 1000;
/**
*
* Due to eventual consistency, the size of the bucket and hence the size of the map may not
* return correctly immediately. Hence, we will try up to the inconsistency window to see if the
* assertion completes.
*/
protected void assertEventually(Runnable assertion) throws InterruptedException {
AssertionError error = null;
for (int i = 0; i < 5; i++) {
try {
assertion.run();
return;
} catch (AssertionError e) {
error = e;
}
Thread.sleep(INCONSISTENCY_WINDOW / 5);
}
if (error != null)
throw error;
}
protected byte[] goodMd5;
protected byte[] badMd5;
@ -70,8 +93,20 @@ public class S3IntegrationTest {
ExecutionException, TimeoutException {
deleteBucket(sourceBucket);
client.putBucketIfNotExists(sourceBucket).get(10, TimeUnit.SECONDS);
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS).getContents().size(),
0, "bucket " + sourceBucket + "wasn't empty");
assertEventuallyBucketEmpty(sourceBucket);
}
protected void assertEventuallyBucketEmpty(final String bucketName) throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
try {
assertEquals(client.listBucket(bucketName).get(10, TimeUnit.SECONDS).getContents()
.size(), 0, "bucket " + bucketName + "wasn't empty");
} catch (Exception e) {
Utils.<RuntimeException> rethrowIfRuntimeOrSameType(e);
}
}
});
}
protected void addObjectToBucket(String sourceBucket, String key) throws InterruptedException,
@ -89,14 +124,27 @@ public class S3IntegrationTest {
protected S3Object validateContent(String sourceBucket, String key) throws InterruptedException,
ExecutionException, TimeoutException, IOException {
assertEquals(client.listBucket(sourceBucket).get(10, TimeUnit.SECONDS).getContents().size(),
1);
assertEventuallyBucketSize(sourceBucket, 1);
S3Object newObject = client.getObject(sourceBucket, key).get(10, TimeUnit.SECONDS);
assert newObject != S3Object.NOT_FOUND;
assertEquals(S3Utils.getContentAsStringAndClose(newObject), TEST_STRING);
return newObject;
}
protected void assertEventuallyBucketSize(final String bucketName, final int count)
throws InterruptedException {
assertEventually(new Runnable() {
public void run() {
try {
assertEquals(client.listBucket(bucketName).get(10, TimeUnit.SECONDS).getContents()
.size(), count);
} catch (Exception e) {
Utils.<RuntimeException> rethrowIfRuntimeOrSameType(e);
}
}
});
}
@BeforeClass(groups = { "integration", "live" })
protected void enableDebug() {
if (debugEnabled()) {

54
aws/s3/core/src/test/java/org/jclouds/aws/s3/S3ObjectMapIntegrationTest.java
View File

@ -23,17 +23,23 @@
*/
package org.jclouds.aws.s3;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.Map.Entry;
import org.apache.commons.io.IOUtils;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.aws.s3.internal.BaseS3Map;
import org.jclouds.aws.s3.util.S3Utils;
import static org.testng.Assert.assertEquals;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;
/**
* Tests to cover @{link LiveS3ObjectMap}
*
@ -51,10 +57,10 @@ public class S3ObjectMapIntegrationTest extends BaseS3MapIntegrationTest<S3Objec
@Override
@Test(groups = { "integration", "live" })
public void testValues() throws IOException {
public void testValues() throws IOException, InterruptedException {
putFiveStrings();
Collection<S3Object> values = map.values();
assertEquals(values.size(), 5);
assertEventuallyMapSize(5);
Set<String> valuesAsString = new HashSet<String>();
for (S3Object object : values) {
valuesAsString.add(S3Utils.getContentAsStringAndClose(object));
@ -64,7 +70,7 @@ public class S3ObjectMapIntegrationTest extends BaseS3MapIntegrationTest<S3Objec
}
@Test(groups = { "integration", "live" })
public void testRemove() throws IOException {
public void testRemove() throws IOException, InterruptedException {
putString("one", "two");
S3Object old = map.remove("one");
assertEquals(S3Utils.getContentAsStringAndClose(old), "two");
@ -72,24 +78,24 @@ public class S3ObjectMapIntegrationTest extends BaseS3MapIntegrationTest<S3Objec
assert old == S3Object.NOT_FOUND;
old = map.get("one");
assert old == S3Object.NOT_FOUND;
assertEquals(map.keySet().size(), 0);
assertEventuallyKeySize(0);
}
@Override
@Test(groups = { "integration", "live" })
public void testEntrySet() throws IOException {
public void testEntrySet() throws IOException, InterruptedException {
putFiveStrings();
Set<Entry<String, S3Object>> entries = map.entrySet();
assertEquals(entries.size(), 5);
for (Entry<String, S3Object> entry : entries) {
assertEquals(S3Utils.getContentAsStringAndClose(entry.getValue()),
fiveStrings.get(entry.getKey()));
assertEquals(S3Utils.getContentAsStringAndClose(entry.getValue()), fiveStrings.get(entry
.getKey()));
S3Object value = entry.getValue();
value.setData("");
value.generateMd5();
entry.setValue(value);
}
assertEquals(map.size(), 5);
assertEventuallyMapSize(5);
for (S3Object value : map.values()) {
assertEquals(S3Utils.getContentAsStringAndClose(value), "");
}
@ -103,22 +109,21 @@ public class S3ObjectMapIntegrationTest extends BaseS3MapIntegrationTest<S3Objec
assert map.containsValue(object);
}
void getOneReturnsAppleAndOldValueIsNull(S3Object old) throws IOException {
void getOneReturnsAppleAndOldValueIsNull(S3Object old) throws IOException, InterruptedException {
assert old == S3Object.NOT_FOUND;
assertEquals(S3Utils.getContentAsStringAndClose(map.get("one")),
"apple");
assert map.size() == 1;
assertEquals(S3Utils.getContentAsStringAndClose(map.get("one")), "apple");
assertEventuallyMapSize(1);
}
void getOneReturnsBearAndOldValueIsApple(S3Object oldValue)
throws IOException {
void getOneReturnsBearAndOldValueIsApple(S3Object oldValue) throws IOException,
InterruptedException {
assertEquals(S3Utils.getContentAsStringAndClose(map.get("one")), "bear");
assertEquals(S3Utils.getContentAsStringAndClose(oldValue), "apple");
assert map.size() == 1;
assertEventuallyMapSize(1);
}
@Test(groups = { "integration", "live" })
public void testPut() throws IOException {
public void testPut() throws IOException, InterruptedException {
S3Object object = new S3Object("one");
object.setData(IOUtils.toInputStream("apple"));
object.generateMd5();
@ -131,7 +136,7 @@ public class S3ObjectMapIntegrationTest extends BaseS3MapIntegrationTest<S3Objec
}
@Test(groups = { "integration", "live" })
public void testPutAll() {
public void testPutAll() throws InterruptedException {
Map<String, S3Object> newMap = new HashMap<String, S3Object>();
for (String key : fiveInputs.keySet()) {
S3Object object = new S3Object(key);
@ -140,9 +145,8 @@ public class S3ObjectMapIntegrationTest extends BaseS3MapIntegrationTest<S3Objec
newMap.put(key, object);
}
map.putAll(newMap);
assertEquals(map.size(), 5);
assertEquals(new TreeSet<String>(map.keySet()), new TreeSet<String>(
fiveInputs.keySet()));
assertEventuallyMapSize(5);
assertEventuallyKeySetEquals(new TreeSet<String>(fiveInputs.keySet()));
fourLeftRemovingOne();
}