Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 414: fixed length problem on string payload by eagerly encoding to UTF-8

This commit is contained in:
Adrian Cole 2010-11-30 11:46:25 +00:00
parent 8ce596be50
commit 53ac4751f5
4 changed files with 85 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
View File

@ -97,7 +97,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
@SuppressWarnings("unchecked")
public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
.getResourceAsStream("/const.txt.gz")));
.getResourceAsStream("/const.txt.gz")));
InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution);
InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
@ -120,24 +120,24 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
Map<Integer, Future<?>> responses = Maps.newHashMap();
for (int i = 0; i < 10; i++) {
responses.put(i, Futures.compose(context.getAsyncBlobStore().getBlob(containerName, key),
new Function<Blob, Void>() {
responses.put(i,
Futures.compose(context.getAsyncBlobStore().getBlob(containerName, key), new Function<Blob, Void>() {
@Override
public Void apply(Blob from) {
try {
assertEquals(CryptoStreams.md5(from.getPayload()), oneHundredOneConstitutionsMD5);
checkContentDisposition(from, expectedContentDisposition);
} catch (IOException e) {
Throwables.propagate(e);
}
return null;
@Override
public Void apply(Blob from) {
try {
assertEquals(CryptoStreams.md5(from.getPayload()), oneHundredOneConstitutionsMD5);
checkContentDisposition(from, expectedContentDisposition);
} catch (IOException e) {
Throwables.propagate(e);
}
return null;
}
}, this.exec));
}, this.exec));
}
Map<Integer, Exception> exceptions = awaitCompletion(responses, exec, 30000l, Logger.CONSOLE,
"get constitution");
"get constitution");
assert exceptions.size() == 0 : exceptions;
} finally {
@ -156,7 +156,6 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
context.getBlobStore().putBlob(containerName, sourceObject);
}
@Test(groups = { "integration", "live" })
public void testGetIfModifiedSince() throws InterruptedException {
String containerName = getContainerName();
@ -365,15 +364,15 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
@DataProvider(name = "delete")
public Object[][] createData() {
return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, { "colon:" },
{ "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" },
{ "colon:" }, { "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
}
@Test(groups = { "integration", "live" }, dataProvider = "delete")
public void deleteObject(String key) throws InterruptedException {
String containerName = getContainerName();
try {
addBlobToContainer(containerName, key);
addBlobToContainer(containerName, key, key, MediaType.TEXT_PLAIN);
context.getBlobStore().removeBlob(containerName, key);
assertContainerEmptyDeleting(containerName, key);
} finally {
@ -383,17 +382,19 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
private void assertContainerEmptyDeleting(String containerName, String key) {
Iterable<? extends StorageMetadata> listing = Iterables.filter(context.getBlobStore().list(containerName),
new Predicate<StorageMetadata>() {
new Predicate<StorageMetadata>() {
@Override
public boolean apply(StorageMetadata input) {
return input.getType() == StorageType.BLOB;
}
@Override
public boolean apply(StorageMetadata input) {
return input.getType() == StorageType.BLOB;
}
});
assertEquals(Iterables.size(listing), 0, String.format(
"deleting %s, we still have %s blobs left in container %s, using encoding %s", key, Iterables
.size(listing), containerName, LOCAL_ENCODING));
});
assertEquals(
Iterables.size(listing),
0,
String.format("deleting %s, we still have %s blobs left in container %s, using encoding %s", key,
Iterables.size(listing), containerName, LOCAL_ENCODING));
}
@Test(groups = { "integration", "live" })
@ -413,13 +414,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
String realObject = Utils.toStringAndClose(new FileInputStream("pom.xml"));
return new Object[][] { { "file", "text/xml", new File("pom.xml"), realObject },
{ "string", "text/xml", realObject, realObject },
{ "bytes", "application/octet-stream", realObject.getBytes(), realObject } };
{ "string", "text/xml", realObject, realObject },
{ "bytes", "application/octet-stream", realObject.getBytes(), realObject } };
}
@Test(groups = { "integration", "live" }, dataProvider = "putTests")
public void testPutObject(String key, String type, Object content, Object realObject) throws InterruptedException,
IOException {
IOException {
Blob blob = context.getBlobStore().newBlob(key);
blob.setPayload(Payloads.newPayload(content));
blob.getMetadata().getContentMetadata().setContentType(type);
@ -486,32 +487,32 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
protected void checkContentType(Blob blob, String contentType) {
assert blob.getPayload().getContentMetadata().getContentType().startsWith(contentType) : blob.getPayload()
.getContentMetadata().getContentType();
.getContentMetadata().getContentType();
assert blob.getMetadata().getContentMetadata().getContentType().startsWith(contentType) : blob.getMetadata()
.getContentMetadata().getContentType();
.getContentMetadata().getContentType();
}
protected void checkContentDisposition(Blob blob, String contentDisposition) {
assert blob.getPayload().getContentMetadata().getContentDisposition().startsWith(contentDisposition) : blob
.getPayload().getContentMetadata().getContentDisposition();
.getPayload().getContentMetadata().getContentDisposition();
assert blob.getMetadata().getContentMetadata().getContentDisposition().startsWith(contentDisposition) : blob
.getMetadata().getContentMetadata().getContentDisposition();
.getMetadata().getContentMetadata().getContentDisposition();
}
protected void checkContentEncoding(Blob blob, String contentEncoding) {
assert blob.getPayload().getContentMetadata().getContentEncoding().startsWith(contentEncoding) : blob
.getPayload().getContentMetadata().getContentEncoding();
.getPayload().getContentMetadata().getContentEncoding();
assert blob.getMetadata().getContentMetadata().getContentEncoding().startsWith(contentEncoding) : blob
.getMetadata().getContentMetadata().getContentEncoding();
.getMetadata().getContentMetadata().getContentEncoding();
}
protected void checkContentLanguage(Blob blob, String contentLanguage) {
assert blob.getPayload().getContentMetadata().getContentLanguage().startsWith(contentLanguage) : blob
.getPayload().getContentMetadata().getContentLanguage();
.getPayload().getContentMetadata().getContentLanguage();
assert blob.getMetadata().getContentMetadata().getContentLanguage().startsWith(contentLanguage) : blob
.getMetadata().getContentMetadata().getContentLanguage();
.getMetadata().getContentMetadata().getContentLanguage();
}
@ -565,7 +566,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
protected void validateMetadata(BlobMetadata metadata) throws IOException {
assert metadata.getContentMetadata().getContentType().startsWith("text/plain") : metadata.getContentMetadata()
.getContentType();
.getContentType();
assertEquals(metadata.getContentMetadata().getContentLength(), new Long(TEST_STRING.length()));
assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff");
checkMD5(metadata);

58
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java
View File

@ -25,8 +25,8 @@ import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CancellationException;
@ -35,6 +35,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.ws.rs.core.MediaType;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.attr.ConsistencyModel;
import org.jclouds.blobstore.domain.Blob;
@ -60,13 +62,13 @@ public class BaseBlobStoreIntegrationTest {
protected static final String TEST_STRING = String.format(XML_STRING_FORMAT, "apple");
protected Map<String, String> fiveStrings = ImmutableMap.of("one", String.format(XML_STRING_FORMAT, "apple"), "two",
String.format(XML_STRING_FORMAT, "bear"), "three", String.format(XML_STRING_FORMAT, "candy"), "four",
String.format(XML_STRING_FORMAT, "dogma"), "five", String.format(XML_STRING_FORMAT, "emma"));
String.format(XML_STRING_FORMAT, "bear"), "three", String.format(XML_STRING_FORMAT, "candy"), "four",
String.format(XML_STRING_FORMAT, "dogma"), "five", String.format(XML_STRING_FORMAT, "emma"));
protected Map<String, String> fiveStringsUnderPath = ImmutableMap.of("path/1", String.format(XML_STRING_FORMAT,
"apple"), "path/2", String.format(XML_STRING_FORMAT, "bear"), "path/3", String.format(XML_STRING_FORMAT,
"candy"), "path/4", String.format(XML_STRING_FORMAT, "dogma"), "path/5", String.format(XML_STRING_FORMAT,
"emma"));
protected Map<String, String> fiveStringsUnderPath = ImmutableMap.of("path/1",
String.format(XML_STRING_FORMAT, "apple"), "path/2", String.format(XML_STRING_FORMAT, "bear"), "path/3",
String.format(XML_STRING_FORMAT, "candy"), "path/4", String.format(XML_STRING_FORMAT, "dogma"), "path/5",
String.format(XML_STRING_FORMAT, "emma"));
public static long INCONSISTENCY_WINDOW = 10000;
protected static volatile AtomicInteger containerIndex = new AtomicInteger(0);
@ -90,7 +92,7 @@ public class BaseBlobStoreIntegrationTest {
@SuppressWarnings("unchecked")
private BlobStoreContext getCloudResources(ITestContext testContext) throws ClassNotFoundException,
InstantiationException, IllegalAccessException, Exception {
InstantiationException, IllegalAccessException, Exception {
String initializerClass = checkNotNull(System.getProperty("test.initializer"), "test.initializer");
Class<BaseTestInitializer> clazz = (Class<BaseTestInitializer>) Class.forName(initializerClass);
BaseTestInitializer initializer = clazz.newInstance();
@ -123,7 +125,7 @@ public class BaseBlobStoreIntegrationTest {
private static volatile boolean initialized = false;
protected void createContainersSharedByAllThreads(BlobStoreContext context, ITestContext testContext)
throws Exception {
throws Exception {
while (!initialized) {
synchronized (BaseBlobStoreIntegrationTest.class) {
if (!initialized) {
@ -174,12 +176,12 @@ public class BaseBlobStoreIntegrationTest {
try {
for (int i = 0; i < 2; i++) {
Iterable<? extends StorageMetadata> testContainers = Iterables.filter(context.getBlobStore().list(),
new Predicate<StorageMetadata>() {
public boolean apply(StorageMetadata input) {
return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER)
&& input.getName().startsWith(CONTAINER_PREFIX.toLowerCase());
}
});
new Predicate<StorageMetadata>() {
public boolean apply(StorageMetadata input) {
return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER)
&& input.getName().startsWith(CONTAINER_PREFIX.toLowerCase());
}
});
for (StorageMetadata container : testContainers) {
deleteContainerOrWarnIfUnable(context, container.getName());
}
@ -200,7 +202,7 @@ public class BaseBlobStoreIntegrationTest {
* we will try up to the inconsistency window to see if the assertion completes.
*/
protected static void assertConsistencyAware(BlobStoreContext context, Runnable assertion)
throws InterruptedException {
throws InterruptedException {
if (context.getConsistencyModel() == ConsistencyModel.STRICT) {
assertion.run();
return;
@ -226,7 +228,7 @@ public class BaseBlobStoreIntegrationTest {
}
protected static void createContainerAndEnsureEmpty(BlobStoreContext context, final String containerName)
throws InterruptedException {
throws InterruptedException {
context.getBlobStore().createContainerInLocation(null, containerName);
if (context.getConsistencyModel() == ConsistencyModel.EVENTUAL)
Thread.sleep(1000);
@ -238,9 +240,13 @@ public class BaseBlobStoreIntegrationTest {
}
protected String addBlobToContainer(String sourceContainer, String key) {
return addBlobToContainer(sourceContainer, key, TEST_STRING, MediaType.TEXT_XML);
}
protected String addBlobToContainer(String sourceContainer, String key, String payload, String contentType) {
Blob sourceObject = context.getBlobStore().newBlob(key);
sourceObject.setPayload(TEST_STRING);
sourceObject.getMetadata().getContentMetadata().setContentType("text/xml");
sourceObject.setPayload(payload);
sourceObject.getMetadata().getContentMetadata().setContentType(contentType);
return addBlobToContainer(sourceContainer, sourceObject);
}
@ -270,19 +276,19 @@ public class BaseBlobStoreIntegrationTest {
}
protected void assertConsistencyAwareContainerSize(final String containerName, final int count)
throws InterruptedException {
throws InterruptedException {
assertConsistencyAware(new Runnable() {
public void run() {
try {
assert context.getBlobStore().countBlobs(containerName) == count : String.format(
"expected only %d values in %s: %s", count, containerName, Sets.newHashSet(Iterables.transform(
context.getBlobStore().list(containerName), new Function<StorageMetadata, String>() {
"expected only %d values in %s: %s", count, containerName, Sets.newHashSet(Iterables.transform(
context.getBlobStore().list(containerName), new Function<StorageMetadata, String>() {
public String apply(StorageMetadata from) {
return from.getName();
}
public String apply(StorageMetadata from) {
return from.getName();
}
})));
})));
} catch (Exception e) {
Throwables.propagateIfPossible(e);
}

15
core/src/main/java/org/jclouds/io/payloads/StringPayload.java
View File

@ -19,18 +19,27 @@
package org.jclouds.io.payloads;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import org.jclouds.util.Utils;
import com.google.common.base.Charsets;
/**
* This implementation converts the String to a byte array using UTF-8 encoding. If you wish to use
* a different encoding, please use {@link ByteArrayPayload}.
*
* @author Adrian Cole
*/
public class StringPayload extends BasePayload<String> {
private final byte[] bytes;
// it is possible to discover length by walking the string and updating current length based on
// character code. However, this is process intense, and assumes an encoding type of UTF-8
public StringPayload(String content) {
super(content);
getContentMetadata().setContentLength((long) content.length());
this.bytes = content.getBytes(Charsets.UTF_8);
getContentMetadata().setContentLength(new Long(bytes.length));
}
/**
@ -38,7 +47,7 @@ public class StringPayload extends BasePayload<String> {
*/
@Override
public InputStream getInput() {
return Utils.toInputStream(content);
return new ByteArrayInputStream(bytes);
}
}

2
core/src/main/java/org/jclouds/util/Utils.java
View File

@ -47,10 +47,10 @@ import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Set;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;