Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 414: fixed length problem on string payload by eagerly encoding to UTF-8

This commit is contained in:
Adrian Cole 2010-11-30 11:46:25 +00:00
parent 8ce596be50
commit 53ac4751f5
4 changed files with 85 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
View File

@ -97,7 +97,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public static InputSupplier<InputStream> getTestDataSupplier() throws IOException { public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
.getResourceAsStream("/const.txt.gz"))); .getResourceAsStream("/const.txt.gz")));
InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution); InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution);
InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier); InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
@ -120,24 +120,24 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
Map<Integer, Future<?>> responses = Maps.newHashMap(); Map<Integer, Future<?>> responses = Maps.newHashMap();
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
responses.put(i, Futures.compose(context.getAsyncBlobStore().getBlob(containerName, key), responses.put(i,
new Function<Blob, Void>() { Futures.compose(context.getAsyncBlobStore().getBlob(containerName, key), new Function<Blob, Void>() {
@Override @Override
public Void apply(Blob from) { public Void apply(Blob from) {
try { try {
assertEquals(CryptoStreams.md5(from.getPayload()), oneHundredOneConstitutionsMD5); assertEquals(CryptoStreams.md5(from.getPayload()), oneHundredOneConstitutionsMD5);
checkContentDisposition(from, expectedContentDisposition); checkContentDisposition(from, expectedContentDisposition);
} catch (IOException e) { } catch (IOException e) {
Throwables.propagate(e); Throwables.propagate(e);
}
return null;
} }
return null;
}
}, this.exec)); }, this.exec));
} }
Map<Integer, Exception> exceptions = awaitCompletion(responses, exec, 30000l, Logger.CONSOLE, Map<Integer, Exception> exceptions = awaitCompletion(responses, exec, 30000l, Logger.CONSOLE,
"get constitution"); "get constitution");
assert exceptions.size() == 0 : exceptions; assert exceptions.size() == 0 : exceptions;
} finally { } finally {
@ -156,7 +156,6 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
context.getBlobStore().putBlob(containerName, sourceObject); context.getBlobStore().putBlob(containerName, sourceObject);
} }
@Test(groups = { "integration", "live" }) @Test(groups = { "integration", "live" })
public void testGetIfModifiedSince() throws InterruptedException { public void testGetIfModifiedSince() throws InterruptedException {
String containerName = getContainerName(); String containerName = getContainerName();
@ -365,15 +364,15 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
@DataProvider(name = "delete") @DataProvider(name = "delete")
public Object[][] createData() { public Object[][] createData() {
return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, { "colon:" }, return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" },
{ "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } }; { "colon:" }, { "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
} }
@Test(groups = { "integration", "live" }, dataProvider = "delete") @Test(groups = { "integration", "live" }, dataProvider = "delete")
public void deleteObject(String key) throws InterruptedException { public void deleteObject(String key) throws InterruptedException {
String containerName = getContainerName(); String containerName = getContainerName();
try { try {
addBlobToContainer(containerName, key); addBlobToContainer(containerName, key, key, MediaType.TEXT_PLAIN);
context.getBlobStore().removeBlob(containerName, key); context.getBlobStore().removeBlob(containerName, key);
assertContainerEmptyDeleting(containerName, key); assertContainerEmptyDeleting(containerName, key);
} finally { } finally {
@ -383,17 +382,19 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
private void assertContainerEmptyDeleting(String containerName, String key) { private void assertContainerEmptyDeleting(String containerName, String key) {
Iterable<? extends StorageMetadata> listing = Iterables.filter(context.getBlobStore().list(containerName), Iterable<? extends StorageMetadata> listing = Iterables.filter(context.getBlobStore().list(containerName),
new Predicate<StorageMetadata>() { new Predicate<StorageMetadata>() {
@Override @Override
public boolean apply(StorageMetadata input) { public boolean apply(StorageMetadata input) {
return input.getType() == StorageType.BLOB; return input.getType() == StorageType.BLOB;
} }
}); });
assertEquals(Iterables.size(listing), 0, String.format( assertEquals(
"deleting %s, we still have %s blobs left in container %s, using encoding %s", key, Iterables Iterables.size(listing),
.size(listing), containerName, LOCAL_ENCODING)); 0,
String.format("deleting %s, we still have %s blobs left in container %s, using encoding %s", key,
Iterables.size(listing), containerName, LOCAL_ENCODING));
} }
@Test(groups = { "integration", "live" }) @Test(groups = { "integration", "live" })
@ -413,13 +414,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
String realObject = Utils.toStringAndClose(new FileInputStream("pom.xml")); String realObject = Utils.toStringAndClose(new FileInputStream("pom.xml"));
return new Object[][] { { "file", "text/xml", new File("pom.xml"), realObject }, return new Object[][] { { "file", "text/xml", new File("pom.xml"), realObject },
{ "string", "text/xml", realObject, realObject }, { "string", "text/xml", realObject, realObject },
{ "bytes", "application/octet-stream", realObject.getBytes(), realObject } }; { "bytes", "application/octet-stream", realObject.getBytes(), realObject } };
} }
@Test(groups = { "integration", "live" }, dataProvider = "putTests") @Test(groups = { "integration", "live" }, dataProvider = "putTests")
public void testPutObject(String key, String type, Object content, Object realObject) throws InterruptedException, public void testPutObject(String key, String type, Object content, Object realObject) throws InterruptedException,
IOException { IOException {
Blob blob = context.getBlobStore().newBlob(key); Blob blob = context.getBlobStore().newBlob(key);
blob.setPayload(Payloads.newPayload(content)); blob.setPayload(Payloads.newPayload(content));
blob.getMetadata().getContentMetadata().setContentType(type); blob.getMetadata().getContentMetadata().setContentType(type);
@ -486,32 +487,32 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
protected void checkContentType(Blob blob, String contentType) { protected void checkContentType(Blob blob, String contentType) {
assert blob.getPayload().getContentMetadata().getContentType().startsWith(contentType) : blob.getPayload() assert blob.getPayload().getContentMetadata().getContentType().startsWith(contentType) : blob.getPayload()
.getContentMetadata().getContentType(); .getContentMetadata().getContentType();
assert blob.getMetadata().getContentMetadata().getContentType().startsWith(contentType) : blob.getMetadata() assert blob.getMetadata().getContentMetadata().getContentType().startsWith(contentType) : blob.getMetadata()
.getContentMetadata().getContentType(); .getContentMetadata().getContentType();
} }
protected void checkContentDisposition(Blob blob, String contentDisposition) { protected void checkContentDisposition(Blob blob, String contentDisposition) {
assert blob.getPayload().getContentMetadata().getContentDisposition().startsWith(contentDisposition) : blob assert blob.getPayload().getContentMetadata().getContentDisposition().startsWith(contentDisposition) : blob
.getPayload().getContentMetadata().getContentDisposition(); .getPayload().getContentMetadata().getContentDisposition();
assert blob.getMetadata().getContentMetadata().getContentDisposition().startsWith(contentDisposition) : blob assert blob.getMetadata().getContentMetadata().getContentDisposition().startsWith(contentDisposition) : blob
.getMetadata().getContentMetadata().getContentDisposition(); .getMetadata().getContentMetadata().getContentDisposition();
} }
protected void checkContentEncoding(Blob blob, String contentEncoding) { protected void checkContentEncoding(Blob blob, String contentEncoding) {
assert blob.getPayload().getContentMetadata().getContentEncoding().startsWith(contentEncoding) : blob assert blob.getPayload().getContentMetadata().getContentEncoding().startsWith(contentEncoding) : blob
.getPayload().getContentMetadata().getContentEncoding(); .getPayload().getContentMetadata().getContentEncoding();
assert blob.getMetadata().getContentMetadata().getContentEncoding().startsWith(contentEncoding) : blob assert blob.getMetadata().getContentMetadata().getContentEncoding().startsWith(contentEncoding) : blob
.getMetadata().getContentMetadata().getContentEncoding(); .getMetadata().getContentMetadata().getContentEncoding();
} }
protected void checkContentLanguage(Blob blob, String contentLanguage) { protected void checkContentLanguage(Blob blob, String contentLanguage) {
assert blob.getPayload().getContentMetadata().getContentLanguage().startsWith(contentLanguage) : blob assert blob.getPayload().getContentMetadata().getContentLanguage().startsWith(contentLanguage) : blob
.getPayload().getContentMetadata().getContentLanguage(); .getPayload().getContentMetadata().getContentLanguage();
assert blob.getMetadata().getContentMetadata().getContentLanguage().startsWith(contentLanguage) : blob assert blob.getMetadata().getContentMetadata().getContentLanguage().startsWith(contentLanguage) : blob
.getMetadata().getContentMetadata().getContentLanguage(); .getMetadata().getContentMetadata().getContentLanguage();
} }
@ -565,7 +566,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
protected void validateMetadata(BlobMetadata metadata) throws IOException { protected void validateMetadata(BlobMetadata metadata) throws IOException {
assert metadata.getContentMetadata().getContentType().startsWith("text/plain") : metadata.getContentMetadata() assert metadata.getContentMetadata().getContentType().startsWith("text/plain") : metadata.getContentMetadata()
.getContentType(); .getContentType();
assertEquals(metadata.getContentMetadata().getContentLength(), new Long(TEST_STRING.length())); assertEquals(metadata.getContentMetadata().getContentLength(), new Long(TEST_STRING.length()));
assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff"); assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff");
checkMD5(metadata); checkMD5(metadata);

58
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobStoreIntegrationTest.java
View File

@ -25,8 +25,8 @@ import static org.testng.Assert.assertEquals;
import java.io.IOException; import java.io.IOException;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CancellationException; import java.util.concurrent.CancellationException;
@ -35,6 +35,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import javax.ws.rs.core.MediaType;
import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.attr.ConsistencyModel; import org.jclouds.blobstore.attr.ConsistencyModel;
import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.Blob;
@ -60,13 +62,13 @@ public class BaseBlobStoreIntegrationTest {
protected static final String TEST_STRING = String.format(XML_STRING_FORMAT, "apple"); protected static final String TEST_STRING = String.format(XML_STRING_FORMAT, "apple");
protected Map<String, String> fiveStrings = ImmutableMap.of("one", String.format(XML_STRING_FORMAT, "apple"), "two", protected Map<String, String> fiveStrings = ImmutableMap.of("one", String.format(XML_STRING_FORMAT, "apple"), "two",
String.format(XML_STRING_FORMAT, "bear"), "three", String.format(XML_STRING_FORMAT, "candy"), "four", String.format(XML_STRING_FORMAT, "bear"), "three", String.format(XML_STRING_FORMAT, "candy"), "four",
String.format(XML_STRING_FORMAT, "dogma"), "five", String.format(XML_STRING_FORMAT, "emma")); String.format(XML_STRING_FORMAT, "dogma"), "five", String.format(XML_STRING_FORMAT, "emma"));
protected Map<String, String> fiveStringsUnderPath = ImmutableMap.of("path/1", String.format(XML_STRING_FORMAT, protected Map<String, String> fiveStringsUnderPath = ImmutableMap.of("path/1",
"apple"), "path/2", String.format(XML_STRING_FORMAT, "bear"), "path/3", String.format(XML_STRING_FORMAT, String.format(XML_STRING_FORMAT, "apple"), "path/2", String.format(XML_STRING_FORMAT, "bear"), "path/3",
"candy"), "path/4", String.format(XML_STRING_FORMAT, "dogma"), "path/5", String.format(XML_STRING_FORMAT, String.format(XML_STRING_FORMAT, "candy"), "path/4", String.format(XML_STRING_FORMAT, "dogma"), "path/5",
"emma")); String.format(XML_STRING_FORMAT, "emma"));
public static long INCONSISTENCY_WINDOW = 10000; public static long INCONSISTENCY_WINDOW = 10000;
protected static volatile AtomicInteger containerIndex = new AtomicInteger(0); protected static volatile AtomicInteger containerIndex = new AtomicInteger(0);
@ -90,7 +92,7 @@ public class BaseBlobStoreIntegrationTest {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private BlobStoreContext getCloudResources(ITestContext testContext) throws ClassNotFoundException, private BlobStoreContext getCloudResources(ITestContext testContext) throws ClassNotFoundException,
InstantiationException, IllegalAccessException, Exception { InstantiationException, IllegalAccessException, Exception {
String initializerClass = checkNotNull(System.getProperty("test.initializer"), "test.initializer"); String initializerClass = checkNotNull(System.getProperty("test.initializer"), "test.initializer");
Class<BaseTestInitializer> clazz = (Class<BaseTestInitializer>) Class.forName(initializerClass); Class<BaseTestInitializer> clazz = (Class<BaseTestInitializer>) Class.forName(initializerClass);
BaseTestInitializer initializer = clazz.newInstance(); BaseTestInitializer initializer = clazz.newInstance();
@ -123,7 +125,7 @@ public class BaseBlobStoreIntegrationTest {
private static volatile boolean initialized = false; private static volatile boolean initialized = false;
protected void createContainersSharedByAllThreads(BlobStoreContext context, ITestContext testContext) protected void createContainersSharedByAllThreads(BlobStoreContext context, ITestContext testContext)
throws Exception { throws Exception {
while (!initialized) { while (!initialized) {
synchronized (BaseBlobStoreIntegrationTest.class) { synchronized (BaseBlobStoreIntegrationTest.class) {
if (!initialized) { if (!initialized) {
@ -174,12 +176,12 @@ public class BaseBlobStoreIntegrationTest {
try { try {
for (int i = 0; i < 2; i++) { for (int i = 0; i < 2; i++) {
Iterable<? extends StorageMetadata> testContainers = Iterables.filter(context.getBlobStore().list(), Iterable<? extends StorageMetadata> testContainers = Iterables.filter(context.getBlobStore().list(),
new Predicate<StorageMetadata>() { new Predicate<StorageMetadata>() {
public boolean apply(StorageMetadata input) { public boolean apply(StorageMetadata input) {
return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER) return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER)
&& input.getName().startsWith(CONTAINER_PREFIX.toLowerCase()); && input.getName().startsWith(CONTAINER_PREFIX.toLowerCase());
} }
}); });
for (StorageMetadata container : testContainers) { for (StorageMetadata container : testContainers) {
deleteContainerOrWarnIfUnable(context, container.getName()); deleteContainerOrWarnIfUnable(context, container.getName());
} }
@ -200,7 +202,7 @@ public class BaseBlobStoreIntegrationTest {
* we will try up to the inconsistency window to see if the assertion completes. * we will try up to the inconsistency window to see if the assertion completes.
*/ */
protected static void assertConsistencyAware(BlobStoreContext context, Runnable assertion) protected static void assertConsistencyAware(BlobStoreContext context, Runnable assertion)
throws InterruptedException { throws InterruptedException {
if (context.getConsistencyModel() == ConsistencyModel.STRICT) { if (context.getConsistencyModel() == ConsistencyModel.STRICT) {
assertion.run(); assertion.run();
return; return;
@ -226,7 +228,7 @@ public class BaseBlobStoreIntegrationTest {
} }
protected static void createContainerAndEnsureEmpty(BlobStoreContext context, final String containerName) protected static void createContainerAndEnsureEmpty(BlobStoreContext context, final String containerName)
throws InterruptedException { throws InterruptedException {
context.getBlobStore().createContainerInLocation(null, containerName); context.getBlobStore().createContainerInLocation(null, containerName);
if (context.getConsistencyModel() == ConsistencyModel.EVENTUAL) if (context.getConsistencyModel() == ConsistencyModel.EVENTUAL)
Thread.sleep(1000); Thread.sleep(1000);
@ -238,9 +240,13 @@ public class BaseBlobStoreIntegrationTest {
} }
protected String addBlobToContainer(String sourceContainer, String key) { protected String addBlobToContainer(String sourceContainer, String key) {
return addBlobToContainer(sourceContainer, key, TEST_STRING, MediaType.TEXT_XML);
}
protected String addBlobToContainer(String sourceContainer, String key, String payload, String contentType) {
Blob sourceObject = context.getBlobStore().newBlob(key); Blob sourceObject = context.getBlobStore().newBlob(key);
sourceObject.setPayload(TEST_STRING); sourceObject.setPayload(payload);
sourceObject.getMetadata().getContentMetadata().setContentType("text/xml"); sourceObject.getMetadata().getContentMetadata().setContentType(contentType);
return addBlobToContainer(sourceContainer, sourceObject); return addBlobToContainer(sourceContainer, sourceObject);
} }
@ -270,19 +276,19 @@ public class BaseBlobStoreIntegrationTest {
} }
protected void assertConsistencyAwareContainerSize(final String containerName, final int count) protected void assertConsistencyAwareContainerSize(final String containerName, final int count)
throws InterruptedException { throws InterruptedException {
assertConsistencyAware(new Runnable() { assertConsistencyAware(new Runnable() {
public void run() { public void run() {
try { try {
assert context.getBlobStore().countBlobs(containerName) == count : String.format( assert context.getBlobStore().countBlobs(containerName) == count : String.format(
"expected only %d values in %s: %s", count, containerName, Sets.newHashSet(Iterables.transform( "expected only %d values in %s: %s", count, containerName, Sets.newHashSet(Iterables.transform(
context.getBlobStore().list(containerName), new Function<StorageMetadata, String>() { context.getBlobStore().list(containerName), new Function<StorageMetadata, String>() {
public String apply(StorageMetadata from) { public String apply(StorageMetadata from) {
return from.getName(); return from.getName();
} }
}))); })));
} catch (Exception e) { } catch (Exception e) {
Throwables.propagateIfPossible(e); Throwables.propagateIfPossible(e);
} }

15
core/src/main/java/org/jclouds/io/payloads/StringPayload.java
View File

@ -19,18 +19,27 @@
package org.jclouds.io.payloads; package org.jclouds.io.payloads;
import java.io.ByteArrayInputStream;
import java.io.InputStream; import java.io.InputStream;
import org.jclouds.util.Utils; import com.google.common.base.Charsets;
/** /**
* This implementation converts the String to a byte array using UTF-8 encoding. If you wish to use
* a different encoding, please use {@link ByteArrayPayload}.
*
* @author Adrian Cole * @author Adrian Cole
*/ */
public class StringPayload extends BasePayload<String> { public class StringPayload extends BasePayload<String> {
private final byte[] bytes;
// it is possible to discover length by walking the string and updating current length based on
// character code. However, this is process intense, and assumes an encoding type of UTF-8
public StringPayload(String content) { public StringPayload(String content) {
super(content); super(content);
getContentMetadata().setContentLength((long) content.length()); this.bytes = content.getBytes(Charsets.UTF_8);
getContentMetadata().setContentLength(new Long(bytes.length));
} }
/** /**
@ -38,7 +47,7 @@ public class StringPayload extends BasePayload<String> {
*/ */
@Override @Override
public InputStream getInput() { public InputStream getInput() {
return Utils.toInputStream(content); return new ByteArrayInputStream(bytes);
} }
} }

2
core/src/main/java/org/jclouds/util/Utils.java
View File

@ -47,10 +47,10 @@ import java.util.Comparator;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import java.util.Map.Entry;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;