mirror of https://github.com/apache/jclouds.git
Issue 414: fixed length problem on string payload by eagerly encoding to UTF-8
This commit is contained in:
parent
8ce596be50
commit
53ac4751f5
|
@ -97,7 +97,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
|
|||
@SuppressWarnings("unchecked")
|
||||
public static InputSupplier<InputStream> getTestDataSupplier() throws IOException {
|
||||
byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class
|
||||
.getResourceAsStream("/const.txt.gz")));
|
||||
.getResourceAsStream("/const.txt.gz")));
|
||||
InputSupplier<ByteArrayInputStream> constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution);
|
||||
|
||||
InputSupplier<InputStream> temp = ByteStreams.join(constitutionSupplier);
|
||||
|
@ -120,24 +120,24 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
|
|||
Map<Integer, Future<?>> responses = Maps.newHashMap();
|
||||
for (int i = 0; i < 10; i++) {
|
||||
|
||||
responses.put(i, Futures.compose(context.getAsyncBlobStore().getBlob(containerName, key),
|
||||
new Function<Blob, Void>() {
|
||||
responses.put(i,
|
||||
Futures.compose(context.getAsyncBlobStore().getBlob(containerName, key), new Function<Blob, Void>() {
|
||||
|
||||
@Override
|
||||
public Void apply(Blob from) {
|
||||
try {
|
||||
assertEquals(CryptoStreams.md5(from.getPayload()), oneHundredOneConstitutionsMD5);
|
||||
checkContentDisposition(from, expectedContentDisposition);
|
||||
} catch (IOException e) {
|
||||
Throwables.propagate(e);
|
||||
}
|
||||
return null;
|
||||
@Override
|
||||
public Void apply(Blob from) {
|
||||
try {
|
||||
assertEquals(CryptoStreams.md5(from.getPayload()), oneHundredOneConstitutionsMD5);
|
||||
checkContentDisposition(from, expectedContentDisposition);
|
||||
} catch (IOException e) {
|
||||
Throwables.propagate(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}, this.exec));
|
||||
}, this.exec));
|
||||
}
|
||||
Map<Integer, Exception> exceptions = awaitCompletion(responses, exec, 30000l, Logger.CONSOLE,
|
||||
"get constitution");
|
||||
"get constitution");
|
||||
assert exceptions.size() == 0 : exceptions;
|
||||
|
||||
} finally {
|
||||
|
@ -156,7 +156,6 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
|
|||
context.getBlobStore().putBlob(containerName, sourceObject);
|
||||
}
|
||||
|
||||
|
||||
@Test(groups = { "integration", "live" })
|
||||
public void testGetIfModifiedSince() throws InterruptedException {
|
||||
String containerName = getContainerName();
|
||||
|
@ -365,15 +364,15 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
|
|||
|
||||
@DataProvider(name = "delete")
|
||||
public Object[][] createData() {
|
||||
return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, { "colon:" },
|
||||
{ "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
|
||||
return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" },
|
||||
{ "colon:" }, { "asteri*k" }, { "quote\"" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } };
|
||||
}
|
||||
|
||||
@Test(groups = { "integration", "live" }, dataProvider = "delete")
|
||||
public void deleteObject(String key) throws InterruptedException {
|
||||
String containerName = getContainerName();
|
||||
try {
|
||||
addBlobToContainer(containerName, key);
|
||||
addBlobToContainer(containerName, key, key, MediaType.TEXT_PLAIN);
|
||||
context.getBlobStore().removeBlob(containerName, key);
|
||||
assertContainerEmptyDeleting(containerName, key);
|
||||
} finally {
|
||||
|
@ -383,17 +382,19 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
|
|||
|
||||
private void assertContainerEmptyDeleting(String containerName, String key) {
|
||||
Iterable<? extends StorageMetadata> listing = Iterables.filter(context.getBlobStore().list(containerName),
|
||||
new Predicate<StorageMetadata>() {
|
||||
new Predicate<StorageMetadata>() {
|
||||
|
||||
@Override
|
||||
public boolean apply(StorageMetadata input) {
|
||||
return input.getType() == StorageType.BLOB;
|
||||
}
|
||||
@Override
|
||||
public boolean apply(StorageMetadata input) {
|
||||
return input.getType() == StorageType.BLOB;
|
||||
}
|
||||
|
||||
});
|
||||
assertEquals(Iterables.size(listing), 0, String.format(
|
||||
"deleting %s, we still have %s blobs left in container %s, using encoding %s", key, Iterables
|
||||
.size(listing), containerName, LOCAL_ENCODING));
|
||||
});
|
||||
assertEquals(
|
||||
Iterables.size(listing),
|
||||
0,
|
||||
String.format("deleting %s, we still have %s blobs left in container %s, using encoding %s", key,
|
||||
Iterables.size(listing), containerName, LOCAL_ENCODING));
|
||||
}
|
||||
|
||||
@Test(groups = { "integration", "live" })
|
||||
|
@ -413,13 +414,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
|
|||
String realObject = Utils.toStringAndClose(new FileInputStream("pom.xml"));
|
||||
|
||||
return new Object[][] { { "file", "text/xml", new File("pom.xml"), realObject },
|
||||
{ "string", "text/xml", realObject, realObject },
|
||||
{ "bytes", "application/octet-stream", realObject.getBytes(), realObject } };
|
||||
{ "string", "text/xml", realObject, realObject },
|
||||
{ "bytes", "application/octet-stream", realObject.getBytes(), realObject } };
|
||||
}
|
||||
|
||||
@Test(groups = { "integration", "live" }, dataProvider = "putTests")
|
||||
public void testPutObject(String key, String type, Object content, Object realObject) throws InterruptedException,
|
||||
IOException {
|
||||
IOException {
|
||||
Blob blob = context.getBlobStore().newBlob(key);
|
||||
blob.setPayload(Payloads.newPayload(content));
|
||||
blob.getMetadata().getContentMetadata().setContentType(type);
|
||||
|
@ -486,32 +487,32 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
|
|||
|
||||
protected void checkContentType(Blob blob, String contentType) {
|
||||
assert blob.getPayload().getContentMetadata().getContentType().startsWith(contentType) : blob.getPayload()
|
||||
.getContentMetadata().getContentType();
|
||||
.getContentMetadata().getContentType();
|
||||
assert blob.getMetadata().getContentMetadata().getContentType().startsWith(contentType) : blob.getMetadata()
|
||||
.getContentMetadata().getContentType();
|
||||
.getContentMetadata().getContentType();
|
||||
}
|
||||
|
||||
protected void checkContentDisposition(Blob blob, String contentDisposition) {
|
||||
assert blob.getPayload().getContentMetadata().getContentDisposition().startsWith(contentDisposition) : blob
|
||||
.getPayload().getContentMetadata().getContentDisposition();
|
||||
.getPayload().getContentMetadata().getContentDisposition();
|
||||
assert blob.getMetadata().getContentMetadata().getContentDisposition().startsWith(contentDisposition) : blob
|
||||
.getMetadata().getContentMetadata().getContentDisposition();
|
||||
.getMetadata().getContentMetadata().getContentDisposition();
|
||||
|
||||
}
|
||||
|
||||
protected void checkContentEncoding(Blob blob, String contentEncoding) {
|
||||
assert blob.getPayload().getContentMetadata().getContentEncoding().startsWith(contentEncoding) : blob
|
||||
.getPayload().getContentMetadata().getContentEncoding();
|
||||
.getPayload().getContentMetadata().getContentEncoding();
|
||||
assert blob.getMetadata().getContentMetadata().getContentEncoding().startsWith(contentEncoding) : blob
|
||||
.getMetadata().getContentMetadata().getContentEncoding();
|
||||
.getMetadata().getContentMetadata().getContentEncoding();
|
||||
|
||||
}
|
||||
|
||||
protected void checkContentLanguage(Blob blob, String contentLanguage) {
|
||||
assert blob.getPayload().getContentMetadata().getContentLanguage().startsWith(contentLanguage) : blob
|
||||
.getPayload().getContentMetadata().getContentLanguage();
|
||||
.getPayload().getContentMetadata().getContentLanguage();
|
||||
assert blob.getMetadata().getContentMetadata().getContentLanguage().startsWith(contentLanguage) : blob
|
||||
.getMetadata().getContentMetadata().getContentLanguage();
|
||||
.getMetadata().getContentMetadata().getContentLanguage();
|
||||
|
||||
}
|
||||
|
||||
|
@ -565,7 +566,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
|
|||
|
||||
protected void validateMetadata(BlobMetadata metadata) throws IOException {
|
||||
assert metadata.getContentMetadata().getContentType().startsWith("text/plain") : metadata.getContentMetadata()
|
||||
.getContentType();
|
||||
.getContentType();
|
||||
assertEquals(metadata.getContentMetadata().getContentLength(), new Long(TEST_STRING.length()));
|
||||
assertEquals(metadata.getUserMetadata().get("adrian"), "powderpuff");
|
||||
checkMD5(metadata);
|
||||
|
|
|
@ -25,8 +25,8 @@ import static org.testng.Assert.assertEquals;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.CancellationException;
|
||||
|
@ -35,6 +35,8 @@ import java.util.concurrent.Executors;
|
|||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import javax.ws.rs.core.MediaType;
|
||||
|
||||
import org.jclouds.blobstore.BlobStoreContext;
|
||||
import org.jclouds.blobstore.attr.ConsistencyModel;
|
||||
import org.jclouds.blobstore.domain.Blob;
|
||||
|
@ -60,13 +62,13 @@ public class BaseBlobStoreIntegrationTest {
|
|||
protected static final String TEST_STRING = String.format(XML_STRING_FORMAT, "apple");
|
||||
|
||||
protected Map<String, String> fiveStrings = ImmutableMap.of("one", String.format(XML_STRING_FORMAT, "apple"), "two",
|
||||
String.format(XML_STRING_FORMAT, "bear"), "three", String.format(XML_STRING_FORMAT, "candy"), "four",
|
||||
String.format(XML_STRING_FORMAT, "dogma"), "five", String.format(XML_STRING_FORMAT, "emma"));
|
||||
String.format(XML_STRING_FORMAT, "bear"), "three", String.format(XML_STRING_FORMAT, "candy"), "four",
|
||||
String.format(XML_STRING_FORMAT, "dogma"), "five", String.format(XML_STRING_FORMAT, "emma"));
|
||||
|
||||
protected Map<String, String> fiveStringsUnderPath = ImmutableMap.of("path/1", String.format(XML_STRING_FORMAT,
|
||||
"apple"), "path/2", String.format(XML_STRING_FORMAT, "bear"), "path/3", String.format(XML_STRING_FORMAT,
|
||||
"candy"), "path/4", String.format(XML_STRING_FORMAT, "dogma"), "path/5", String.format(XML_STRING_FORMAT,
|
||||
"emma"));
|
||||
protected Map<String, String> fiveStringsUnderPath = ImmutableMap.of("path/1",
|
||||
String.format(XML_STRING_FORMAT, "apple"), "path/2", String.format(XML_STRING_FORMAT, "bear"), "path/3",
|
||||
String.format(XML_STRING_FORMAT, "candy"), "path/4", String.format(XML_STRING_FORMAT, "dogma"), "path/5",
|
||||
String.format(XML_STRING_FORMAT, "emma"));
|
||||
|
||||
public static long INCONSISTENCY_WINDOW = 10000;
|
||||
protected static volatile AtomicInteger containerIndex = new AtomicInteger(0);
|
||||
|
@ -90,7 +92,7 @@ public class BaseBlobStoreIntegrationTest {
|
|||
|
||||
@SuppressWarnings("unchecked")
|
||||
private BlobStoreContext getCloudResources(ITestContext testContext) throws ClassNotFoundException,
|
||||
InstantiationException, IllegalAccessException, Exception {
|
||||
InstantiationException, IllegalAccessException, Exception {
|
||||
String initializerClass = checkNotNull(System.getProperty("test.initializer"), "test.initializer");
|
||||
Class<BaseTestInitializer> clazz = (Class<BaseTestInitializer>) Class.forName(initializerClass);
|
||||
BaseTestInitializer initializer = clazz.newInstance();
|
||||
|
@ -123,7 +125,7 @@ public class BaseBlobStoreIntegrationTest {
|
|||
private static volatile boolean initialized = false;
|
||||
|
||||
protected void createContainersSharedByAllThreads(BlobStoreContext context, ITestContext testContext)
|
||||
throws Exception {
|
||||
throws Exception {
|
||||
while (!initialized) {
|
||||
synchronized (BaseBlobStoreIntegrationTest.class) {
|
||||
if (!initialized) {
|
||||
|
@ -174,12 +176,12 @@ public class BaseBlobStoreIntegrationTest {
|
|||
try {
|
||||
for (int i = 0; i < 2; i++) {
|
||||
Iterable<? extends StorageMetadata> testContainers = Iterables.filter(context.getBlobStore().list(),
|
||||
new Predicate<StorageMetadata>() {
|
||||
public boolean apply(StorageMetadata input) {
|
||||
return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER)
|
||||
&& input.getName().startsWith(CONTAINER_PREFIX.toLowerCase());
|
||||
}
|
||||
});
|
||||
new Predicate<StorageMetadata>() {
|
||||
public boolean apply(StorageMetadata input) {
|
||||
return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER)
|
||||
&& input.getName().startsWith(CONTAINER_PREFIX.toLowerCase());
|
||||
}
|
||||
});
|
||||
for (StorageMetadata container : testContainers) {
|
||||
deleteContainerOrWarnIfUnable(context, container.getName());
|
||||
}
|
||||
|
@ -200,7 +202,7 @@ public class BaseBlobStoreIntegrationTest {
|
|||
* we will try up to the inconsistency window to see if the assertion completes.
|
||||
*/
|
||||
protected static void assertConsistencyAware(BlobStoreContext context, Runnable assertion)
|
||||
throws InterruptedException {
|
||||
throws InterruptedException {
|
||||
if (context.getConsistencyModel() == ConsistencyModel.STRICT) {
|
||||
assertion.run();
|
||||
return;
|
||||
|
@ -226,7 +228,7 @@ public class BaseBlobStoreIntegrationTest {
|
|||
}
|
||||
|
||||
protected static void createContainerAndEnsureEmpty(BlobStoreContext context, final String containerName)
|
||||
throws InterruptedException {
|
||||
throws InterruptedException {
|
||||
context.getBlobStore().createContainerInLocation(null, containerName);
|
||||
if (context.getConsistencyModel() == ConsistencyModel.EVENTUAL)
|
||||
Thread.sleep(1000);
|
||||
|
@ -238,9 +240,13 @@ public class BaseBlobStoreIntegrationTest {
|
|||
}
|
||||
|
||||
protected String addBlobToContainer(String sourceContainer, String key) {
|
||||
return addBlobToContainer(sourceContainer, key, TEST_STRING, MediaType.TEXT_XML);
|
||||
}
|
||||
|
||||
protected String addBlobToContainer(String sourceContainer, String key, String payload, String contentType) {
|
||||
Blob sourceObject = context.getBlobStore().newBlob(key);
|
||||
sourceObject.setPayload(TEST_STRING);
|
||||
sourceObject.getMetadata().getContentMetadata().setContentType("text/xml");
|
||||
sourceObject.setPayload(payload);
|
||||
sourceObject.getMetadata().getContentMetadata().setContentType(contentType);
|
||||
return addBlobToContainer(sourceContainer, sourceObject);
|
||||
}
|
||||
|
||||
|
@ -270,19 +276,19 @@ public class BaseBlobStoreIntegrationTest {
|
|||
}
|
||||
|
||||
protected void assertConsistencyAwareContainerSize(final String containerName, final int count)
|
||||
throws InterruptedException {
|
||||
throws InterruptedException {
|
||||
assertConsistencyAware(new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
assert context.getBlobStore().countBlobs(containerName) == count : String.format(
|
||||
"expected only %d values in %s: %s", count, containerName, Sets.newHashSet(Iterables.transform(
|
||||
context.getBlobStore().list(containerName), new Function<StorageMetadata, String>() {
|
||||
"expected only %d values in %s: %s", count, containerName, Sets.newHashSet(Iterables.transform(
|
||||
context.getBlobStore().list(containerName), new Function<StorageMetadata, String>() {
|
||||
|
||||
public String apply(StorageMetadata from) {
|
||||
return from.getName();
|
||||
}
|
||||
public String apply(StorageMetadata from) {
|
||||
return from.getName();
|
||||
}
|
||||
|
||||
})));
|
||||
})));
|
||||
} catch (Exception e) {
|
||||
Throwables.propagateIfPossible(e);
|
||||
}
|
||||
|
|
|
@ -19,18 +19,27 @@
|
|||
|
||||
package org.jclouds.io.payloads;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.jclouds.util.Utils;
|
||||
import com.google.common.base.Charsets;
|
||||
|
||||
/**
|
||||
* This implementation converts the String to a byte array using UTF-8 encoding. If you wish to use
|
||||
* a different encoding, please use {@link ByteArrayPayload}.
|
||||
*
|
||||
* @author Adrian Cole
|
||||
*/
|
||||
public class StringPayload extends BasePayload<String> {
|
||||
|
||||
private final byte[] bytes;
|
||||
|
||||
// it is possible to discover length by walking the string and updating current length based on
|
||||
// character code. However, this is process intense, and assumes an encoding type of UTF-8
|
||||
public StringPayload(String content) {
|
||||
super(content);
|
||||
getContentMetadata().setContentLength((long) content.length());
|
||||
this.bytes = content.getBytes(Charsets.UTF_8);
|
||||
getContentMetadata().setContentLength(new Long(bytes.length));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -38,7 +47,7 @@ public class StringPayload extends BasePayload<String> {
|
|||
*/
|
||||
@Override
|
||||
public InputStream getInput() {
|
||||
return Utils.toInputStream(content);
|
||||
return new ByteArrayInputStream(bytes);
|
||||
}
|
||||
|
||||
}
|
|
@ -47,10 +47,10 @@ import java.util.Comparator;
|
|||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
|
Loading…
Reference in New Issue