Apache/jclouds
1
0
Fork 0
mirror of https://github.com/apache/jclouds.git synced 2025-02-26 12:35:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a test for uploading large objects

Browse Source 
This is disabled by default.  Also rework upload helper to avoid using
an unbounded amount of memory.
This commit is contained in:
Andrew Gaul 2024-02-21 21:57:51 +09:00
parent ed160d30c5
commit 6bb738195f
1 changed files with 27 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
View File

@ -699,6 +699,22 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
testPut(payload, hashCode, payload, length, new PutOptions().multipart(true));
}
@Test(groups = { "integration", "live" }, enabled = false)
public void testPutMultipartByteSourceBig() throws Exception {
long length = 5L * 1024 * 1024 * 1024 + 1;
BlobStore blobStore = view.getBlobStore();
MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
blobStore.getMinimumMultipartPartSize(), blobStore.getMaximumMultipartPartSize(),
blobStore.getMaximumNumberOfParts());
// make sure that we are creating multiple parts
assertThat(algorithm.calculateChunkSize(length)).isLessThan(length);
// TODO: this is inefficient due to skip recomputing entire stream
ByteSource byteSource = TestUtils.randomByteSource().slice(0, length);
Payload payload = new ByteSourcePayload(byteSource);
HashCode hashCode = byteSource.hash(Hashing.md5());
testPut(payload, hashCode, payload, length, new PutOptions().multipart(true));
}
@Test(groups = { "integration", "live" })
public void testPutMultipartInputStream() throws Exception {
long length = Math.max(getMinimumMultipartBlobSize(), MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE + 1);
@ -858,12 +874,17 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
Blob blob = blobStore.getBlob(container, blobName);
assertThat(blob.getMetadata().getContentMetadata().getContentLength()).isEqualTo(length);
InputStream is = null;
try {
is = blob.getPayload().openStream();
assertThat(is).hasContentEqualTo(expectedPayload.openStream());
} finally {
Closeables2.closeQuietly(is);
try (InputStream actual = blob.getPayload().openStream();
InputStream expected = expectedPayload.openStream()) {
while (length > 0) {
int bufferLength = (int) Math.min(1 << 20, length);
byte[] actualBuffer = new byte[bufferLength];
ByteStreams.readFully(actual, actualBuffer);
byte[] expectedBuffer = new byte[bufferLength];
ByteStreams.readFully(expected, expectedBuffer);
assertThat(actualBuffer).isEqualTo(expectedBuffer);
length -= bufferLength;
}
}
validateMetadata(blob.getMetadata(), container, blob.getMetadata().getName());
checkContentMetadata(blob);