Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-1606: JCLOUDS-1608: Fix MPU off-by-one 

Previously GCS could not upload large objects due to its 32 part
limit.
This commit is contained in:
Jan Vermeulen 2024-02-20 21:41:47 +09:00 committed by Andrew Gaul
parent a983eaced9
commit 47f34770c9
2 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
View File

@ -72,6 +72,7 @@ public final class MultipartUploadSlicingAlgorithm {
this.maximumNumberOfParts = maximumNumberOfParts; this.maximumNumberOfParts = maximumNumberOfParts;
} }
// TODO: This algorithm is needlessly complicated.
public long calculateChunkSize(long length) { public long calculateChunkSize(long length) {
long unitPartSize = defaultPartSize; // first try with default part size long unitPartSize = defaultPartSize; // first try with default part size
int parts = (int)(length / unitPartSize); int parts = (int)(length / unitPartSize);
@ -112,6 +113,10 @@ public final class MultipartUploadSlicingAlgorithm {
if (remainder == 0 && parts > 0) { if (remainder == 0 && parts > 0) {
parts -= 1; parts -= 1;
} }
if (remainder > 0 && parts == maximumNumberOfParts) {
parts -= 1;
partSize = length / parts;
}
this.chunkSize = partSize; this.chunkSize = partSize;
this.parts = parts; this.parts = parts;
this.remaining = length - partSize * parts; this.remaining = length - partSize * parts;

6
blobstore/src/test/java/org/jclouds/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java
View File

@ -137,9 +137,9 @@ public final class MpuPartitioningAlgorithmTest {
// then the number of parts is increasing // then the number of parts is increasing
length += 1; length += 1;
chunkSize = strategy.calculateChunkSize(length); chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MAX_PART_SIZE); assertEquals(chunkSize, 5369246044L);
assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS); assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS - 1);
assertEquals(strategy.getRemaining(), 1); assertEquals(strategy.getRemaining(), 6045);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
} }
} }