Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 430: two magic numbers parametrizable from now one

This commit is contained in:
Tibor Kiss 2011-03-19 17:18:43 +01:00
parent 6d90793ca6
commit 1c78722297
3 changed files with 20 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
View File

@ -21,12 +21,12 @@ package org.jclouds.aws.s3.blobstore.strategy.internal;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import java.util.Map; import java.util.Map;
import java.util.SortedMap; import java.util.SortedMap;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import javax.annotation.Resource; import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Named; import javax.inject.Named;
import org.jclouds.aws.s3.AWSS3Client; import org.jclouds.aws.s3.AWSS3Client;
@ -43,6 +43,7 @@ import org.jclouds.util.Throwables2;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import com.google.inject.Inject;
/** /**
* Provides a sequential multipart upload strategy. * Provides a sequential multipart upload strategy.
@ -61,8 +62,15 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
@Named(BlobStoreConstants.BLOBSTORE_LOGGER) @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
protected Logger logger = Logger.NULL; protected Logger logger = Logger.NULL;
static final long DEFAULT_PART_SIZE = 33554432; // 32mb @Inject(optional = true)
static final int MAGNITUDE_BASE = 100; @Named("jclouds.mpu.parts.size")
@VisibleForTesting
long DEFAULT_PART_SIZE = 33554432; // 32mb
@Inject(optional = true)
@Named("jclouds.mpu.parts.magnitude")
@VisibleForTesting
int magnitudeBase = 100;
private final AWSS3BlobStore ablobstore; private final AWSS3BlobStore ablobstore;
private final PayloadSlicer slicer; private final PayloadSlicer slicer;
@ -88,7 +96,7 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
long unitPartSize = DEFAULT_PART_SIZE; // first try with default part size long unitPartSize = DEFAULT_PART_SIZE; // first try with default part size
long parts = length / unitPartSize; long parts = length / unitPartSize;
long partSize = unitPartSize; long partSize = unitPartSize;
int magnitude = (int) (parts / MAGNITUDE_BASE); int magnitude = (int) (parts / magnitudeBase);
if (magnitude > 0) { if (magnitude > 0) {
partSize = magnitude * unitPartSize; partSize = magnitude * unitPartSize;
if (partSize > MAX_PART_SIZE) { if (partSize > MAX_PART_SIZE) {

4
providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/MpuGraphData.java
View File

@ -67,8 +67,8 @@ public class MpuGraphData {
SequentialMultipartUploadStrategy strategy = new SequentialMultipartUploadStrategy(ablobStore, slicer); SequentialMultipartUploadStrategy strategy = new SequentialMultipartUploadStrategy(ablobStore, slicer);
foreach(1L, foreach(1L,
SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE * SequentialMultipartUploadStrategy.MAGNITUDE_BASE, SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE * SequentialMultipartUploadStrategy.magnitudeBase,
MultipartUploadStrategy.MAX_PART_SIZE * SequentialMultipartUploadStrategy.MAGNITUDE_BASE, MultipartUploadStrategy.MAX_PART_SIZE * SequentialMultipartUploadStrategy.magnitudeBase,
MultipartUploadStrategy.MAX_PART_SIZE * MultipartUploadStrategy.MAX_NUMBER_OF_PARTS, MultipartUploadStrategy.MAX_PART_SIZE * MultipartUploadStrategy.MAX_NUMBER_OF_PARTS,
strategy); strategy);

12
providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java
View File

@ -96,10 +96,10 @@ public class MpuPartitioningAlgorithmTest {
SequentialMultipartUploadStrategy strategy = new SequentialMultipartUploadStrategy(ablobStore, slicer); SequentialMultipartUploadStrategy strategy = new SequentialMultipartUploadStrategy(ablobStore, slicer);
// upper limit while we still have exactly DEFAULT_PART_SIZE chunkSize // upper limit while we still have exactly DEFAULT_PART_SIZE chunkSize
long length = SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE * SequentialMultipartUploadStrategy.MAGNITUDE_BASE; long length = SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE * SequentialMultipartUploadStrategy.magnitudeBase;
long chunkSize = strategy.calculateChunkSize(length); long chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE); assertEquals(chunkSize, SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE);
assertEquals(strategy.getParts(), SequentialMultipartUploadStrategy.MAGNITUDE_BASE - 1); assertEquals(strategy.getParts(), SequentialMultipartUploadStrategy.magnitudeBase - 1);
assertEquals(strategy.getRemaining(), SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE); assertEquals(strategy.getRemaining(), SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
@ -107,7 +107,7 @@ public class MpuPartitioningAlgorithmTest {
length += 1; length += 1;
chunkSize = strategy.calculateChunkSize(length); chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE * 2); assertEquals(chunkSize, SequentialMultipartUploadStrategy.DEFAULT_PART_SIZE * 2);
assertEquals(strategy.getParts(), SequentialMultipartUploadStrategy.MAGNITUDE_BASE / 2); assertEquals(strategy.getParts(), SequentialMultipartUploadStrategy.magnitudeBase / 2);
assertEquals(strategy.getRemaining(), 1); assertEquals(strategy.getRemaining(), 1);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
@ -130,10 +130,10 @@ public class MpuPartitioningAlgorithmTest {
SequentialMultipartUploadStrategy strategy = new SequentialMultipartUploadStrategy(ablobStore, slicer); SequentialMultipartUploadStrategy strategy = new SequentialMultipartUploadStrategy(ablobStore, slicer);
// upper limit while we still have exactly MAGNITUDE_BASE parts (together with the remaining) // upper limit while we still have exactly MAGNITUDE_BASE parts (together with the remaining)
long length = MultipartUploadStrategy.MAX_PART_SIZE * SequentialMultipartUploadStrategy.MAGNITUDE_BASE; long length = MultipartUploadStrategy.MAX_PART_SIZE * SequentialMultipartUploadStrategy.magnitudeBase;
long chunkSize = strategy.calculateChunkSize(length); long chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUploadStrategy.MAX_PART_SIZE); assertEquals(chunkSize, MultipartUploadStrategy.MAX_PART_SIZE);
assertEquals(strategy.getParts(), SequentialMultipartUploadStrategy.MAGNITUDE_BASE - 1); assertEquals(strategy.getParts(), SequentialMultipartUploadStrategy.magnitudeBase - 1);
assertEquals(strategy.getRemaining(), MultipartUploadStrategy.MAX_PART_SIZE); assertEquals(strategy.getRemaining(), MultipartUploadStrategy.MAX_PART_SIZE);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
@ -141,7 +141,7 @@ public class MpuPartitioningAlgorithmTest {
length += 1; length += 1;
chunkSize = strategy.calculateChunkSize(length); chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUploadStrategy.MAX_PART_SIZE); assertEquals(chunkSize, MultipartUploadStrategy.MAX_PART_SIZE);
assertEquals(strategy.getParts(), SequentialMultipartUploadStrategy.MAGNITUDE_BASE); assertEquals(strategy.getParts(), SequentialMultipartUploadStrategy.magnitudeBase);
assertEquals(strategy.getRemaining(), 1); assertEquals(strategy.getRemaining(), 1);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);