Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-619: Introduce MultipartNamingStrategy to generate part names correctly.

This commit is contained in:
Markus von Rüden 2014-07-01 14:24:50 +02:00 committed by Andrew Gaul
parent c0d7b3d248
commit 0f0207a045
4 changed files with 170 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.openstack.swift.blobstore.strategy.internal;
import javax.inject.Singleton;
@Singleton
public class MultipartNamingStrategy {
private static final String PART_SEPARATOR = "/";
protected String getPartName(String key, int partNumber, int totalParts) {
int base = (int) Math.log10(totalParts) + 1;
return String.format("%s%s%0" + base + "d", key, PART_SEPARATOR, partNumber);
}
}

11
apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java
View File

@ -34,7 +34,6 @@ import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
import com.google.inject.Inject;
public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy {
private static final String PART_SEPARATOR = "/";
@Resource
@Named(BlobStoreConstants.BLOBSTORE_LOGGER)
@ -45,15 +44,18 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
private final BlobToObject blob2Object;
private final MultipartUploadSlicingAlgorithm algorithm;
private final PayloadSlicer slicer;
private final MultipartNamingStrategy namingStrategy;
@Inject
public SequentialMultipartUploadStrategy(CommonSwiftClient client, Provider<BlobBuilder> blobBuilders,
BlobToObject blob2Object, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer) {
BlobToObject blob2Object, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer,
MultipartNamingStrategy namingStrategy) {
this.client = checkNotNull(client, "client");
this.blobBuilders = checkNotNull(blobBuilders, "blobBuilders");
this.blob2Object = checkNotNull(blob2Object, "blob2Object");
this.algorithm = checkNotNull(algorithm, "algorithm");
this.slicer = checkNotNull(slicer, "slicer");
this.namingStrategy = checkNotNull(namingStrategy, "namingStrategy");
}
@Override
@ -68,10 +70,11 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
if (partCount > 0) {
for (Payload part : slicer.slice(payload, chunkSize)) {
int partNum = algorithm.getNextPart();
String partName = namingStrategy.getPartName(key, partNum, partCount);
Blob blobPart = blobBuilders.get()
.name(key + PART_SEPARATOR + partNum)
.name(partName)
.payload(part)
.contentDisposition(key + PART_SEPARATOR + partNum)
.contentDisposition(partName)
.build();
client.putObject(container, blob2Object.apply(blobPart));
}

74
apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java
View File

@ -26,11 +26,14 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.Random;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.io.ByteSources;
import org.jclouds.io.Payload;
import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
import org.testng.ITestContext;
@ -40,6 +43,7 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import com.google.common.hash.Hashing;
import com.google.common.io.ByteSource;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import com.google.common.io.InputSupplier;
@ -83,18 +87,18 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
throw new SkipException("not yet implemented");
}
@BeforeClass(groups = { "integration", "live" }, dependsOnMethods = "setupContext")
@Override
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
oneHundredOneConstitutions = getTestDataSupplier();
}
@BeforeClass(groups = {"integration", "live"}, dependsOnMethods = "setupContext")
@Override
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
oneHundredOneConstitutions = getTestDataSupplier();
}
@Override
protected void checkContentDisposition(Blob blob, String contentDisposition) {
// This works for Swift Server 1.4.4/SWauth 1.0.3 but was null in previous versions.
// TODO: Better testing for the different versions.
super.checkContentDisposition(blob,contentDisposition);
// This works for Swift Server 1.4.4/SWauth 1.0.3 but was null in previous versions.
// TODO: Better testing for the different versions.
super.checkContentDisposition(blob, contentDisposition);
}
// not supported in swift
@ -128,7 +132,40 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
"A multipart blob wasn't actually created - " +
"there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs");
} finally {
returnContainer(containerName);
returnContainer(containerName);
}
}
/**
* Checks that when there are more than 9 chunks the object names
* are set correctly so that the order of the object names matches
* the upload order.
* See issue https://issues.apache.org/jira/browse/JCLOUDS-619
*/
@Test(groups = {"integration", "live"})
public void testMultipartChunkedFilenames() throws InterruptedException, IOException {
String containerName = getContainerName();
try {
BlobStore blobStore = view.getBlobStore();
String objectName = "object.txt";
long countBefore = blobStore.countBlobs(containerName);
// we want 11 parts
ByteSource inputSource = createByteSource(PART_SIZE * 11);
addMultipartBlobToContainer(containerName, objectName, inputSource);
// did we create enough parts?
long countAfter = blobStore.countBlobs(containerName);
assertNotEquals(countAfter, countBefore, "No blob was created");
assertEquals(countAfter, countBefore + 12,
"12 parts (11 objects + 1 manifest) were expected.");
// download and check if correct
Blob read = blobStore.getBlob(containerName, objectName);
Payload readPayload = read.getPayload();
assertTrue(inputSource.contentEquals(ByteSources.asByteSource(readPayload.openStream())));
} finally {
returnContainer(containerName);
}
}
@ -165,15 +202,28 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
protected void addMultipartBlobToContainer(String containerName, String key) throws IOException {
File fileToUpload = createFileBiggerThan(PART_SIZE);
addMultipartBlobToContainer(containerName, key, Files.asByteSource(fileToUpload));
}
protected void addMultipartBlobToContainer(String containerName, String key, ByteSource byteSource) throws IOException {
BlobStore blobStore = view.getBlobStore();
blobStore.createContainerInLocation(null, containerName);
Blob blob = blobStore.blobBuilder(key)
.payload(fileToUpload)
.build();
.payload(byteSource)
.contentLength(byteSource.size())
.build();
blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
}
private ByteSource createByteSource(long size) throws IOException {
final Random random = new Random();
final byte[] randomBytes = new byte[(int) MultipartUpload.MIN_PART_SIZE];
random.nextBytes(randomBytes);
ByteSource byteSource = ByteSources.repeatingArrayByteSource(randomBytes).slice(0, size);
assertEquals(byteSource.size(), size);
return byteSource;
}
@SuppressWarnings("unchecked")
private File createFileBiggerThan(long partSize) throws IOException {
long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1;

71
apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.openstack.swift.blobstore.strategy.internal;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@Test(testName = "MultipartNamingStrategyTest")
public class MultipartNamingStrategyTest {
@Test
public void testGetPartNameFirstOneHundred() {
final MultipartNamingStrategy strategy = new MultipartNamingStrategy();
final String key = "file.txt";
final int numberParts = 100;
// check the first 100
for (int i = 0; i < numberParts; i++) {
String partName = strategy.getPartName(key, i + 1, numberParts);
assertEquals(String.format("file.txt/%03d", i + 1), partName);
}
}
@Test
public void testGetPartNameChoices() {
final MultipartNamingStrategy strategy = new MultipartNamingStrategy();
final String key = "file.txt";
// check less than 10 parts
assertEquals(strategy.getPartName(key, 1, 5), "file.txt/1");
assertEquals(strategy.getPartName(key, 2, 5), "file.txt/2");
assertEquals(strategy.getPartName(key, 5, 5), "file.txt/5");
// check <= 10 parts
assertEquals(strategy.getPartName(key, 1, 10), "file.txt/01");
assertEquals(strategy.getPartName(key, 2, 10), "file.txt/02");
assertEquals(strategy.getPartName(key, 10, 10), "file.txt/10");
// check <= 100 parts
assertEquals(strategy.getPartName(key, 1, 100), "file.txt/001");
assertEquals(strategy.getPartName(key, 9, 100), "file.txt/009");
assertEquals(strategy.getPartName(key, 10, 100), "file.txt/010");
assertEquals(strategy.getPartName(key, 99, 100), "file.txt/099");
assertEquals(strategy.getPartName(key, 100, 100), "file.txt/100");
// check <= 5000 parts
assertEquals(strategy.getPartName(key, 1, 5000), "file.txt/0001");
assertEquals(strategy.getPartName(key, 10, 5000), "file.txt/0010");
assertEquals(strategy.getPartName(key, 99, 5000), "file.txt/0099");
assertEquals(strategy.getPartName(key, 100, 5000), "file.txt/0100");
assertEquals(strategy.getPartName(key, 999, 5000), "file.txt/0999");
assertEquals(strategy.getPartName(key, 4999, 5000), "file.txt/4999");
assertEquals(strategy.getPartName(key, 5000, 500), "file.txt/5000");
}
}