diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java new file mode 100644 index 0000000000..7ffebc51be --- /dev/null +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import javax.inject.Singleton; + +@Singleton +public class MultipartNamingStrategy { + + private static final String PART_SEPARATOR = "/"; + + protected String getPartName(String key, int partNumber, int totalParts) { + int base = (int) Math.log10(totalParts) + 1; + return String.format("%s%s%0" + base + "d", key, PART_SEPARATOR, partNumber); + } +} diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java index 1874ccd72d..5f21ddcda9 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java @@ -34,7 +34,6 @@ import org.jclouds.openstack.swift.blobstore.functions.BlobToObject; import com.google.inject.Inject; public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy { - private static final String PART_SEPARATOR = "/"; @Resource @Named(BlobStoreConstants.BLOBSTORE_LOGGER) @@ -45,15 +44,18 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg private final BlobToObject blob2Object; private final MultipartUploadSlicingAlgorithm algorithm; private final PayloadSlicer slicer; + private final MultipartNamingStrategy namingStrategy; @Inject public SequentialMultipartUploadStrategy(CommonSwiftClient client, Provider blobBuilders, - BlobToObject blob2Object, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer) { + BlobToObject blob2Object, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer, + MultipartNamingStrategy namingStrategy) { this.client = checkNotNull(client, "client"); this.blobBuilders = checkNotNull(blobBuilders, "blobBuilders"); this.blob2Object = checkNotNull(blob2Object, "blob2Object"); this.algorithm = checkNotNull(algorithm, "algorithm"); this.slicer = checkNotNull(slicer, "slicer"); + this.namingStrategy = checkNotNull(namingStrategy, "namingStrategy"); } @Override @@ -68,10 +70,11 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg if (partCount > 0) { for (Payload part : slicer.slice(payload, chunkSize)) { int partNum = algorithm.getNextPart(); + String partName = namingStrategy.getPartName(key, partNum, partCount); Blob blobPart = blobBuilders.get() - .name(key + PART_SEPARATOR + partNum) + .name(partName) .payload(part) - .contentDisposition(key + PART_SEPARATOR + partNum) + .contentDisposition(partName) .build(); client.putObject(container, blob2Object.apply(blobPart)); } diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java index 6277dc4314..8d87787b9b 100644 --- a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java @@ -26,11 +26,14 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Properties; +import java.util.Random; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest; import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.io.ByteSources; +import org.jclouds.io.Payload; import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties; import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload; import org.testng.ITestContext; @@ -40,6 +43,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import com.google.common.hash.Hashing; +import com.google.common.io.ByteSource; import com.google.common.io.ByteStreams; import com.google.common.io.Files; import com.google.common.io.InputSupplier; @@ -83,18 +87,18 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { throw new SkipException("not yet implemented"); } - @BeforeClass(groups = { "integration", "live" }, dependsOnMethods = "setupContext") - @Override - public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { - super.setUpResourcesOnThisThread(testContext); - oneHundredOneConstitutions = getTestDataSupplier(); - } + @BeforeClass(groups = {"integration", "live"}, dependsOnMethods = "setupContext") + @Override + public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { + super.setUpResourcesOnThisThread(testContext); + oneHundredOneConstitutions = getTestDataSupplier(); + } @Override protected void checkContentDisposition(Blob blob, String contentDisposition) { - // This works for Swift Server 1.4.4/SWauth 1.0.3 but was null in previous versions. - // TODO: Better testing for the different versions. - super.checkContentDisposition(blob,contentDisposition); + // This works for Swift Server 1.4.4/SWauth 1.0.3 but was null in previous versions. + // TODO: Better testing for the different versions. + super.checkContentDisposition(blob, contentDisposition); } // not supported in swift @@ -128,7 +132,40 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { "A multipart blob wasn't actually created - " + "there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs"); } finally { - returnContainer(containerName); + returnContainer(containerName); + } + } + + /** + * Checks that when there are more than 9 chunks the object names + * are set correctly so that the order of the object names matches + * the upload order. + * See issue https://issues.apache.org/jira/browse/JCLOUDS-619 + */ + @Test(groups = {"integration", "live"}) + public void testMultipartChunkedFilenames() throws InterruptedException, IOException { + String containerName = getContainerName(); + try { + BlobStore blobStore = view.getBlobStore(); + String objectName = "object.txt"; + long countBefore = blobStore.countBlobs(containerName); + + // we want 11 parts + ByteSource inputSource = createByteSource(PART_SIZE * 11); + addMultipartBlobToContainer(containerName, objectName, inputSource); + + // did we create enough parts? + long countAfter = blobStore.countBlobs(containerName); + assertNotEquals(countAfter, countBefore, "No blob was created"); + assertEquals(countAfter, countBefore + 12, + "12 parts (11 objects + 1 manifest) were expected."); + + // download and check if correct + Blob read = blobStore.getBlob(containerName, objectName); + Payload readPayload = read.getPayload(); + assertTrue(inputSource.contentEquals(ByteSources.asByteSource(readPayload.openStream()))); + } finally { + returnContainer(containerName); } } @@ -165,15 +202,28 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { protected void addMultipartBlobToContainer(String containerName, String key) throws IOException { File fileToUpload = createFileBiggerThan(PART_SIZE); + addMultipartBlobToContainer(containerName, key, Files.asByteSource(fileToUpload)); + } + protected void addMultipartBlobToContainer(String containerName, String key, ByteSource byteSource) throws IOException { BlobStore blobStore = view.getBlobStore(); blobStore.createContainerInLocation(null, containerName); Blob blob = blobStore.blobBuilder(key) - .payload(fileToUpload) - .build(); + .payload(byteSource) + .contentLength(byteSource.size()) + .build(); blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart()); } + private ByteSource createByteSource(long size) throws IOException { + final Random random = new Random(); + final byte[] randomBytes = new byte[(int) MultipartUpload.MIN_PART_SIZE]; + random.nextBytes(randomBytes); + ByteSource byteSource = ByteSources.repeatingArrayByteSource(randomBytes).slice(0, size); + assertEquals(byteSource.size(), size); + return byteSource; + } + @SuppressWarnings("unchecked") private File createFileBiggerThan(long partSize) throws IOException { long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1; diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java new file mode 100644 index 0000000000..364c410642 --- /dev/null +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.blobstore.strategy.internal; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +@Test(testName = "MultipartNamingStrategyTest") +public class MultipartNamingStrategyTest { + + @Test + public void testGetPartNameFirstOneHundred() { + final MultipartNamingStrategy strategy = new MultipartNamingStrategy(); + final String key = "file.txt"; + final int numberParts = 100; + + // check the first 100 + for (int i = 0; i < numberParts; i++) { + String partName = strategy.getPartName(key, i + 1, numberParts); + assertEquals(String.format("file.txt/%03d", i + 1), partName); + } + } + + @Test + public void testGetPartNameChoices() { + final MultipartNamingStrategy strategy = new MultipartNamingStrategy(); + final String key = "file.txt"; + + // check less than 10 parts + assertEquals(strategy.getPartName(key, 1, 5), "file.txt/1"); + assertEquals(strategy.getPartName(key, 2, 5), "file.txt/2"); + assertEquals(strategy.getPartName(key, 5, 5), "file.txt/5"); + + // check <= 10 parts + assertEquals(strategy.getPartName(key, 1, 10), "file.txt/01"); + assertEquals(strategy.getPartName(key, 2, 10), "file.txt/02"); + assertEquals(strategy.getPartName(key, 10, 10), "file.txt/10"); + + // check <= 100 parts + assertEquals(strategy.getPartName(key, 1, 100), "file.txt/001"); + assertEquals(strategy.getPartName(key, 9, 100), "file.txt/009"); + assertEquals(strategy.getPartName(key, 10, 100), "file.txt/010"); + assertEquals(strategy.getPartName(key, 99, 100), "file.txt/099"); + assertEquals(strategy.getPartName(key, 100, 100), "file.txt/100"); + + // check <= 5000 parts + assertEquals(strategy.getPartName(key, 1, 5000), "file.txt/0001"); + assertEquals(strategy.getPartName(key, 10, 5000), "file.txt/0010"); + assertEquals(strategy.getPartName(key, 99, 5000), "file.txt/0099"); + assertEquals(strategy.getPartName(key, 100, 5000), "file.txt/0100"); + assertEquals(strategy.getPartName(key, 999, 5000), "file.txt/0999"); + assertEquals(strategy.getPartName(key, 4999, 5000), "file.txt/4999"); + assertEquals(strategy.getPartName(key, 5000, 500), "file.txt/5000"); + } +} +