From a29d75a5d11f4d7ea330815035d0f307d8518551 Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Mon, 8 Jun 2015 17:25:52 -0700 Subject: [PATCH] JCLOUDS-929: Implement generic delimiter support. The patch adds delimiter option support in the generic blob store interface. A live integration test is added to verify that jclouds correctly lists objects separated by a delimiter. --- .../integration/AtmosContainerLiveTest.java | 5 +++ .../integration/SwiftContainerLiveTest.java | 7 ++++ .../integration/S3ContainerLiveTest.java | 7 ++++ .../integration/SwiftContainerLiveTest.java | 7 +++- .../options/ListContainerOptions.java | 42 ++++++++++++++++--- .../internal/DeleteAllKeysInList.java | 3 ++ .../internal/BaseContainerLiveTest.java | 24 ++++++++++- .../blobstore/options/ListOptionsTest.java | 7 ++++ .../AzureBlobContainerLiveTest.java | 7 ++++ ...HPCloudObjectStorageContainerLiveTest.java | 5 +++ 10 files changed, 107 insertions(+), 7 deletions(-) diff --git a/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosContainerLiveTest.java b/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosContainerLiveTest.java index e76f580ed8..7c9e0b122e 100644 --- a/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosContainerLiveTest.java +++ b/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosContainerLiveTest.java @@ -30,4 +30,9 @@ public class AtmosContainerLiveTest extends BaseContainerLiveTest { public void testContainerListWithPrefix() { throw new SkipException("Prefix option has not been plumbed down to Atmos"); } + + @Override + public void testDelimiterList() { + throw new SkipException("Delimiter support is not yet implemented"); + } } diff --git a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftContainerLiveTest.java b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftContainerLiveTest.java index 9bd85d6f5f..06dffda401 100644 --- a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftContainerLiveTest.java +++ b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftContainerLiveTest.java @@ -21,6 +21,7 @@ import static org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties.CRED import java.util.Properties; import org.jclouds.blobstore.integration.internal.BaseContainerLiveTest; +import org.testng.SkipException; import org.testng.annotations.Test; @Test(groups = "live", testName = "SwiftContainerLiveTest") @@ -36,4 +37,10 @@ public class SwiftContainerLiveTest extends BaseContainerLiveTest { setIfTestSystemPropertyPresent(props, CREDENTIAL_TYPE); return props; } + + @Override + @Test + public void testDelimiterList() { + throw new SkipException("Delimiter support is not yet implemented"); + } } diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3ContainerLiveTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3ContainerLiveTest.java index 7c90ea6203..8bcb78adcc 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3ContainerLiveTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3ContainerLiveTest.java @@ -18,6 +18,7 @@ package org.jclouds.s3.blobstore.integration; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; import org.jclouds.blobstore.integration.internal.BaseContainerLiveTest; +import org.testng.SkipException; import org.testng.annotations.Test; @Test(groups = "live", testName = "S3ContainerLiveTest") @@ -27,4 +28,10 @@ public class S3ContainerLiveTest extends BaseContainerLiveTest { provider = "s3"; BaseBlobStoreIntegrationTest.SANITY_CHECK_RETURNED_BUCKET_NAME = true; } + + @Override + @Test + public void testDelimiterList() { + throw new SkipException("not yet implemented"); + } } diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftContainerLiveTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftContainerLiveTest.java index e668f17147..6569b4cd7f 100644 --- a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftContainerLiveTest.java +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftContainerLiveTest.java @@ -33,7 +33,7 @@ public class SwiftContainerLiveTest extends BaseContainerLiveTest { setIfTestSystemPropertyPresent(props, KeystoneProperties.CREDENTIAL_TYPE); return props; } - + public SwiftContainerLiveTest() { provider = System.getProperty("test.swift.provider", "swift"); } @@ -58,4 +58,9 @@ public class SwiftContainerLiveTest extends BaseContainerLiveTest { public void testContainerListWithPrefix() { throw new SkipException("Prefix option has not been plumbed down to Swift"); } + + @Override + public void testDelimiterList() { + throw new SkipException("The test fails as the path parameter elides subdirectories"); + } } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java b/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java index e3d3f1c650..18d6555346 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java @@ -39,6 +39,7 @@ public class ListContainerOptions extends ListOptions implements Cloneable { public static final ImmutableListContainerOptions NONE = new ImmutableListContainerOptions( new ListContainerOptions()); + private String delimiter; private String dir; private String prefix; private boolean recursive; @@ -48,12 +49,13 @@ public class ListContainerOptions extends ListOptions implements Cloneable { } ListContainerOptions(Integer maxKeys, String marker, String dir, boolean recursive, - boolean detailed, String prefix) { + boolean detailed, String prefix, String delimiter) { super(maxKeys, marker); this.dir = dir; this.recursive = recursive; this.detailed = detailed; this.prefix = prefix; + this.delimiter = delimiter; } public static class ImmutableListContainerOptions extends ListContainerOptions { @@ -124,6 +126,16 @@ public class ListContainerOptions extends ListOptions implements Cloneable { return delegate.clone(); } + @Override + public ListContainerOptions delimiter(String delimiterString) { + throw new UnsupportedOperationException(); + } + + @Override + public String getDelimiter() { + return delegate.getDelimiter(); + } + @Override public String toString() { return delegate.toString(); @@ -135,6 +147,10 @@ public class ListContainerOptions extends ListOptions implements Cloneable { return dir; } + public String getDelimiter() { + return delimiter; + } + public boolean isRecursive() { return recursive; } @@ -149,7 +165,7 @@ public class ListContainerOptions extends ListOptions implements Cloneable { /** * This will list the contents of a virtual or real directory path. - * + * */ public ListContainerOptions inDirectory(String dir) { checkNotNull(dir, "dir"); @@ -200,6 +216,15 @@ public class ListContainerOptions extends ListOptions implements Cloneable { return this; } + /** + * specify the delimiter to be used when listing + * + */ + public ListContainerOptions delimiter(String delimiterString) { + this.delimiter = delimiterString; + return this; + } + public static class Builder { /** @@ -249,11 +274,18 @@ public class ListContainerOptions extends ListOptions implements Cloneable { ListContainerOptions options = new ListContainerOptions(); return options.prefix(prefix); } + /** + * @see ListContainerOptions#delimiter(String) + */ + public static ListContainerOptions delimiter(String delimiterString) { + ListContainerOptions options = new ListContainerOptions(); + return options.delimiter(delimiterString); + } } @Override public ListContainerOptions clone() { - return new ListContainerOptions(getMaxResults(), getMarker(), dir, recursive, detailed, prefix); + return new ListContainerOptions(getMaxResults(), getMarker(), dir, recursive, detailed, prefix, delimiter); } @Override @@ -282,6 +314,6 @@ public class ListContainerOptions extends ListOptions implements Cloneable { Objects.equal(getMarker(), other.getMarker()) && Objects.equal(getMaxResults(), other.getMaxResults()); } - - + + } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java index 529d57e532..7bd75e4e71 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java @@ -382,6 +382,9 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr public void execute(final String containerName, ListContainerOptions listOptions) { + if (listOptions.getDelimiter() != null || listOptions.getPrefix() != null) { + throw new IllegalArgumentException("Prefix and delimiter support has not yet been added"); + } final AtomicBoolean deleteFailure = new AtomicBoolean(); int retries = maxErrors; diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java index 9523a7c0e5..995cdbdeeb 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerLiveTest.java @@ -133,6 +133,29 @@ public class BaseContainerLiveTest extends BaseBlobStoreIntegrationTest { blobStore.putBlob(containerName, blobStore.blobBuilder("foo").payload("").build()); checkEqualNames(ImmutableSet.of(prefix, prefix + "foo", prefix + "bar"), blobStore.list(containerName, ListContainerOptions.Builder.prefix(prefix))); + } + finally { + returnContainer(containerName); + } + } + + @Test(groups = "live") + public void testDelimiterList() throws InterruptedException { + final String containerName = getContainerName(); + BlobStore blobStore = view.getBlobStore(); + String payload = "foo"; + try { + blobStore.putBlob(containerName, blobStore.blobBuilder("test/foo/foo").payload(payload).build()); + blobStore.putBlob(containerName, blobStore.blobBuilder("test/bar/foo").payload(payload).build()); + blobStore.putBlob(containerName, blobStore.blobBuilder("foo").payload(payload).build()); + blobStore.putBlob(containerName, blobStore.blobBuilder("test-foo").payload(payload).build()); + blobStore.putBlob(containerName, blobStore.blobBuilder("test-bar").payload(payload).build()); + checkEqualNames(ImmutableSet.of("foo", "test/", "test-foo", "test-bar"), blobStore.list( + containerName, ListContainerOptions.Builder.delimiter("/"))); + checkEqualNames(ImmutableSet.of("test/foo/foo", "test/bar/foo", "foo", "test-foo", "test-bar"), + blobStore.list(containerName, ListContainerOptions.Builder.delimiter("\\"))); + checkEqualNames(ImmutableSet.of("test-", "test/foo/foo", "test/bar/foo", "foo"), blobStore.list( + containerName, ListContainerOptions.Builder.delimiter("-"))); } finally { returnContainer(containerName); } @@ -173,5 +196,4 @@ public class BaseContainerLiveTest extends BaseBlobStoreIntegrationTest { recycleContainerAndAddToPool(containerName); } } - } diff --git a/blobstore/src/test/java/org/jclouds/blobstore/options/ListOptionsTest.java b/blobstore/src/test/java/org/jclouds/blobstore/options/ListOptionsTest.java index 1cccb1457e..e7d6f67604 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/options/ListOptionsTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/options/ListOptionsTest.java @@ -49,6 +49,13 @@ public class ListOptionsTest { assertEquals(options.getDir(), "test"); } + @Test + public void testDelimiter() { + ListContainerOptions options = new ListContainerOptions(); + options.delimiter("-"); + assertEquals(options.getDelimiter(), "-"); + } + @Test public void testPathStatic() { ListContainerOptions options = inDirectory("test"); diff --git a/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/integration/AzureBlobContainerLiveTest.java b/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/integration/AzureBlobContainerLiveTest.java index 5bfdcf2142..78fdb36da8 100644 --- a/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/integration/AzureBlobContainerLiveTest.java +++ b/providers/azureblob/src/test/java/org/jclouds/azureblob/blobstore/integration/AzureBlobContainerLiveTest.java @@ -17,6 +17,7 @@ package org.jclouds.azureblob.blobstore.integration; import org.jclouds.blobstore.integration.internal.BaseContainerLiveTest; +import org.testng.SkipException; import org.testng.annotations.Test; @Test(groups = { "live" }) @@ -24,4 +25,10 @@ public class AzureBlobContainerLiveTest extends BaseContainerLiveTest { public AzureBlobContainerLiveTest() { provider = "azureblob"; } + + @Override + @Test + public void testDelimiterList() { + throw new SkipException("The delimiter support has not been plumbed through to Azure blob"); + } } diff --git a/providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java b/providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java index 7fd5e9a5e0..ca112cdc10 100644 --- a/providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java +++ b/providers/hpcloud-objectstorage/src/test/java/org/jclouds/hpcloud/objectstorage/blobstore/integration/HPCloudObjectStorageContainerLiveTest.java @@ -34,4 +34,9 @@ public class HPCloudObjectStorageContainerLiveTest extends BaseContainerLiveTest @Test public void testPublicAccessInNonDefaultLocationWithBigBlob() { throw new SkipException("Locations are ignored"); } + @Override + @Test + public void testDelimiterList() { + throw new SkipException("\"path\" parameter elides subdirectories"); + } }