Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #592 from andrewgaul/clear-blobstore-recurse 

Recurse in subdirectories before deleting blobs
This commit is contained in:
Adrian Cole 2012-05-01 18:47:26 -07:00
parent 7a7e40c0d6 519a3ac193
commit df0da2e1fa
2 changed files with 74 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java
View File

@ -93,9 +93,9 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
message = message + " recursively";
Map<StorageMetadata, Exception> exceptions = Maps.newHashMap();
PageSet<? extends StorageMetadata> listing;
Iterable<? extends StorageMetadata> toDelete;
int maxErrors = 3; // TODO parameterize
for (int i = 0; i < maxErrors; ) {
// fetch partial directory listing
try {
listing = connection.list(containerName, options).get();
} catch (ExecutionException ee) {
@ -108,36 +108,52 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
} catch (InterruptedException ie) {
throw Throwables.propagate(ie);
}
toDelete = filterListing(listing, options);
Map<StorageMetadata, Future<?>> responses = Maps.newHashMap();
try {
for (final StorageMetadata md : toDelete) {
// recurse on subdirectories
if (options.isRecursive()) {
for (StorageMetadata md : listing) {
String fullPath = parentIsFolder(options, md) ? options.getDir() + "/"
+ md.getName() : md.getName();
switch (md.getType()) {
case BLOB:
responses.put(md, connection.removeBlob(containerName, fullPath));
break;
case FOLDER:
if (options.isRecursive() && !fullPath.equals(options.getDir())) {
execute(containerName, options.clone().inDirectory(fullPath));
}
responses.put(md, connection.deleteDirectory(containerName, fullPath));
break;
case RELATIVE_PATH:
if (options.isRecursive() && !fullPath.equals(options.getDir())) {
execute(containerName, options.clone().inDirectory(fullPath));
}
responses.put(md, connection.deleteDirectory(containerName, md.getName()));
break;
case CONTAINER:
throw new IllegalArgumentException("Container type not supported");
}
}
} finally {
exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, message);
}
// remove blobs and now-empty subdirectories
Map<StorageMetadata, Future<?>> responses = Maps.newHashMap();
for (StorageMetadata md : listing) {
String fullPath = parentIsFolder(options, md) ? options.getDir() + "/"
+ md.getName() : md.getName();
switch (md.getType()) {
case BLOB:
responses.put(md, connection.removeBlob(containerName, fullPath));
break;
case FOLDER:
if (options.isRecursive()) {
responses.put(md, connection.deleteDirectory(containerName, fullPath));
}
break;
case RELATIVE_PATH:
if (options.isRecursive()) {
responses.put(md, connection.deleteDirectory(containerName, md.getName()));
}
break;
case CONTAINER:
throw new IllegalArgumentException("Container type not supported");
}
}
exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, message);
if (!exceptions.isEmpty()) {
++i;
retryHandler.imposeBackoffExponentialDelay(i, message);
@ -157,29 +173,4 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
private boolean parentIsFolder(final ListContainerOptions options, final StorageMetadata md) {
return (options.getDir() != null && md.getName().indexOf('/') == -1);
}
private Iterable<? extends StorageMetadata> filterListing(
final PageSet<? extends StorageMetadata> listing,
final ListContainerOptions options) {
Iterable<? extends StorageMetadata> toDelete = Iterables.filter(listing,
new Predicate<StorageMetadata>() {
@Override
public boolean apply(StorageMetadata input) {
switch (input.getType()) {
case BLOB:
return true;
case FOLDER:
case RELATIVE_PATH:
if (options.isRecursive())
return true;
break;
}
return false;
}
});
return toDelete;
}
}

68
blobstore/src/test/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInListTest.java
View File

@ -23,8 +23,8 @@ import static org.testng.Assert.assertEquals;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.google.common.io.Closeables;
@ -38,40 +38,60 @@ import com.google.inject.Injector;
public class DeleteAllKeysInListTest {
private BlobStore blobstore;
private DeleteAllKeysInList deleter;
private static final String containerName = "container";
private static final String directoryName = "directory";
@BeforeClass
@BeforeMethod
void setupBlobStore() {
Injector injector = ContextBuilder.newBuilder("transient").buildInjector();
blobstore = injector.getInstance(BlobStore.class);
deleter = injector.getInstance(DeleteAllKeysInList.class);
createDataSet();
}
@AfterMethod
void close() {
Closeables.closeQuietly(blobstore.getContext());
}
public void testExecuteWithoutOptionsClearsRecursively() {
blobstore.createContainerInLocation(null, "goodies");
for (int i = 0; i < 1001; i++) {
blobstore.putBlob("goodies", blobstore.blobBuilder(i + "").payload(i + "").build());
}
assertEquals(blobstore.countBlobs("goodies"), 1001);
deleter.execute("goodies");
assertEquals(blobstore.countBlobs("goodies"), 0);
deleter.execute(containerName);
assertEquals(blobstore.countBlobs(containerName), 0);
}
public void testExecuteRecursive() {
deleter.execute(containerName, ListContainerOptions.Builder.recursive());
assertEquals(blobstore.countBlobs(containerName), 0);
}
public void testExecuteNonRecursive() {
blobstore.createContainerInLocation(null, "foo");
for (int i = 0; i < 1001; i++) {
blobstore.putBlob("foo", blobstore.blobBuilder(i + "").payload(i + "").build());
}
for (int i = 0; i < 1001; i++) {
blobstore.putBlob("foo", blobstore.blobBuilder("dir/" + i + "").payload(i + "").build());
}
assertEquals(blobstore.countBlobs("foo"), 2002);
deleter.execute("foo", ListContainerOptions.Builder.inDirectory("dir"));
assertEquals(blobstore.countBlobs("foo"), 1001);
deleter.execute(containerName, ListContainerOptions.NONE);
assertEquals(blobstore.countBlobs(containerName), 2222);
}
@AfterClass
void close() {
if (blobstore != null)
Closeables.closeQuietly(blobstore.getContext());
public void testExecuteInDirectory() {
deleter.execute(containerName, ListContainerOptions.Builder.inDirectory(directoryName));
assertEquals(blobstore.countBlobs(containerName), 1111);
}
/**
* Create a container "container" with 1111 blobs named "blob-%d". Create a
* subdirectory "directory" which contains 2222 more blobs named
* "directory/blob-%d".
*/
private void createDataSet() {
String blobNameFmt = "blob-%d";
String directoryBlobNameFmt = "%s/blob-%d";
blobstore.createContainerInLocation(null, containerName);
for (int i = 0; i < 1111; i++) {
String blobName = String.format(blobNameFmt, i);
blobstore.putBlob(containerName, blobstore.blobBuilder(blobName).payload(blobName).build());
}
for (int i = 0; i < 2222; i++) {
String directoryBlobName = String.format(directoryBlobNameFmt, directoryName, i);
blobstore.putBlob(containerName, blobstore.blobBuilder(directoryBlobName).payload(directoryBlobName).build());
}
assertEquals(blobstore.countBlobs(containerName), 3333);
}
}