Improve S3 grouped deletes
* Move the grouping to outside the main while(true) loop
This commit is contained in:
parent
b19f1a5b06
commit
bb222f0f7f
|
@ -98,18 +98,20 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
|
||||||
@Override
|
@Override
|
||||||
public void delete(BlobPath path) {
|
public void delete(BlobPath path) {
|
||||||
ObjectListing prevListing = null;
|
ObjectListing prevListing = null;
|
||||||
|
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
|
||||||
|
//we can do at most 1K objects per delete
|
||||||
|
int objectCount = 0;
|
||||||
|
//We don't know the bucket name until first object listing
|
||||||
|
DeleteObjectsRequest multiObjectDeleteRequest = null;
|
||||||
|
ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
|
||||||
while (true) {
|
while (true) {
|
||||||
ObjectListing list;
|
ObjectListing list;
|
||||||
if (prevListing != null) {
|
if (prevListing != null) {
|
||||||
list = client.listNextBatchOfObjects(prevListing);
|
list = client.listNextBatchOfObjects(prevListing);
|
||||||
} else {
|
} else {
|
||||||
list = client.listObjects(bucket, path.buildAsString("/") + "/");
|
list = client.listObjects(bucket, path.buildAsString("/") + "/");
|
||||||
|
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
|
||||||
}
|
}
|
||||||
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
|
|
||||||
//we can do at most 1K objects per delete
|
|
||||||
int objectCount = 0;
|
|
||||||
DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
|
|
||||||
ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
|
|
||||||
for (S3ObjectSummary summary : list.getObjectSummaries()) {
|
for (S3ObjectSummary summary : list.getObjectSummaries()) {
|
||||||
objectCount++;
|
objectCount++;
|
||||||
keys.add(new KeyVersion(summary.getKey()));
|
keys.add(new KeyVersion(summary.getKey()));
|
||||||
|
@ -122,16 +124,16 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
|
||||||
objectCount = 0;
|
objectCount = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (objectCount > 0) {
|
|
||||||
multiObjectDeleteRequest.setKeys(keys);
|
|
||||||
client.deleteObjects(multiObjectDeleteRequest);
|
|
||||||
}
|
|
||||||
if (list.isTruncated()) {
|
if (list.isTruncated()) {
|
||||||
prevListing = list;
|
prevListing = list;
|
||||||
} else {
|
} else {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (objectCount > 0) {
|
||||||
|
multiObjectDeleteRequest.setKeys(keys);
|
||||||
|
client.deleteObjects(multiObjectDeleteRequest);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
Loading…
Reference in New Issue