honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve S3 grouped deletes 

* Move the grouping to outside the main while(true) loop
This commit is contained in:
Holden Karau 2012-03-06 15:47:58 -08:00
parent b19f1a5b06
commit bb222f0f7f
1 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
View File

@ -98,18 +98,20 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
@Override
public void delete(BlobPath path) {
ObjectListing prevListing = null;
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
//we can do at most 1K objects per delete
int objectCount = 0;
//We don't know the bucket name until first object listing
DeleteObjectsRequest multiObjectDeleteRequest = null;
ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
while (true) {
ObjectListing list;
if (prevListing != null) {
list = client.listNextBatchOfObjects(prevListing);
} else {
list = client.listObjects(bucket, path.buildAsString("/") + "/");
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
}
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
//we can do at most 1K objects per delete
int objectCount = 0;
DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
for (S3ObjectSummary summary : list.getObjectSummaries()) {
objectCount++;
keys.add(new KeyVersion(summary.getKey()));
@ -122,16 +124,16 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
objectCount = 0;
}
}
if (objectCount > 0) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
}
if (list.isTruncated()) {
prevListing = list;
} else {
break;
}
}
if (objectCount > 0) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
}
}
@Override