honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Group S3 delete requests. 

* In the new AWS SDK deleting multiple objects
  in a single request is now supported. See:
http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
This commit is contained in:
Holden Karau 2012-03-05 16:32:06 -08:00
parent d234b3cdc8
commit b19f1a5b06
1 changed files with 22 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
View File

@ -20,6 +20,8 @@
package org.elasticsearch.cloud.aws.blobstore; package org.elasticsearch.cloud.aws.blobstore;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
@ -31,6 +33,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.ByteSizeValue;
import java.util.ArrayList;
import java.util.concurrent.Executor; import java.util.concurrent.Executor;
/** /**
@ -102,8 +105,26 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
} else { } else {
list = client.listObjects(bucket, path.buildAsString("/") + "/"); list = client.listObjects(bucket, path.buildAsString("/") + "/");
} }
//From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
//we can do at most 1K objects per delete
int objectCount = 0;
DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
for (S3ObjectSummary summary : list.getObjectSummaries()) { for (S3ObjectSummary summary : list.getObjectSummaries()) {
client.deleteObject(summary.getBucketName(), summary.getKey()); objectCount++;
keys.add(new KeyVersion(summary.getKey()));
//Every 500 objects batch the delete request
if (objectCount > 500) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
keys.clear();
objectCount = 0;
}
}
if (objectCount > 0) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
} }
if (list.isTruncated()) { if (list.isTruncated()) {
prevListing = list; prevListing = list;