Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-03-01 14:59:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add more logging around failed S3DataSegmentMover DeleteExceptions (#3104)

Browse Source 
* Add more logging around failed S3DataSegmentMover DeleteExceptions

* Fix test NPE
This commit is contained in:
Charles Allen 2016-06-16 14:58:33 -07:00 committed by Fangjin Yang
parent 6c2fd75e46
commit 674f94083e
1 changed files with 20 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java
View File

@ -26,6 +26,7 @@ import com.google.common.collect.Maps;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.metamx.common.ISE; import com.metamx.common.ISE;
import com.metamx.common.MapUtils; import com.metamx.common.MapUtils;
import com.metamx.common.StringUtils;
import com.metamx.common.logger.Logger; import com.metamx.common.logger.Logger;
import io.druid.segment.loading.DataSegmentMover; import io.druid.segment.loading.DataSegmentMover;
import io.druid.segment.loading.SegmentLoadingException; import io.druid.segment.loading.SegmentLoadingException;
@ -133,18 +134,33 @@ public class S3DataSegmentMover implements DataSegmentMover
s3Object.getStorageClass().equals(S3Object.STORAGE_CLASS_GLACIER)) { s3Object.getStorageClass().equals(S3Object.STORAGE_CLASS_GLACIER)) {
log.warn("Cannot move file[s3://%s/%s] of storage class glacier, skipping.", s3Bucket, s3Path); log.warn("Cannot move file[s3://%s/%s] of storage class glacier, skipping.", s3Bucket, s3Path);
} else { } else {
log.info( final String copyMsg = StringUtils.safeFormat(
"Moving file[s3://%s/%s] to [s3://%s/%s]", "[s3://%s/%s] to [s3://%s/%s]", s3Bucket,
s3Bucket,
s3Path, s3Path,
targetS3Bucket, targetS3Bucket,
targetS3Path targetS3Path
); );
log.info(
"Moving file %s",
copyMsg
);
final S3Object target = new S3Object(targetS3Path); final S3Object target = new S3Object(targetS3Path);
if (!config.getDisableAcl()) { if (!config.getDisableAcl()) {
target.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); target.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL);
} }
s3Client.moveObject(s3Bucket, s3Path, targetS3Bucket, target, false); final Map<String, Object> copyResult = s3Client.moveObject(
s3Bucket,
s3Path,
targetS3Bucket,
target,
false
);
if (copyResult != null && copyResult.containsKey("DeleteException")) {
log.error("Error Deleting data after copy %s: %s", copyMsg, copyResult);
// Maybe retry deleting here?
} else {
log.debug("Finished moving file %s", copyMsg);
}
} }
} else { } else {
// ensure object exists in target location // ensure object exists in target location