Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

safely move files

This commit is contained in:
Xavier Léauté 2013-12-16 13:54:30 -08:00
parent e333776aca
commit f3b8d9c047
1 changed files with 33 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java
View File

@ -69,26 +69,8 @@ public class S3DataSegmentMover implements DataSegmentMover
throw new SegmentLoadingException("Target S3 baseKey is not specified"); throw new SegmentLoadingException("Target S3 baseKey is not specified");
} }
if (s3Client.isObjectInBucket(s3Bucket, s3Path)) { safeMove(s3Bucket, s3Path, targetS3Bucket, targetS3Path);
log.info( safeMove(s3Bucket, s3DescriptorPath, targetS3Bucket, targetS3DescriptorPath);
"Moving index file[s3://%s/%s] to [s3://%s/%s]",
s3Bucket,
s3Path,
targetS3Bucket,
targetS3Path
);
s3Client.moveObject(s3Bucket, s3Path, targetS3Bucket, new S3Object(targetS3Path), false);
}
if (s3Client.isObjectInBucket(s3Bucket, s3DescriptorPath)) {
log.info(
"Moving descriptor file[s3://%s/%s] to [s3://%s/%s]",
s3Bucket,
s3DescriptorPath,
targetS3Bucket,
targetS3DescriptorPath
);
s3Client.moveObject(s3Bucket, s3DescriptorPath, targetS3Bucket, new S3Object(targetS3DescriptorPath), false);
}
return segment.withLoadSpec( return segment.withLoadSpec(
ImmutableMap.<String, Object>builder() ImmutableMap.<String, Object>builder()
@ -102,4 +84,35 @@ public class S3DataSegmentMover implements DataSegmentMover
throw new SegmentLoadingException(e, "Unable to move segment[%s]", segment.getIdentifier()); throw new SegmentLoadingException(e, "Unable to move segment[%s]", segment.getIdentifier());
} }
} }
private void safeMove(String s3Bucket, String s3Path, String targetS3Bucket, String targetS3Path)
throws ServiceException, SegmentLoadingException
{
if (s3Client.isObjectInBucket(s3Bucket, s3Path)) {
log.info(
"Moving file[s3://%s/%s] to [s3://%s/%s]",
s3Bucket,
s3Path,
targetS3Bucket,
targetS3Path
);
s3Client.moveObject(s3Bucket, s3Path, targetS3Bucket, new S3Object(targetS3Path), false);
} else {
// ensure object exists in target location
if(s3Client.isObjectInBucket(targetS3Bucket, targetS3Path)) {
log.info(
"Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]",
s3Bucket, s3Path,
targetS3Bucket, targetS3Path
);
}
else {
throw new SegmentLoadingException(
"Unable to move file [s3://%s/%s] to [s3://%s/%s], not present in either source or target location",
s3Bucket, s3Path,
targetS3Bucket, targetS3Path
);
}
}
}
} }