Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-10568. Add s3 server-side encryption. Contributed by David S. Wang. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1592133 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers 2014-05-03 00:25:09 +00:00
parent d777cb8996
commit 1ac2ce067c
3 changed files with 20 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -439,6 +439,8 @@ Release 2.5.0 - UNRELEASED
HADOOP-10562. Namenode exits on exception without printing stack trace HADOOP-10562. Namenode exits on exception without printing stack trace
in AbstractDelegationTokenSecretManager. (Arpit Agarwal) in AbstractDelegationTokenSecretManager. (Arpit Agarwal)
HADOOP-10568. Add s3 server-side encryption. (David S. Wang via atm)
Release 2.4.1 - UNRELEASED Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

11
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
View File

@ -63,6 +63,8 @@ class Jets3tNativeFileSystemStore implements NativeFileSystemStore {
private boolean multipartEnabled; private boolean multipartEnabled;
private long multipartCopyBlockSize; private long multipartCopyBlockSize;
static final long MAX_PART_SIZE = (long)5 * 1024 * 1024 * 1024; static final long MAX_PART_SIZE = (long)5 * 1024 * 1024 * 1024;
private String serverSideEncryptionAlgorithm;
public static final Log LOG = public static final Log LOG =
LogFactory.getLog(Jets3tNativeFileSystemStore.class); LogFactory.getLog(Jets3tNativeFileSystemStore.class);
@ -87,6 +89,7 @@ class Jets3tNativeFileSystemStore implements NativeFileSystemStore {
multipartCopyBlockSize = Math.min( multipartCopyBlockSize = Math.min(
conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE), conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE),
MAX_PART_SIZE); MAX_PART_SIZE);
serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm");
bucket = new S3Bucket(uri.getHost()); bucket = new S3Bucket(uri.getHost());
} }
@ -107,6 +110,7 @@ class Jets3tNativeFileSystemStore implements NativeFileSystemStore {
object.setDataInputStream(in); object.setDataInputStream(in);
object.setContentType("binary/octet-stream"); object.setContentType("binary/octet-stream");
object.setContentLength(file.length()); object.setContentLength(file.length());
object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
if (md5Hash != null) { if (md5Hash != null) {
object.setMd5Hash(md5Hash); object.setMd5Hash(md5Hash);
} }
@ -130,6 +134,7 @@ class Jets3tNativeFileSystemStore implements NativeFileSystemStore {
object.setDataInputFile(file); object.setDataInputFile(file);
object.setContentType("binary/octet-stream"); object.setContentType("binary/octet-stream");
object.setContentLength(file.length()); object.setContentLength(file.length());
object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
if (md5Hash != null) { if (md5Hash != null) {
object.setMd5Hash(md5Hash); object.setMd5Hash(md5Hash);
} }
@ -156,6 +161,7 @@ class Jets3tNativeFileSystemStore implements NativeFileSystemStore {
object.setDataInputStream(new ByteArrayInputStream(new byte[0])); object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
object.setContentType("binary/octet-stream"); object.setContentType("binary/octet-stream");
object.setContentLength(0); object.setContentLength(0);
object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
s3Service.putObject(bucket, object); s3Service.putObject(bucket, object);
} catch (S3ServiceException e) { } catch (S3ServiceException e) {
handleS3ServiceException(e); handleS3ServiceException(e);
@ -317,8 +323,11 @@ class Jets3tNativeFileSystemStore implements NativeFileSystemStore {
return; return;
} }
} }
S3Object dstObject = new S3Object(dstKey);
dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(), s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
new S3Object(dstKey), false); dstObject, false);
} catch (ServiceException e) { } catch (ServiceException e) {
handleServiceException(srcKey, e); handleServiceException(srcKey, e);
} }

8
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -575,6 +575,14 @@
</description> </description>
</property> </property>
<property>
<name>fs.s3n.server-side-encryption-algorithm</name>
<value></value>
<description>Specify a server-side encryption algorithm for S3.
The default is NULL, and the only other currently allowable value is AES256.
</description>
</property>
<property> <property>
<name>io.seqfile.compress.blocksize</name> <name>io.seqfile.compress.blocksize</name>
<value>1000000</value> <value>1000000</value>