Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12269. Update aws-sdk dependency to 1.10.6 (Thomas Demoor via Lei (Eddy) Xu)

This commit is contained in:
Lei Xu 2015-08-04 18:51:52 -07:00
parent c95993cbaf
commit d5403747b5
9 changed files with 49 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -865,6 +865,12 @@ for ldap providers in the same way as above does.
<description>Minimum age in seconds of multipart uploads to purge</description> <description>Minimum age in seconds of multipart uploads to purge</description>
</property> </property>
<property>
<name>fs.s3a.signing-algorithm</name>
<description>Override the default signing algorithm so legacy
implementations can still be used</description>
</property>
<property> <property>
<name>fs.s3a.buffer.dir</name> <name>fs.s3a.buffer.dir</name>
<value>${hadoop.tmp.dir}/s3a</value> <value>${hadoop.tmp.dir}/s3a</value>

4
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
View File

@ -132,7 +132,7 @@
</exclusion> </exclusion>
<exclusion> <exclusion>
<groupId>com.amazonaws</groupId> <groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId> <artifactId>aws-java-sdk-s3</artifactId>
</exclusion> </exclusion>
<exclusion> <exclusion>
<groupId>org.eclipse.jdt</groupId> <groupId>org.eclipse.jdt</groupId>
@ -175,7 +175,7 @@
</exclusion> </exclusion>
<exclusion> <exclusion>
<groupId>com.amazonaws</groupId> <groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId> <artifactId>aws-java-sdk-s3</artifactId>
</exclusion> </exclusion>
<exclusion> <exclusion>
<groupId>org.eclipse.jdt</groupId> <groupId>org.eclipse.jdt</groupId>

4
hadoop-project/pom.xml
View File

@ -657,8 +657,8 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.amazonaws</groupId> <groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId> <artifactId>aws-java-sdk-s3</artifactId>
<version>1.7.4</version> <version>1.10.6</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.mina</groupId> <groupId>org.apache.mina</groupId>

2
hadoop-tools/hadoop-aws/pom.xml
View File

@ -119,7 +119,7 @@
<dependency> <dependency>
<groupId>com.amazonaws</groupId> <groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId> <artifactId>aws-java-sdk-s3</artifactId>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>

5
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -82,7 +82,7 @@ public class Constants {
// minimum size in bytes before we start a multipart uploads or copy // minimum size in bytes before we start a multipart uploads or copy
public static final String MIN_MULTIPART_THRESHOLD = "fs.s3a.multipart.threshold"; public static final String MIN_MULTIPART_THRESHOLD = "fs.s3a.multipart.threshold";
public static final int DEFAULT_MIN_MULTIPART_THRESHOLD = Integer.MAX_VALUE; public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = Integer.MAX_VALUE;
// comma separated list of directories // comma separated list of directories
public static final String BUFFER_DIR = "fs.s3a.buffer.dir"; public static final String BUFFER_DIR = "fs.s3a.buffer.dir";
@ -112,6 +112,9 @@ public class Constants {
public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM = public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM =
"fs.s3a.server-side-encryption-algorithm"; "fs.s3a.server-side-encryption-algorithm";
//override signature algorithm used for signing requests
public static final String SIGNING_ALGORITHM = "fs.s3a.signing-algorithm";
public static final String S3N_FOLDER_SUFFIX = "_$folder$"; public static final String S3N_FOLDER_SUFFIX = "_$folder$";
public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size"; public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size";
public static final String FS_S3A = "s3a"; public static final String FS_S3A = "s3a";

2
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
View File

@ -267,7 +267,7 @@ public class S3AFastOutputStream extends OutputStream {
private ObjectMetadata createDefaultMetadata() { private ObjectMetadata createDefaultMetadata() {
ObjectMetadata om = new ObjectMetadata(); ObjectMetadata om = new ObjectMetadata();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm); om.setSSEAlgorithm(serverSideEncryptionAlgorithm);
} }
return om; return om;
} }

26
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -87,7 +87,7 @@ public class S3AFileSystem extends FileSystem {
private long partSize; private long partSize;
private TransferManager transfers; private TransferManager transfers;
private ThreadPoolExecutor threadPoolExecutor; private ThreadPoolExecutor threadPoolExecutor;
private int multiPartThreshold; private long multiPartThreshold;
public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class); public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
private CannedAccessControlList cannedACL; private CannedAccessControlList cannedACL;
private String serverSideEncryptionAlgorithm; private String serverSideEncryptionAlgorithm;
@ -191,8 +191,12 @@ public class S3AFileSystem extends FileSystem {
DEFAULT_ESTABLISH_TIMEOUT)); DEFAULT_ESTABLISH_TIMEOUT));
awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT,
DEFAULT_SOCKET_TIMEOUT)); DEFAULT_SOCKET_TIMEOUT));
String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
if(!signerOverride.isEmpty()) {
awsConf.setSignerOverride(signerOverride);
}
String proxyHost = conf.getTrimmed(PROXY_HOST,""); String proxyHost = conf.getTrimmed(PROXY_HOST, "");
int proxyPort = conf.getInt(PROXY_PORT, -1); int proxyPort = conf.getInt(PROXY_PORT, -1);
if (!proxyHost.isEmpty()) { if (!proxyHost.isEmpty()) {
awsConf.setProxyHost(proxyHost); awsConf.setProxyHost(proxyHost);
@ -246,7 +250,7 @@ public class S3AFileSystem extends FileSystem {
maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS); maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE); partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD, multiPartThreshold = conf.getLong(MIN_MULTIPART_THRESHOLD,
DEFAULT_MIN_MULTIPART_THRESHOLD); DEFAULT_MIN_MULTIPART_THRESHOLD);
if (partSize < 5 * 1024 * 1024) { if (partSize < 5 * 1024 * 1024) {
@ -403,7 +407,7 @@ public class S3AFileSystem extends FileSystem {
if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) { if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) {
return new FSDataOutputStream(new S3AFastOutputStream(s3, this, bucket, return new FSDataOutputStream(new S3AFastOutputStream(s3, this, bucket,
key, progress, statistics, cannedACL, key, progress, statistics, cannedACL,
serverSideEncryptionAlgorithm, partSize, (long)multiPartThreshold, serverSideEncryptionAlgorithm, partSize, multiPartThreshold,
threadPoolExecutor), statistics); threadPoolExecutor), statistics);
} }
// We pass null to FSDataOutputStream so it won't count writes that are being buffered to a file // We pass null to FSDataOutputStream so it won't count writes that are being buffered to a file
@ -1027,7 +1031,7 @@ public class S3AFileSystem extends FileSystem {
final ObjectMetadata om = new ObjectMetadata(); final ObjectMetadata om = new ObjectMetadata();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm); om.setSSEAlgorithm(serverSideEncryptionAlgorithm);
} }
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile); PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
putObjectRequest.setCannedAcl(cannedACL); putObjectRequest.setCannedAcl(cannedACL);
@ -1035,8 +1039,8 @@ public class S3AFileSystem extends FileSystem {
ProgressListener progressListener = new ProgressListener() { ProgressListener progressListener = new ProgressListener() {
public void progressChanged(ProgressEvent progressEvent) { public void progressChanged(ProgressEvent progressEvent) {
switch (progressEvent.getEventCode()) { switch (progressEvent.getEventType()) {
case ProgressEvent.PART_COMPLETED_EVENT_CODE: case TRANSFER_PART_COMPLETED_EVENT:
statistics.incrementWriteOps(1); statistics.incrementWriteOps(1);
break; break;
default: default:
@ -1091,7 +1095,7 @@ public class S3AFileSystem extends FileSystem {
ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey); ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
final ObjectMetadata dstom = srcom.clone(); final ObjectMetadata dstom = srcom.clone();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
dstom.setServerSideEncryption(serverSideEncryptionAlgorithm); dstom.setSSEAlgorithm(serverSideEncryptionAlgorithm);
} }
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey); CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
copyObjectRequest.setCannedAccessControlList(cannedACL); copyObjectRequest.setCannedAccessControlList(cannedACL);
@ -1099,8 +1103,8 @@ public class S3AFileSystem extends FileSystem {
ProgressListener progressListener = new ProgressListener() { ProgressListener progressListener = new ProgressListener() {
public void progressChanged(ProgressEvent progressEvent) { public void progressChanged(ProgressEvent progressEvent) {
switch (progressEvent.getEventCode()) { switch (progressEvent.getEventType()) {
case ProgressEvent.PART_COMPLETED_EVENT_CODE: case TRANSFER_PART_COMPLETED_EVENT:
statistics.incrementWriteOps(1); statistics.incrementWriteOps(1);
break; break;
default: default:
@ -1187,7 +1191,7 @@ public class S3AFileSystem extends FileSystem {
final ObjectMetadata om = new ObjectMetadata(); final ObjectMetadata om = new ObjectMetadata();
om.setContentLength(0L); om.setContentLength(0L);
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm); om.setSSEAlgorithm(serverSideEncryptionAlgorithm);
} }
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, im, om); PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, im, om);
putObjectRequest.setCannedAcl(cannedACL); putObjectRequest.setCannedAcl(cannedACL);

15
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOutputStream.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a; package org.apache.hadoop.fs.s3a;
import com.amazonaws.event.ProgressEvent; import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressEventType;
import com.amazonaws.event.ProgressListener; import com.amazonaws.event.ProgressListener;
import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.CannedAccessControlList;
@ -41,6 +42,8 @@ import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import static com.amazonaws.event.ProgressEventType.TRANSFER_COMPLETED_EVENT;
import static com.amazonaws.event.ProgressEventType.TRANSFER_PART_STARTED_EVENT;
import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.Constants.*;
public class S3AOutputStream extends OutputStream { public class S3AOutputStream extends OutputStream {
@ -52,7 +55,7 @@ public class S3AOutputStream extends OutputStream {
private TransferManager transfers; private TransferManager transfers;
private Progressable progress; private Progressable progress;
private long partSize; private long partSize;
private int partSizeThreshold; private long partSizeThreshold;
private S3AFileSystem fs; private S3AFileSystem fs;
private CannedAccessControlList cannedACL; private CannedAccessControlList cannedACL;
private FileSystem.Statistics statistics; private FileSystem.Statistics statistics;
@ -76,7 +79,8 @@ public class S3AOutputStream extends OutputStream {
this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm; this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm;
partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE); partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
partSizeThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD); partSizeThreshold = conf.getLong(MIN_MULTIPART_THRESHOLD,
DEFAULT_MIN_MULTIPART_THRESHOLD);
if (conf.get(BUFFER_DIR, null) != null) { if (conf.get(BUFFER_DIR, null) != null) {
lDirAlloc = new LocalDirAllocator(BUFFER_DIR); lDirAlloc = new LocalDirAllocator(BUFFER_DIR);
@ -116,7 +120,7 @@ public class S3AOutputStream extends OutputStream {
try { try {
final ObjectMetadata om = new ObjectMetadata(); final ObjectMetadata om = new ObjectMetadata();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm); om.setSSEAlgorithm(serverSideEncryptionAlgorithm);
} }
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile); PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile);
putObjectRequest.setCannedAcl(cannedACL); putObjectRequest.setCannedAcl(cannedACL);
@ -184,8 +188,9 @@ public class S3AOutputStream extends OutputStream {
} }
// There are 3 http ops here, but this should be close enough for now // There are 3 http ops here, but this should be close enough for now
if (progressEvent.getEventCode() == ProgressEvent.PART_STARTED_EVENT_CODE || ProgressEventType pet = progressEvent.getEventType();
progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) { if (pet == TRANSFER_PART_STARTED_EVENT ||
pet == TRANSFER_COMPLETED_EVENT) {
statistics.incrementWriteOps(1); statistics.incrementWriteOps(1);
} }

6
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
View File

@ -288,6 +288,12 @@ If you do any of these: change your credentials immediately!
<description>Minimum age in seconds of multipart uploads to purge</description> <description>Minimum age in seconds of multipart uploads to purge</description>
</property> </property>
<property>
<name>fs.s3a.signing-algorithm</name>
<description>Override the default signing algorithm so legacy
implementations can still be used</description>
</property>
<property> <property>
<name>fs.s3a.buffer.dir</name> <name>fs.s3a.buffer.dir</name>
<value>${hadoop.tmp.dir}/s3a</value> <value>${hadoop.tmp.dir}/s3a</value>