HADOOP-13050. Upgrade to AWS SDK 10.11+. Contributed by Chris Nauroth and Steve Loughran

This commit is contained in:
Mingliang Liu 2016-11-21 17:29:18 -08:00
parent 3a09e5970d
commit af791b774b
6 changed files with 22 additions and 6 deletions

View File

@ -126,7 +126,7 @@
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version> <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version> <native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout> <surefire.fork.timeout>900</surefire.fork.timeout>
<aws-java-sdk.version>1.10.6</aws-java-sdk.version> <aws-java-sdk.version>1.11.45</aws-java-sdk.version>
</properties> </properties>
<dependencyManagement> <dependencyManagement>
@ -875,6 +875,11 @@
<artifactId>jackson-jaxrs-json-provider</artifactId> <artifactId>jackson-jaxrs-json-provider</artifactId>
<version>${jackson2.version}</version> <version>${jackson2.version}</version>
</dependency> </dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
<version>${jackson2.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.mockito</groupId> <groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId> <artifactId>mockito-all</artifactId>

View File

@ -344,6 +344,10 @@
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId> <artifactId>jackson-annotations</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
</dependency>
<dependency> <dependency>
<groupId>joda-time</groupId> <groupId>joda-time</groupId>
<artifactId>joda-time</artifactId> <artifactId>joda-time</artifactId>

View File

@ -180,7 +180,7 @@ interface S3ClientFactory {
userAgent = userAgentPrefix + ", " + userAgent; userAgent = userAgentPrefix + ", " + userAgent;
} }
LOG.debug("Using User-Agent: {}", userAgent); LOG.debug("Using User-Agent: {}", userAgent);
awsConf.setUserAgent(userAgent); awsConf.setUserAgentPrefix(userAgent);
} }
/** /**
@ -225,7 +225,9 @@ interface S3ClientFactory {
final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false); final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
if (pathStyleAccess) { if (pathStyleAccess) {
LOG.debug("Enabling path style access!"); LOG.debug("Enabling path style access!");
s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); s3.setS3ClientOptions(S3ClientOptions.builder()
.setPathStyleAccess(true)
.build());
} }
} }
} }

View File

@ -110,7 +110,8 @@ public class ITestS3AAWSCredentialsProvider {
@SuppressWarnings("unused") @SuppressWarnings("unused")
public GoodCredentialsProvider(URI name, Configuration conf) { public GoodCredentialsProvider(URI name, Configuration conf) {
super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY), super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider()); conf.get(SECRET_KEY)),
InstanceProfileCredentialsProvider.getInstance());
} }
} }

View File

@ -399,7 +399,8 @@ public class ITestS3AConfiguration {
assertNotNull(s3); assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration"); "clientConfiguration");
assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent()); assertEquals("Hadoop " + VersionInfo.getVersion(),
awsConf.getUserAgentPrefix());
} }
@Test @Test
@ -413,7 +414,7 @@ public class ITestS3AConfiguration {
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration"); "clientConfiguration");
assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(), assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
awsConf.getUserAgent()); awsConf.getUserAgentPrefix());
} }
@Test @Test

View File

@ -117,6 +117,9 @@ public class ITestS3AMiniYarnCluster extends AbstractS3ATestBase {
Map<String, Integer> result = new HashMap<>(); Map<String, Integer> result = new HashMap<>();
for (String line : outputAsStr.split("\n")) { for (String line : outputAsStr.split("\n")) {
String[] tokens = line.split("\t"); String[] tokens = line.split("\t");
assertTrue("Not enough tokens in in string \" "+ line
+ "\" from output \"" + outputAsStr + "\"",
tokens.length > 1);
result.put(tokens[0], Integer.parseInt(tokens[1])); result.put(tokens[0], Integer.parseInt(tokens[1]));
} }
return result; return result;