HADOOP-13050. Upgrade to AWS SDK 10.11+. Contributed by Chris Nauroth and Steve Loughran

This commit is contained in:
Mingliang Liu 2016-11-21 17:29:18 -08:00
parent 3a09e5970d
commit af791b774b
6 changed files with 22 additions and 6 deletions

View File

@ -126,7 +126,7 @@
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
<aws-java-sdk.version>1.10.6</aws-java-sdk.version>
<aws-java-sdk.version>1.11.45</aws-java-sdk.version>
</properties>
<dependencyManagement>
@ -875,6 +875,11 @@
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>${jackson2.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
<version>${jackson2.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>

View File

@ -344,6 +344,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>

View File

@ -180,7 +180,7 @@ interface S3ClientFactory {
userAgent = userAgentPrefix + ", " + userAgent;
}
LOG.debug("Using User-Agent: {}", userAgent);
awsConf.setUserAgent(userAgent);
awsConf.setUserAgentPrefix(userAgent);
}
/**
@ -225,7 +225,9 @@ interface S3ClientFactory {
final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
if (pathStyleAccess) {
LOG.debug("Enabling path style access!");
s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
s3.setS3ClientOptions(S3ClientOptions.builder()
.setPathStyleAccess(true)
.build());
}
}
}

View File

@ -110,7 +110,8 @@ public class ITestS3AAWSCredentialsProvider {
@SuppressWarnings("unused")
public GoodCredentialsProvider(URI name, Configuration conf) {
super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
conf.get(SECRET_KEY)),
InstanceProfileCredentialsProvider.getInstance());
}
}

View File

@ -399,7 +399,8 @@ public class ITestS3AConfiguration {
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
assertEquals("Hadoop " + VersionInfo.getVersion(),
awsConf.getUserAgentPrefix());
}
@Test
@ -413,7 +414,7 @@ public class ITestS3AConfiguration {
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
awsConf.getUserAgent());
awsConf.getUserAgentPrefix());
}
@Test

View File

@ -117,6 +117,9 @@ public class ITestS3AMiniYarnCluster extends AbstractS3ATestBase {
Map<String, Integer> result = new HashMap<>();
for (String line : outputAsStr.split("\n")) {
String[] tokens = line.split("\t");
assertTrue("Not enough tokens in in string \" "+ line
+ "\" from output \"" + outputAsStr + "\"",
tokens.length > 1);
result.put(tokens[0], Integer.parseInt(tokens[1]));
}
return result;