HADOOP-13050. Upgrade to AWS SDK 10.11+. Contributed by Chris Nauroth and Steve Loughran
This commit is contained in:
parent
3a09e5970d
commit
af791b774b
|
@ -126,7 +126,7 @@
|
||||||
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
|
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
|
||||||
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
|
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
|
||||||
<surefire.fork.timeout>900</surefire.fork.timeout>
|
<surefire.fork.timeout>900</surefire.fork.timeout>
|
||||||
<aws-java-sdk.version>1.10.6</aws-java-sdk.version>
|
<aws-java-sdk.version>1.11.45</aws-java-sdk.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<dependencyManagement>
|
<dependencyManagement>
|
||||||
|
@ -875,6 +875,11 @@
|
||||||
<artifactId>jackson-jaxrs-json-provider</artifactId>
|
<artifactId>jackson-jaxrs-json-provider</artifactId>
|
||||||
<version>${jackson2.version}</version>
|
<version>${jackson2.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||||
|
<artifactId>jackson-dataformat-cbor</artifactId>
|
||||||
|
<version>${jackson2.version}</version>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.mockito</groupId>
|
<groupId>org.mockito</groupId>
|
||||||
<artifactId>mockito-all</artifactId>
|
<artifactId>mockito-all</artifactId>
|
||||||
|
|
|
@ -344,6 +344,10 @@
|
||||||
<groupId>com.fasterxml.jackson.core</groupId>
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
<artifactId>jackson-annotations</artifactId>
|
<artifactId>jackson-annotations</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||||
|
<artifactId>jackson-dataformat-cbor</artifactId>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>joda-time</groupId>
|
<groupId>joda-time</groupId>
|
||||||
<artifactId>joda-time</artifactId>
|
<artifactId>joda-time</artifactId>
|
||||||
|
|
|
@ -180,7 +180,7 @@ interface S3ClientFactory {
|
||||||
userAgent = userAgentPrefix + ", " + userAgent;
|
userAgent = userAgentPrefix + ", " + userAgent;
|
||||||
}
|
}
|
||||||
LOG.debug("Using User-Agent: {}", userAgent);
|
LOG.debug("Using User-Agent: {}", userAgent);
|
||||||
awsConf.setUserAgent(userAgent);
|
awsConf.setUserAgentPrefix(userAgent);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -225,7 +225,9 @@ interface S3ClientFactory {
|
||||||
final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
|
final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
|
||||||
if (pathStyleAccess) {
|
if (pathStyleAccess) {
|
||||||
LOG.debug("Enabling path style access!");
|
LOG.debug("Enabling path style access!");
|
||||||
s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
|
s3.setS3ClientOptions(S3ClientOptions.builder()
|
||||||
|
.setPathStyleAccess(true)
|
||||||
|
.build());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -110,7 +110,8 @@ public class ITestS3AAWSCredentialsProvider {
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
public GoodCredentialsProvider(URI name, Configuration conf) {
|
public GoodCredentialsProvider(URI name, Configuration conf) {
|
||||||
super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
|
super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
|
||||||
conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
|
conf.get(SECRET_KEY)),
|
||||||
|
InstanceProfileCredentialsProvider.getInstance());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -399,7 +399,8 @@ public class ITestS3AConfiguration {
|
||||||
assertNotNull(s3);
|
assertNotNull(s3);
|
||||||
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
|
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
|
||||||
"clientConfiguration");
|
"clientConfiguration");
|
||||||
assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
|
assertEquals("Hadoop " + VersionInfo.getVersion(),
|
||||||
|
awsConf.getUserAgentPrefix());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -413,7 +414,7 @@ public class ITestS3AConfiguration {
|
||||||
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
|
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
|
||||||
"clientConfiguration");
|
"clientConfiguration");
|
||||||
assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
|
assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
|
||||||
awsConf.getUserAgent());
|
awsConf.getUserAgentPrefix());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -117,6 +117,9 @@ public class ITestS3AMiniYarnCluster extends AbstractS3ATestBase {
|
||||||
Map<String, Integer> result = new HashMap<>();
|
Map<String, Integer> result = new HashMap<>();
|
||||||
for (String line : outputAsStr.split("\n")) {
|
for (String line : outputAsStr.split("\n")) {
|
||||||
String[] tokens = line.split("\t");
|
String[] tokens = line.split("\t");
|
||||||
|
assertTrue("Not enough tokens in in string \" "+ line
|
||||||
|
+ "\" from output \"" + outputAsStr + "\"",
|
||||||
|
tokens.length > 1);
|
||||||
result.put(tokens[0], Integer.parseInt(tokens[1]));
|
result.put(tokens[0], Integer.parseInt(tokens[1]));
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
|
|
Loading…
Reference in New Issue