Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12851. S3AFileSystem Uptake of ProviderUtils.excludeIncompatibleCredentialProviders. Contributed by Larry McCay. 

(cherry picked from commit d251e55415)
(cherry picked from commit e402371b6a)
This commit is contained in:
Chris Nauroth 2016-02-29 20:03:42 -08:00
parent 475a277e60
commit f1236c5d7c
3 changed files with 63 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -1061,6 +1061,10 @@ Release 2.8.0 - UNRELEASED
HADOOP-12813. Migrate TestRPC and related codes to rebase on
ProtobufRpcEngine. (Kai Zheng via wheat9)
HADOOP-12851. S3AFileSystem Uptake of
ProviderUtils.excludeIncompatibleCredentialProviders.
(Larry McCay via cnauroth)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

11
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -68,6 +68,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.util.Progressable;
import static org.apache.hadoop.fs.s3a.Constants.*;
@ -355,9 +356,11 @@ public class S3AFileSystem extends FileSystem {
accessKey = userInfo;
}
}
Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
conf, S3AFileSystem.class);
if (accessKey == null) {
try {
final char[] key = conf.getPassword(ACCESS_KEY);
final char[] key = c.getPassword(ACCESS_KEY);
if (key != null) {
accessKey = (new String(key)).trim();
}
@ -367,7 +370,7 @@ public class S3AFileSystem extends FileSystem {
}
if (secretKey == null) {
try {
final char[] pass = conf.getPassword(SECRET_KEY);
final char[] pass = c.getPassword(SECRET_KEY);
if (pass != null) {
secretKey = (new String(pass)).trim();
}
@ -1057,8 +1060,8 @@ public class S3AFileSystem extends FileSystem {
|| objects.getObjectSummaries().size() > 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("Found path as directory (with /): " +
objects.getCommonPrefixes().size() + "/" +
objects.getObjectSummaries().size());
objects.getCommonPrefixes().size() + "/" +
objects.getObjectSummaries().size());
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
LOG.debug("Summary: " + summary.getKey() + " " + summary.getSize());

34
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
View File

@ -29,6 +29,7 @@ import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@ -318,4 +319,37 @@ public class TestS3AConfiguration {
assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getAccessKey());
assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getAccessSecret());
}
@Test
public void testExcludingS3ACredentialProvider() throws Exception {
// set up conf to have a cred provider
final Configuration conf = new Configuration();
final File file = tempDir.newFile("test.jks");
final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
file.toURI());
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
"jceks://s3a/foobar," + jks.toString());
// first make sure that the s3a based provider is removed
Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
conf, S3AFileSystem.class);
String newPath = conf.get(
CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
assertFalse("Provider Path incorrect", newPath.contains("s3a://"));
// now let's make sure the new path is created by the S3AFileSystem
// and the integration still works. Let's provision the keys through
// the altered configuration instance and then try and access them
// using the original config with the s3a provider in the path.
provisionAccessKeys(c);
S3AFileSystem s3afs = new S3AFileSystem();
conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
URI uriWithUserInfo = new URI("s3a://123:456@foobar");
S3AFileSystem.AWSAccessKeys creds =
s3afs.getAWSAccessKeys(uriWithUserInfo, conf);
assertEquals("AccessKey incorrect.", "123", creds.getAccessKey());
assertEquals("SecretKey incorrect.", "456", creds.getAccessSecret());
}
}