Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-14241. Add ADLS sensitive config keys to default list. Contributed by John Zhuge. 

(cherry picked from commit 0344bea3fd)
This commit is contained in:
John Zhuge 2017-04-19 12:04:03 -07:00
parent e278eb3ab5
commit 3a39dd5391
4 changed files with 39 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
View File

@ -42,7 +42,8 @@ public class ConfigRedactor {
String sensitiveRegexList = conf.get( String sensitiveRegexList = conf.get(
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS, HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS,
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT); HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT);
List<String> sensitiveRegexes = Arrays.asList(sensitiveRegexList.split(",")); List<String> sensitiveRegexes =
Arrays.asList(sensitiveRegexList.trim().split("[,\\s]+"));
compiledPatterns = new ArrayList<Pattern>(); compiledPatterns = new ArrayList<Pattern>();
for (String regex : sensitiveRegexes) { for (String regex : sensitiveRegexes) {
Pattern p = Pattern.compile(regex); Pattern p = Pattern.compile(regex);

16
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
View File

@ -829,12 +829,14 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS = public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
"hadoop.security.sensitive-config-keys"; "hadoop.security.sensitive-config-keys";
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT = public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
"secret$" + "," + String.join(",",
"password$" + "," + "secret$",
"ssl.keystore.pass$" + "," + "password$",
"fs.s3.*[Ss]ecret.?[Kk]ey" + "," + "ssl.keystore.pass$",
"fs.azure\\.account.key.*" + "," + "fs.s3.*[Ss]ecret.?[Kk]ey",
"dfs.webhdfs.oauth2.[a-z]+.token" + "," + "fs.azure\\.account.key.*",
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS; "credential$",
"oauth.*token$",
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS);
} }

15
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -526,9 +526,18 @@
<property> <property>
<name>hadoop.security.sensitive-config-keys</name> <name>hadoop.security.sensitive-config-keys</name>
<value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value> <value>
<description>A comma-separated list of regular expressions to match against secret$
configuration keys that should be redacted where appropriate, for password$
ssl.keystore.pass$
fs.s3.*[Ss]ecret.?[Kk]ey
fs.azure.account.key.*
credential$
oauth.*token$
hadoop.security.sensitive-config-keys
</value>
<description>A comma-separated or multi-line list of regular expressions to
match configuration keys that should be redacted where appropriate, for
example, when logging modified properties during a reconfiguration, example, when logging modified properties during a reconfiguration,
private credentials should not be logged. private credentials should not be logged.
</description> </description>

17
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
View File

@ -34,15 +34,30 @@ public class TestConfigRedactor {
private static final String ORIGINAL_VALUE = "Hello, World!"; private static final String ORIGINAL_VALUE = "Hello, World!";
@Test @Test
public void redact() throws Exception { public void testRedactWithCoreDefault() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
testRedact(conf);
}
@Test
public void testRedactNoCoreDefault() throws Exception {
Configuration conf = new Configuration(false);
testRedact(conf);
}
private void testRedact(Configuration conf) throws Exception {
ConfigRedactor redactor = new ConfigRedactor(conf); ConfigRedactor redactor = new ConfigRedactor(conf);
String processedText; String processedText;
List<String> sensitiveKeys = Arrays.asList( List<String> sensitiveKeys = Arrays.asList(
"fs.s3a.secret.key", "fs.s3a.secret.key",
"fs.s3a.bucket.BUCKET.secret.key",
"fs.s3n.awsSecretKey", "fs.s3n.awsSecretKey",
"fs.azure.account.key.abcdefg.blob.core.windows.net", "fs.azure.account.key.abcdefg.blob.core.windows.net",
"fs.adl.oauth2.refresh.token",
"fs.adl.oauth2.credential",
"dfs.adls.oauth2.refresh.token",
"dfs.adls.oauth2.credential",
"dfs.webhdfs.oauth2.access.token", "dfs.webhdfs.oauth2.access.token",
"dfs.webhdfs.oauth2.refresh.token", "dfs.webhdfs.oauth2.refresh.token",
"ssl.server.keystore.keypassword", "ssl.server.keystore.keypassword",