Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11080. Update HttpFS to use ConfigRedactor. Contributed by Sean Mackrory.

This commit is contained in:
Andrew Wang 2016-11-02 19:11:05 -07:00
parent 7a662bea89
commit 99d0b3eb2b
4 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
View File

@ -783,7 +783,9 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS = public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
"hadoop.security.sensitive-config-keys"; "hadoop.security.sensitive-config-keys";
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT = public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
"secret$" + "," +
"password$" + "," + "password$" + "," +
"ssl.keystore.pass$" + "," +
"fs.s3.*[Ss]ecret.?[Kk]ey" + "," + "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
"fs.azure\\.account.key.*" + "," + "fs.azure\\.account.key.*" + "," +
"dfs.webhdfs.oauth2.[a-z]+.token" + "," + "dfs.webhdfs.oauth2.[a-z]+.token" + "," +

2
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -513,7 +513,7 @@
<property> <property>
<name>hadoop.security.sensitive-config-keys</name> <name>hadoop.security.sensitive-config-keys</name>
<value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value> <value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
<description>A comma-separated list of regular expressions to match against <description>A comma-separated list of regular expressions to match against
configuration keys that should be redacted where appropriate, for configuration keys that should be redacted where appropriate, for
example, when logging modified properties during a reconfiguration, example, when logging modified properties during a reconfiguration,

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
View File

@ -47,6 +47,7 @@ public class TestConfigRedactor {
"dfs.webhdfs.oauth2.refresh.token", "dfs.webhdfs.oauth2.refresh.token",
"ssl.server.keystore.keypassword", "ssl.server.keystore.keypassword",
"ssl.server.keystore.password", "ssl.server.keystore.password",
"httpfs.ssl.keystore.pass",
"hadoop.security.sensitive-config-keys" "hadoop.security.sensitive-config-keys"
); );
for (String key : sensitiveKeys) { for (String key : sensitiveKeys) {
@ -60,6 +61,7 @@ public class TestConfigRedactor {
"fs.defaultFS", "fs.defaultFS",
"dfs.replication", "dfs.replication",
"ssl.server.keystore.location", "ssl.server.keystore.location",
"httpfs.config.dir",
"hadoop.security.credstore.java-keystore-provider.password-file" "hadoop.security.credstore.java-keystore-provider.password-file"
); );
for (String key : normalKeys) { for (String key : normalKeys) {

15
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.lib.server; package org.apache.hadoop.lib.server;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.ConfigRedactor;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.lib.util.Check; import org.apache.hadoop.lib.util.Check;
import org.apache.hadoop.lib.util.ConfigurationUtils; import org.apache.hadoop.lib.util.ConfigurationUtils;
@ -482,15 +483,13 @@ public class Server {
} }
ConfigurationUtils.injectDefaults(defaultConf, config); ConfigurationUtils.injectDefaults(defaultConf, config);
ConfigRedactor redactor = new ConfigRedactor(config);
for (String name : System.getProperties().stringPropertyNames()) { for (String name : System.getProperties().stringPropertyNames()) {
String value = System.getProperty(name); String value = System.getProperty(name);
if (name.startsWith(getPrefix() + ".")) { if (name.startsWith(getPrefix() + ".")) {
config.set(name, value); config.set(name, value);
if (name.endsWith(".password") || name.endsWith(".secret")) { String redacted = redactor.redact(name, value);
value = "*MASKED*"; log.info("System property sets {}: {}", name, redacted);
}
log.info("System property sets {}: {}", name, value);
} }
} }
@ -499,10 +498,8 @@ public class Server {
for (Map.Entry<String, String> entry : config) { for (Map.Entry<String, String> entry : config) {
String name = entry.getKey(); String name = entry.getKey();
String value = config.get(entry.getKey()); String value = config.get(entry.getKey());
if (name.endsWith(".password") || name.endsWith(".secret")) { String redacted = redactor.redact(name, value);
value = "*MASKED*"; log.debug(" {}: {}", entry.getKey(), redacted);
}
log.debug(" {}: {}", entry.getKey(), value);
} }
log.debug("------------------------------------------------------"); log.debug("------------------------------------------------------");
} }