Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11080. Update HttpFS to use ConfigRedactor. Contributed by Sean Mackrory.

This commit is contained in:
Andrew Wang 2016-11-02 19:11:05 -07:00
parent 8602e2cb0f
commit e393dd788e
4 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
View File

@ -731,7 +731,9 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS = public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
"hadoop.security.sensitive-config-keys"; "hadoop.security.sensitive-config-keys";
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT = public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
"secret$" + "," +
"password$" + "," + "password$" + "," +
"ssl.keystore.pass$" + "," +
"fs.s3.*[Ss]ecret.?[Kk]ey" + "," + "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
"fs.azure\\.account.key.*" + "," + "fs.azure\\.account.key.*" + "," +
"dfs.webhdfs.oauth2.[a-z]+.token" + "," + "dfs.webhdfs.oauth2.[a-z]+.token" + "," +

2
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -432,7 +432,7 @@
<property> <property>
<name>hadoop.security.sensitive-config-keys</name> <name>hadoop.security.sensitive-config-keys</name>
<value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value> <value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
<description>A comma-separated list of regular expressions to match against <description>A comma-separated list of regular expressions to match against
configuration keys that should be redacted where appropriate, for configuration keys that should be redacted where appropriate, for
example, when logging modified properties during a reconfiguration, example, when logging modified properties during a reconfiguration,

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
View File

@ -47,6 +47,7 @@ public void redact() throws Exception {
"dfs.webhdfs.oauth2.refresh.token", "dfs.webhdfs.oauth2.refresh.token",
"ssl.server.keystore.keypassword", "ssl.server.keystore.keypassword",
"ssl.server.keystore.password", "ssl.server.keystore.password",
"httpfs.ssl.keystore.pass",
"hadoop.security.sensitive-config-keys" "hadoop.security.sensitive-config-keys"
); );
for (String key : sensitiveKeys) { for (String key : sensitiveKeys) {
@ -60,6 +61,7 @@ public void redact() throws Exception {
"fs.defaultFS", "fs.defaultFS",
"dfs.replication", "dfs.replication",
"ssl.server.keystore.location", "ssl.server.keystore.location",
"httpfs.config.dir",
"hadoop.security.credstore.java-keystore-provider.password-file" "hadoop.security.credstore.java-keystore-provider.password-file"
); );
for (String key : normalKeys) { for (String key : normalKeys) {

15
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.lib.server; package org.apache.hadoop.lib.server;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.ConfigRedactor;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.lib.util.Check; import org.apache.hadoop.lib.util.Check;
import org.apache.hadoop.lib.util.ConfigurationUtils; import org.apache.hadoop.lib.util.ConfigurationUtils;
@ -482,15 +483,13 @@ protected void initConfig() throws ServerException {
} }
ConfigurationUtils.injectDefaults(defaultConf, config); ConfigurationUtils.injectDefaults(defaultConf, config);
ConfigRedactor redactor = new ConfigRedactor(config);
for (String name : System.getProperties().stringPropertyNames()) { for (String name : System.getProperties().stringPropertyNames()) {
String value = System.getProperty(name); String value = System.getProperty(name);
if (name.startsWith(getPrefix() + ".")) { if (name.startsWith(getPrefix() + ".")) {
config.set(name, value); config.set(name, value);
if (name.endsWith(".password") || name.endsWith(".secret")) { String redacted = redactor.redact(name, value);
value = "*MASKED*"; log.info("System property sets {}: {}", name, redacted);
}
log.info("System property sets {}: {}", name, value);
} }
} }
@ -499,10 +498,8 @@ protected void initConfig() throws ServerException {
for (Map.Entry<String, String> entry : config) { for (Map.Entry<String, String> entry : config) {
String name = entry.getKey(); String name = entry.getKey();
String value = config.get(entry.getKey()); String value = config.get(entry.getKey());
if (name.endsWith(".password") || name.endsWith(".secret")) { String redacted = redactor.redact(name, value);
value = "*MASKED*"; log.debug(" {}: {}", entry.getKey(), redacted);
}
log.debug(" {}: {}", entry.getKey(), value);
} }
log.debug("------------------------------------------------------"); log.debug("------------------------------------------------------");
} }