diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java index 0ba756c7d4b..90b260e34fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java @@ -42,7 +42,8 @@ public class ConfigRedactor { String sensitiveRegexList = conf.get( HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS, HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT); - List sensitiveRegexes = Arrays.asList(sensitiveRegexList.split(",")); + List sensitiveRegexes = + Arrays.asList(sensitiveRegexList.trim().split("[,\\s]+")); compiledPatterns = new ArrayList(); for (String regex : sensitiveRegexes) { Pattern p = Pattern.compile(regex); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 86feee2cdc2..deb30365ec0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -18,10 +18,13 @@ package org.apache.hadoop.fs; +import java.util.Arrays; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.JceAesCtrCryptoCodec; import org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec; +import org.apache.hadoop.util.StringUtils; /** * This class contains constants for configuration keys used @@ -829,12 +832,14 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS = "hadoop.security.sensitive-config-keys"; public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT = - "secret$" + "," + - "password$" + "," + - "ssl.keystore.pass$" + "," + - "fs.s3.*[Ss]ecret.?[Kk]ey" + "," + - "fs.azure\\.account.key.*" + "," + - "dfs.webhdfs.oauth2.[a-z]+.token" + "," + - HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS; + StringUtils.join(",", Arrays.asList( + "secret$", + "password$", + "ssl.keystore.pass$", + "fs.s3.*[Ss]ecret.?[Kk]ey", + "fs.azure\\.account.key.*", + "credential$", + "oauth.*token$", + HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS)); } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 61c9df7da3e..685d5656adf 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -526,9 +526,18 @@ hadoop.security.sensitive-config-keys - secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys - A comma-separated list of regular expressions to match against - configuration keys that should be redacted where appropriate, for + + secret$ + password$ + ssl.keystore.pass$ + fs.s3.*[Ss]ecret.?[Kk]ey + fs.azure.account.key.* + credential$ + oauth.*token$ + hadoop.security.sensitive-config-keys + + A comma-separated or multi-line list of regular expressions to + match configuration keys that should be redacted where appropriate, for example, when logging modified properties during a reconfiguration, private credentials should not be logged. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java index eedb9b2b7cd..ba08de4bcc7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java @@ -34,15 +34,30 @@ public class TestConfigRedactor { private static final String ORIGINAL_VALUE = "Hello, World!"; @Test - public void redact() throws Exception { + public void testRedactWithCoreDefault() throws Exception { Configuration conf = new Configuration(); + testRedact(conf); + } + + @Test + public void testRedactNoCoreDefault() throws Exception { + Configuration conf = new Configuration(false); + testRedact(conf); + } + + private void testRedact(Configuration conf) throws Exception { ConfigRedactor redactor = new ConfigRedactor(conf); String processedText; List sensitiveKeys = Arrays.asList( "fs.s3a.secret.key", + "fs.s3a.bucket.BUCKET.secret.key", "fs.s3n.awsSecretKey", "fs.azure.account.key.abcdefg.blob.core.windows.net", + "fs.adl.oauth2.refresh.token", + "fs.adl.oauth2.credential", + "dfs.adls.oauth2.refresh.token", + "dfs.adls.oauth2.credential", "dfs.webhdfs.oauth2.access.token", "dfs.webhdfs.oauth2.refresh.token", "ssl.server.keystore.keypassword",