HADOOP-14241. Add ADLS sensitive config keys to default list. Contributed by John Zhuge.
This commit is contained in:
parent
dd43b895c2
commit
0344bea3fd
|
@ -42,7 +42,8 @@ public class ConfigRedactor {
|
|||
String sensitiveRegexList = conf.get(
|
||||
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS,
|
||||
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT);
|
||||
List<String> sensitiveRegexes = Arrays.asList(sensitiveRegexList.split(","));
|
||||
List<String> sensitiveRegexes =
|
||||
Arrays.asList(sensitiveRegexList.trim().split("[,\\s]+"));
|
||||
compiledPatterns = new ArrayList<Pattern>();
|
||||
for (String regex : sensitiveRegexes) {
|
||||
Pattern p = Pattern.compile(regex);
|
||||
|
|
|
@ -810,12 +810,14 @@ public class CommonConfigurationKeysPublic {
|
|||
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
|
||||
"hadoop.security.sensitive-config-keys";
|
||||
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
|
||||
"secret$" + "," +
|
||||
"password$" + "," +
|
||||
"ssl.keystore.pass$" + "," +
|
||||
"fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
|
||||
"fs.azure\\.account.key.*" + "," +
|
||||
"dfs.webhdfs.oauth2.[a-z]+.token" + "," +
|
||||
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
|
||||
String.join(",",
|
||||
"secret$",
|
||||
"password$",
|
||||
"ssl.keystore.pass$",
|
||||
"fs.s3.*[Ss]ecret.?[Kk]ey",
|
||||
"fs.azure\\.account.key.*",
|
||||
"credential$",
|
||||
"oauth.*token$",
|
||||
HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS);
|
||||
}
|
||||
|
||||
|
|
|
@ -517,9 +517,18 @@
|
|||
|
||||
<property>
|
||||
<name>hadoop.security.sensitive-config-keys</name>
|
||||
<value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
|
||||
<description>A comma-separated list of regular expressions to match against
|
||||
configuration keys that should be redacted where appropriate, for
|
||||
<value>
|
||||
secret$
|
||||
password$
|
||||
ssl.keystore.pass$
|
||||
fs.s3.*[Ss]ecret.?[Kk]ey
|
||||
fs.azure.account.key.*
|
||||
credential$
|
||||
oauth.*token$
|
||||
hadoop.security.sensitive-config-keys
|
||||
</value>
|
||||
<description>A comma-separated or multi-line list of regular expressions to
|
||||
match configuration keys that should be redacted where appropriate, for
|
||||
example, when logging modified properties during a reconfiguration,
|
||||
private credentials should not be logged.
|
||||
</description>
|
||||
|
|
|
@ -34,15 +34,30 @@ public class TestConfigRedactor {
|
|||
private static final String ORIGINAL_VALUE = "Hello, World!";
|
||||
|
||||
@Test
|
||||
public void redact() throws Exception {
|
||||
public void testRedactWithCoreDefault() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
testRedact(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRedactNoCoreDefault() throws Exception {
|
||||
Configuration conf = new Configuration(false);
|
||||
testRedact(conf);
|
||||
}
|
||||
|
||||
private void testRedact(Configuration conf) throws Exception {
|
||||
ConfigRedactor redactor = new ConfigRedactor(conf);
|
||||
String processedText;
|
||||
|
||||
List<String> sensitiveKeys = Arrays.asList(
|
||||
"fs.s3a.secret.key",
|
||||
"fs.s3a.bucket.BUCKET.secret.key",
|
||||
"fs.s3n.awsSecretKey",
|
||||
"fs.azure.account.key.abcdefg.blob.core.windows.net",
|
||||
"fs.adl.oauth2.refresh.token",
|
||||
"fs.adl.oauth2.credential",
|
||||
"dfs.adls.oauth2.refresh.token",
|
||||
"dfs.adls.oauth2.credential",
|
||||
"dfs.webhdfs.oauth2.access.token",
|
||||
"dfs.webhdfs.oauth2.refresh.token",
|
||||
"ssl.server.keystore.keypassword",
|
||||
|
|
Loading…
Reference in New Issue