Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-14260. Configuration.dumpConfiguration should redact sensitive information. Contributed by John Zhuge. 

(cherry picked from commit 582648befa)
This commit is contained in:
John Zhuge 2017-08-11 10:54:41 -07:00
parent 76c4aee2e3
commit e208a4e7ee
2 changed files with 53 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -3062,7 +3062,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("property");
appendJSONProperty(dumpGenerator, config, propertyName);
appendJSONProperty(dumpGenerator, config, propertyName,
new ConfigRedactor(config));
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
@ -3102,11 +3103,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
dumpGenerator.flush();
ConfigRedactor redactor = new ConfigRedactor(config);
synchronized (config) {
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
appendJSONProperty(dumpGenerator,
config,
item.getKey().toString());
appendJSONProperty(dumpGenerator, config, item.getKey().toString(),
redactor);
}
}
dumpGenerator.writeEndArray();
@ -3124,12 +3125,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @throws IOException
*/
private static void appendJSONProperty(JsonGenerator jsonGen,
Configuration config, String name) throws IOException {
Configuration config, String name, ConfigRedactor redactor)
throws IOException {
// skip writing if given property name is empty or null
if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
jsonGen.writeStartObject();
jsonGen.writeStringField("key", name);
jsonGen.writeStringField("value", config.get(name));
jsonGen.writeStringField("value",
redactor.redact(name, config.get(name)));
jsonGen.writeBooleanField("isFinal",
config.finalParameters.contains(name));
String[] resources = config.updatingResource.get(name);

48
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -48,6 +48,7 @@ import static org.junit.Assert.assertArrayEquals;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@ -83,6 +84,11 @@ public class TestConfiguration extends TestCase {
/** Four apostrophes. */
public static final String ESCAPED = "&apos;&#39;&#0039;&#x27;";
private static final String SENSITIVE_CONFIG_KEYS =
CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
private BufferedWriter out;
@Override
protected void setUp() throws Exception {
super.setUp();
@ -91,6 +97,9 @@ public class TestConfiguration extends TestCase {
@Override
protected void tearDown() throws Exception {
if(out != null) {
out.close();
}
super.tearDown();
new File(CONFIG).delete();
new File(CONFIG2).delete();
@ -791,8 +800,6 @@ public class TestConfiguration extends TestCase {
new File(new File(relConfig).getParent()).delete();
}
BufferedWriter out;
public void testIntegerRanges() {
Configuration conf = new Configuration();
conf.set("first", "-100");
@ -1654,8 +1661,41 @@ public class TestConfiguration extends TestCase {
assertEquals(fileResource.toString(),prop.getResource());
}
}
public void testDumpSensitiveProperty() throws IOException {
final String myPassword = "ThisIsMyPassword";
Configuration testConf = new Configuration(false);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.password", myPassword);
endConfig();
Path fileResource = new Path(CONFIG);
testConf.addResource(fileResource);
try (StringWriter outWriter = new StringWriter()) {
testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
Configuration.dumpConfiguration(testConf, "test.password", outWriter);
assertFalse(outWriter.toString().contains(myPassword));
}
}
public void testDumpSensitiveConfiguration() throws IOException {
final String myPassword = "ThisIsMyPassword";
Configuration testConf = new Configuration(false);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.password", myPassword);
endConfig();
Path fileResource = new Path(CONFIG);
testConf.addResource(fileResource);
try (StringWriter outWriter = new StringWriter()) {
testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
Configuration.dumpConfiguration(testConf, outWriter);
assertFalse(outWriter.toString().contains(myPassword));
}
}
public void testGetValByRegex() {
Configuration conf = new Configuration();
String key1 = "t.abc.key1";