HADOOP-14260. Configuration.dumpConfiguration should redact sensitive information. Contributed by John Zhuge.
This commit is contained in:
parent
4222c97108
commit
582648befa
|
@ -3146,7 +3146,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
||||||
JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
|
JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
|
||||||
dumpGenerator.writeStartObject();
|
dumpGenerator.writeStartObject();
|
||||||
dumpGenerator.writeFieldName("property");
|
dumpGenerator.writeFieldName("property");
|
||||||
appendJSONProperty(dumpGenerator, config, propertyName);
|
appendJSONProperty(dumpGenerator, config, propertyName,
|
||||||
|
new ConfigRedactor(config));
|
||||||
dumpGenerator.writeEndObject();
|
dumpGenerator.writeEndObject();
|
||||||
dumpGenerator.flush();
|
dumpGenerator.flush();
|
||||||
}
|
}
|
||||||
|
@ -3186,11 +3187,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
||||||
dumpGenerator.writeFieldName("properties");
|
dumpGenerator.writeFieldName("properties");
|
||||||
dumpGenerator.writeStartArray();
|
dumpGenerator.writeStartArray();
|
||||||
dumpGenerator.flush();
|
dumpGenerator.flush();
|
||||||
|
ConfigRedactor redactor = new ConfigRedactor(config);
|
||||||
synchronized (config) {
|
synchronized (config) {
|
||||||
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
|
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
|
||||||
appendJSONProperty(dumpGenerator,
|
appendJSONProperty(dumpGenerator, config, item.getKey().toString(),
|
||||||
config,
|
redactor);
|
||||||
item.getKey().toString());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
dumpGenerator.writeEndArray();
|
dumpGenerator.writeEndArray();
|
||||||
|
@ -3208,12 +3209,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
private static void appendJSONProperty(JsonGenerator jsonGen,
|
private static void appendJSONProperty(JsonGenerator jsonGen,
|
||||||
Configuration config, String name) throws IOException {
|
Configuration config, String name, ConfigRedactor redactor)
|
||||||
|
throws IOException {
|
||||||
// skip writing if given property name is empty or null
|
// skip writing if given property name is empty or null
|
||||||
if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
|
if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
|
||||||
jsonGen.writeStartObject();
|
jsonGen.writeStartObject();
|
||||||
jsonGen.writeStringField("key", name);
|
jsonGen.writeStringField("key", name);
|
||||||
jsonGen.writeStringField("value", config.get(name));
|
jsonGen.writeStringField("value",
|
||||||
|
redactor.redact(name, config.get(name)));
|
||||||
jsonGen.writeBooleanField("isFinal",
|
jsonGen.writeBooleanField("isFinal",
|
||||||
config.finalParameters.contains(name));
|
config.finalParameters.contains(name));
|
||||||
String[] resources = config.updatingResource.get(name);
|
String[] resources = config.updatingResource.get(name);
|
||||||
|
|
|
@ -49,6 +49,7 @@ import static org.junit.Assert.assertArrayEquals;
|
||||||
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
import org.apache.hadoop.conf.Configuration.IntegerRanges;
|
import org.apache.hadoop.conf.Configuration.IntegerRanges;
|
||||||
|
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.io.IOUtils;
|
import org.apache.hadoop.io.IOUtils;
|
||||||
import org.apache.hadoop.net.NetUtils;
|
import org.apache.hadoop.net.NetUtils;
|
||||||
|
@ -82,6 +83,11 @@ public class TestConfiguration extends TestCase {
|
||||||
/** Four apostrophes. */
|
/** Four apostrophes. */
|
||||||
public static final String ESCAPED = "''''";
|
public static final String ESCAPED = "''''";
|
||||||
|
|
||||||
|
private static final String SENSITIVE_CONFIG_KEYS =
|
||||||
|
CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
|
||||||
|
|
||||||
|
private BufferedWriter out;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void setUp() throws Exception {
|
protected void setUp() throws Exception {
|
||||||
super.setUp();
|
super.setUp();
|
||||||
|
@ -90,6 +96,9 @@ public class TestConfiguration extends TestCase {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void tearDown() throws Exception {
|
protected void tearDown() throws Exception {
|
||||||
|
if(out != null) {
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
super.tearDown();
|
super.tearDown();
|
||||||
new File(CONFIG).delete();
|
new File(CONFIG).delete();
|
||||||
new File(CONFIG2).delete();
|
new File(CONFIG2).delete();
|
||||||
|
@ -878,8 +887,6 @@ public class TestConfiguration extends TestCase {
|
||||||
new File(new File(relConfig).getParent()).delete();
|
new File(new File(relConfig).getParent()).delete();
|
||||||
}
|
}
|
||||||
|
|
||||||
BufferedWriter out;
|
|
||||||
|
|
||||||
public void testIntegerRanges() {
|
public void testIntegerRanges() {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
conf.set("first", "-100");
|
conf.set("first", "-100");
|
||||||
|
@ -1787,8 +1794,41 @@ public class TestConfiguration extends TestCase {
|
||||||
assertEquals(fileResource.toString(),prop.getResource());
|
assertEquals(fileResource.toString(),prop.getResource());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testDumpSensitiveProperty() throws IOException {
|
||||||
|
final String myPassword = "ThisIsMyPassword";
|
||||||
|
Configuration testConf = new Configuration(false);
|
||||||
|
out = new BufferedWriter(new FileWriter(CONFIG));
|
||||||
|
startConfig();
|
||||||
|
appendProperty("test.password", myPassword);
|
||||||
|
endConfig();
|
||||||
|
Path fileResource = new Path(CONFIG);
|
||||||
|
testConf.addResource(fileResource);
|
||||||
|
|
||||||
|
try (StringWriter outWriter = new StringWriter()) {
|
||||||
|
testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
|
||||||
|
Configuration.dumpConfiguration(testConf, "test.password", outWriter);
|
||||||
|
assertFalse(outWriter.toString().contains(myPassword));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testDumpSensitiveConfiguration() throws IOException {
|
||||||
|
final String myPassword = "ThisIsMyPassword";
|
||||||
|
Configuration testConf = new Configuration(false);
|
||||||
|
out = new BufferedWriter(new FileWriter(CONFIG));
|
||||||
|
startConfig();
|
||||||
|
appendProperty("test.password", myPassword);
|
||||||
|
endConfig();
|
||||||
|
Path fileResource = new Path(CONFIG);
|
||||||
|
testConf.addResource(fileResource);
|
||||||
|
|
||||||
|
try (StringWriter outWriter = new StringWriter()) {
|
||||||
|
testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
|
||||||
|
Configuration.dumpConfiguration(testConf, outWriter);
|
||||||
|
assertFalse(outWriter.toString().contains(myPassword));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void testGetValByRegex() {
|
public void testGetValByRegex() {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
String key1 = "t.abc.key1";
|
String key1 = "t.abc.key1";
|
||||||
|
|
Loading…
Reference in New Issue