HADOOP-6184. Provide an API to dump Configuration in a JSON format. Contributed by V.V.Chaitanya Krishna.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@807149 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
76a77aea78
commit
280ab0cf7d
|
@ -505,6 +505,9 @@ Trunk (unreleased changes)
|
|||
HADOOP-6173. Change src/native/packageNativeHadoop.sh to package all
|
||||
native library files. (Hong Tang via szetszwo)
|
||||
|
||||
HADOOP-6184. Provide an API to dump Configuration in a JSON format.
|
||||
(V.V.Chaitanya Krishna via yhemanth)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HADOOP-5595. NameNode does not need to run a replicator to choose a
|
||||
|
|
|
@ -28,6 +28,7 @@ import java.io.InputStream;
|
|||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.io.Reader;
|
||||
import java.io.Writer;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
@ -62,6 +63,8 @@ import org.apache.hadoop.io.Writable;
|
|||
import org.apache.hadoop.io.WritableUtils;
|
||||
import org.apache.hadoop.util.ReflectionUtils;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.codehaus.jackson.JsonFactory;
|
||||
import org.codehaus.jackson.JsonGenerator;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
@ -155,7 +158,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
|||
private boolean loadDefaults = true;
|
||||
|
||||
/**
|
||||
* Configurtion objects
|
||||
* Configuration objects
|
||||
*/
|
||||
private static final WeakHashMap<Configuration,Object> REGISTRY =
|
||||
new WeakHashMap<Configuration,Object>();
|
||||
|
@ -167,6 +170,18 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
|||
private static final ArrayList<String> defaultResources =
|
||||
new ArrayList<String>();
|
||||
|
||||
/**
|
||||
* Flag to indicate if the storage of resource which updates a key needs
|
||||
* to be stored for each key
|
||||
*/
|
||||
private boolean storeResource;
|
||||
|
||||
/**
|
||||
* Stores the mapping of key to the resource which modifies or loads
|
||||
* the key most recently
|
||||
*/
|
||||
private HashMap<String, String> updatingResource;
|
||||
|
||||
static{
|
||||
//print deprecation warning if hadoop-site.xml is found in classpath
|
||||
ClassLoader cL = Thread.currentThread().getContextClassLoader();
|
||||
|
@ -214,6 +229,23 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
|||
synchronized(Configuration.class) {
|
||||
REGISTRY.put(this, null);
|
||||
}
|
||||
this.storeResource = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* A new configuration with the same settings and additional facility for
|
||||
* storage of resource to each key which loads or updates
|
||||
* the key most recently
|
||||
* @param other the configuration from which to clone settings
|
||||
* @param storeResource flag to indicate if the storage of resource to
|
||||
* each key is to be stored
|
||||
*/
|
||||
private Configuration(Configuration other, boolean storeResource) {
|
||||
this(other);
|
||||
this.storeResource = storeResource;
|
||||
if (storeResource) {
|
||||
updatingResource = new HashMap<String, String>();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1081,8 +1113,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
|||
if (properties == null) {
|
||||
properties = new Properties();
|
||||
loadResources(properties, resources, quietmode);
|
||||
if (overlay!= null)
|
||||
if (overlay!= null) {
|
||||
properties.putAll(overlay);
|
||||
if (storeResource) {
|
||||
for (Map.Entry<Object,Object> item: overlay.entrySet()) {
|
||||
updatingResource.put((String) item.getKey(), "Unknown");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
@ -1251,6 +1289,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
|||
if (attr != null && value != null) {
|
||||
if (!finalParameters.contains(attr)) {
|
||||
properties.setProperty(attr, value);
|
||||
if (storeResource) {
|
||||
updatingResource.put(attr, name.toString());
|
||||
}
|
||||
if (finalParameter)
|
||||
finalParameters.add(attr);
|
||||
} else {
|
||||
|
@ -1322,6 +1363,43 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes out all the parameters and their properties (final and resource) to
|
||||
* the given {@link Writer}
|
||||
* The format of the output would be
|
||||
* { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
|
||||
* key2.isFinal,key2.resource}... ] }
|
||||
* It does not output the parameters of the configuration object which is
|
||||
* loaded from an input stream.
|
||||
* @param out the Writer to write to
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void dumpConfiguration(Configuration conf,
|
||||
Writer out) throws IOException {
|
||||
Configuration config = new Configuration(conf,true);
|
||||
config.reloadConfiguration();
|
||||
JsonFactory dumpFactory = new JsonFactory();
|
||||
JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
|
||||
dumpGenerator.writeStartObject();
|
||||
dumpGenerator.writeFieldName("properties");
|
||||
dumpGenerator.writeStartArray();
|
||||
dumpGenerator.flush();
|
||||
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
|
||||
dumpGenerator.writeStartObject();
|
||||
dumpGenerator.writeStringField("key", (String) item.getKey());
|
||||
dumpGenerator.writeStringField("value",
|
||||
config.get((String) item.getKey()));
|
||||
dumpGenerator.writeBooleanField("isFinal",
|
||||
config.finalParameters.contains(item.getKey()));
|
||||
dumpGenerator.writeStringField("resource",
|
||||
config.updatingResource.get(item.getKey()));
|
||||
dumpGenerator.writeEndObject();
|
||||
}
|
||||
dumpGenerator.writeEndArray();
|
||||
dumpGenerator.writeEndObject();
|
||||
dumpGenerator.flush();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the {@link ClassLoader} for this job.
|
||||
*
|
||||
|
|
|
@ -21,12 +21,15 @@ import java.io.BufferedWriter;
|
|||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Random;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
|
||||
public class TestConfiguration extends TestCase {
|
||||
|
@ -408,6 +411,182 @@ public class TestConfiguration extends TestCase {
|
|||
assertTrue(other.getClassLoader() instanceof Fake_ClassLoader);
|
||||
}
|
||||
|
||||
static class JsonConfiguration {
|
||||
JsonProperty[] properties;
|
||||
|
||||
public JsonProperty[] getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
public void setProperties(JsonProperty[] properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
}
|
||||
|
||||
static class JsonProperty {
|
||||
String key;
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
public void setKey(String key) {
|
||||
this.key = key;
|
||||
}
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
public void setValue(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
public boolean getIsFinal() {
|
||||
return isFinal;
|
||||
}
|
||||
public void setIsFinal(boolean isFinal) {
|
||||
this.isFinal = isFinal;
|
||||
}
|
||||
public String getResource() {
|
||||
return resource;
|
||||
}
|
||||
public void setResource(String resource) {
|
||||
this.resource = resource;
|
||||
}
|
||||
String value;
|
||||
boolean isFinal;
|
||||
String resource;
|
||||
}
|
||||
|
||||
public void testDumpConfiguration () throws IOException {
|
||||
StringWriter outWriter = new StringWriter();
|
||||
Configuration.dumpConfiguration(conf, outWriter);
|
||||
String jsonStr = outWriter.toString();
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
JsonConfiguration jconf =
|
||||
mapper.readValue(jsonStr, JsonConfiguration.class);
|
||||
int defaultLength = jconf.getProperties().length;
|
||||
|
||||
// add 3 keys to the existing configuration properties
|
||||
out=new BufferedWriter(new FileWriter(CONFIG));
|
||||
startConfig();
|
||||
appendProperty("test.key1", "value1");
|
||||
appendProperty("test.key2", "value2",true);
|
||||
appendProperty("test.key3", "value3");
|
||||
endConfig();
|
||||
Path fileResource = new Path(CONFIG);
|
||||
conf.addResource(fileResource);
|
||||
out.close();
|
||||
|
||||
outWriter = new StringWriter();
|
||||
Configuration.dumpConfiguration(conf, outWriter);
|
||||
jsonStr = outWriter.toString();
|
||||
mapper = new ObjectMapper();
|
||||
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
|
||||
int length = jconf.getProperties().length;
|
||||
// check for consistency in the number of properties parsed in Json format.
|
||||
assertEquals(length, defaultLength+3);
|
||||
|
||||
//change few keys in another resource file
|
||||
out=new BufferedWriter(new FileWriter(CONFIG2));
|
||||
startConfig();
|
||||
appendProperty("test.key1", "newValue1");
|
||||
appendProperty("test.key2", "newValue2");
|
||||
endConfig();
|
||||
Path fileResource1 = new Path(CONFIG2);
|
||||
conf.addResource(fileResource1);
|
||||
out.close();
|
||||
|
||||
outWriter = new StringWriter();
|
||||
Configuration.dumpConfiguration(conf, outWriter);
|
||||
jsonStr = outWriter.toString();
|
||||
mapper = new ObjectMapper();
|
||||
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
|
||||
|
||||
// put the keys and their corresponding attributes into a hashmap for their
|
||||
// efficient retrieval
|
||||
HashMap<String,JsonProperty> confDump = new HashMap<String,JsonProperty>();
|
||||
for(JsonProperty prop : jconf.getProperties()) {
|
||||
confDump.put(prop.getKey(), prop);
|
||||
}
|
||||
// check if the value and resource of test.key1 is changed
|
||||
assertEquals("newValue1", confDump.get("test.key1").getValue());
|
||||
assertEquals(false, confDump.get("test.key1").getIsFinal());
|
||||
assertEquals(fileResource1.toString(),
|
||||
confDump.get("test.key1").getResource());
|
||||
// check if final parameter test.key2 is not changed, since it is first
|
||||
// loaded as final parameter
|
||||
assertEquals("value2", confDump.get("test.key2").getValue());
|
||||
assertEquals(true, confDump.get("test.key2").getIsFinal());
|
||||
assertEquals(fileResource.toString(),
|
||||
confDump.get("test.key2").getResource());
|
||||
// check for other keys which are not modified later
|
||||
assertEquals("value3", confDump.get("test.key3").getValue());
|
||||
assertEquals(false, confDump.get("test.key3").getIsFinal());
|
||||
assertEquals(fileResource.toString(),
|
||||
confDump.get("test.key3").getResource());
|
||||
// check for resource to be "Unknown" for keys which are loaded using 'set'
|
||||
// and expansion of properties
|
||||
conf.set("test.key4", "value4");
|
||||
conf.set("test.key5", "value5");
|
||||
conf.set("test.key6", "${test.key5}");
|
||||
outWriter = new StringWriter();
|
||||
Configuration.dumpConfiguration(conf, outWriter);
|
||||
jsonStr = outWriter.toString();
|
||||
mapper = new ObjectMapper();
|
||||
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
|
||||
confDump = new HashMap<String, JsonProperty>();
|
||||
for(JsonProperty prop : jconf.getProperties()) {
|
||||
confDump.put(prop.getKey(), prop);
|
||||
}
|
||||
assertEquals("value5",confDump.get("test.key6").getValue());
|
||||
assertEquals("Unknown", confDump.get("test.key4").getResource());
|
||||
outWriter.close();
|
||||
}
|
||||
|
||||
public void testDumpConfiguratioWithoutDefaults() throws IOException {
|
||||
// check for case when default resources are not loaded
|
||||
Configuration config = new Configuration(false);
|
||||
StringWriter outWriter = new StringWriter();
|
||||
Configuration.dumpConfiguration(config, outWriter);
|
||||
String jsonStr = outWriter.toString();
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
JsonConfiguration jconf =
|
||||
mapper.readValue(jsonStr, JsonConfiguration.class);
|
||||
|
||||
//ensure that no properties are loaded.
|
||||
assertEquals(0, jconf.getProperties().length);
|
||||
|
||||
// add 2 keys
|
||||
out=new BufferedWriter(new FileWriter(CONFIG));
|
||||
startConfig();
|
||||
appendProperty("test.key1", "value1");
|
||||
appendProperty("test.key2", "value2",true);
|
||||
endConfig();
|
||||
Path fileResource = new Path(CONFIG);
|
||||
config.addResource(fileResource);
|
||||
out.close();
|
||||
|
||||
outWriter = new StringWriter();
|
||||
Configuration.dumpConfiguration(config, outWriter);
|
||||
jsonStr = outWriter.toString();
|
||||
mapper = new ObjectMapper();
|
||||
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
|
||||
|
||||
HashMap<String, JsonProperty>confDump = new HashMap<String, JsonProperty>();
|
||||
for (JsonProperty prop : jconf.getProperties()) {
|
||||
confDump.put(prop.getKey(), prop);
|
||||
}
|
||||
//ensure only 2 keys are loaded
|
||||
assertEquals(2,jconf.getProperties().length);
|
||||
//ensure the values are consistent
|
||||
assertEquals(confDump.get("test.key1").getValue(),"value1");
|
||||
assertEquals(confDump.get("test.key2").getValue(),"value2");
|
||||
//check the final tag
|
||||
assertEquals(false, confDump.get("test.key1").getIsFinal());
|
||||
assertEquals(true, confDump.get("test.key2").getIsFinal());
|
||||
//check the resource for each property
|
||||
for (JsonProperty prop : jconf.getProperties()) {
|
||||
assertEquals(fileResource.toString(),prop.getResource());
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] argv) throws Exception {
|
||||
junit.textui.TestRunner.main(new String[]{
|
||||
TestConfiguration.class.getName()
|
||||
|
|
Loading…
Reference in New Issue