HADOOP-6184. Provide an API to dump Configuration in a JSON format. Contributed by V.V.Chaitanya Krishna.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@807149 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -505,6 +505,9 @@ Trunk (unreleased changes)
     HADOOP-6173. Change src/native/packageNativeHadoop.sh to package all
     native library files.  (Hong Tang via szetszwo)
 
+    HADOOP-6184. Provide an API to dump Configuration in a JSON format.
+    (V.V.Chaitanya Krishna via yhemanth)
+ 
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a

src/java/org/apache/hadoop/conf/Configuration.java

@@ -28,6 +28,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.io.Writer;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -62,6 +63,8 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -155,7 +158,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   private boolean loadDefaults = true;
   
   /**
-   * Configurtion objects
+   * Configuration objects
    */
   private static final WeakHashMap<Configuration,Object> REGISTRY = 
     new WeakHashMap<Configuration,Object>();
@@ -167,6 +170,18 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   private static final ArrayList<String> defaultResources = 
     new ArrayList<String>();
   
+  /**
+   * Flag to indicate if the storage of resource which updates a key needs 
+   * to be stored for each key
+   */
+  private boolean storeResource;
+  
+  /**
+   * Stores the mapping of key to the resource which modifies or loads 
+   * the key most recently
+   */
+  private HashMap<String, String> updatingResource;
+  
   static{
     //print deprecation warning if hadoop-site.xml is found in classpath
     ClassLoader cL = Thread.currentThread().getContextClassLoader();
@@ -214,6 +229,23 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     synchronized(Configuration.class) {
       REGISTRY.put(this, null);
     }
+    this.storeResource = false;
+  }
+  
+  /**
+   * A new configuration with the same settings and additional facility for
+   * storage of resource to each key which loads or updates 
+   * the key most recently
+   * @param other the configuration from which to clone settings
+   * @param storeResource flag to indicate if the storage of resource to 
+   * each key is to be stored
+   */
+  private Configuration(Configuration other, boolean storeResource) {
+    this(other);
+    this.storeResource = storeResource;
+    if (storeResource) {
+      updatingResource = new HashMap<String, String>();
+    }
   }
   
   /** 
@@ -1081,8 +1113,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     if (properties == null) {
       properties = new Properties();
       loadResources(properties, resources, quietmode);
-      if (overlay!= null)
+      if (overlay!= null) {
         properties.putAll(overlay);
+        if (storeResource) {
+          for (Map.Entry<Object,Object> item: overlay.entrySet()) {
+            updatingResource.put((String) item.getKey(), "Unknown");
+          }
+        }
+      }
     }
     return properties;
   }
@@ -1251,6 +1289,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
         if (attr != null && value != null) {
           if (!finalParameters.contains(attr)) {
             properties.setProperty(attr, value);
+            if (storeResource) {
+              updatingResource.put(attr, name.toString());
+            }
             if (finalParameter)
               finalParameters.add(attr);
           } else {
@@ -1322,6 +1363,43 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     }
   }
 
+  /**
+   *  Writes out all the parameters and their properties (final and resource) to
+   *  the given {@link Writer}
+   *  The format of the output would be 
+   *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
+   *  key2.isFinal,key2.resource}... ] } 
+   *  It does not output the parameters of the configuration object which is 
+   *  loaded from an input stream.
+   * @param out the Writer to write to
+   * @throws IOException
+   */
+  public static void dumpConfiguration(Configuration conf, 
+      Writer out) throws IOException {
+    Configuration config = new Configuration(conf,true);
+    config.reloadConfiguration();
+    JsonFactory dumpFactory = new JsonFactory();
+    JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+    dumpGenerator.writeStartObject();
+    dumpGenerator.writeFieldName("properties");
+    dumpGenerator.writeStartArray();
+    dumpGenerator.flush();
+    for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
+      dumpGenerator.writeStartObject();
+      dumpGenerator.writeStringField("key", (String) item.getKey());
+      dumpGenerator.writeStringField("value", 
+          config.get((String) item.getKey()));
+      dumpGenerator.writeBooleanField("isFinal",
+          config.finalParameters.contains(item.getKey()));
+      dumpGenerator.writeStringField("resource",
+          config.updatingResource.get(item.getKey()));
+      dumpGenerator.writeEndObject();
+    }
+    dumpGenerator.writeEndArray();
+    dumpGenerator.writeEndObject();
+    dumpGenerator.flush();
+  }
+  
   /**
    * Get the {@link ClassLoader} for this job.
    * 

src/test/core/org/apache/hadoop/conf/TestConfiguration.java

@@ -21,12 +21,15 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.StringWriter;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Random;
 
 import junit.framework.TestCase;
 
 import org.apache.hadoop.fs.Path;
+import org.codehaus.jackson.map.ObjectMapper;
 
 
 public class TestConfiguration extends TestCase {
@@ -408,6 +411,182 @@ public class TestConfiguration extends TestCase {
     assertTrue(other.getClassLoader() instanceof Fake_ClassLoader);
   }
   
+  static class JsonConfiguration {
+    JsonProperty[] properties;
+
+    public JsonProperty[] getProperties() {
+      return properties;
+    }
+
+    public void setProperties(JsonProperty[] properties) {
+      this.properties = properties;
+    }
+  }
+  
+  static class JsonProperty {
+    String key;
+    public String getKey() {
+      return key;
+    }
+    public void setKey(String key) {
+      this.key = key;
+    }
+    public String getValue() {
+      return value;
+    }
+    public void setValue(String value) {
+      this.value = value;
+    }
+    public boolean getIsFinal() {
+      return isFinal;
+    }
+    public void setIsFinal(boolean isFinal) {
+      this.isFinal = isFinal;
+    }
+    public String getResource() {
+      return resource;
+    }
+    public void setResource(String resource) {
+      this.resource = resource;
+    }
+    String value;
+    boolean isFinal;
+    String resource;
+  }
+  
+  public void testDumpConfiguration () throws IOException {
+    StringWriter outWriter = new StringWriter();
+    Configuration.dumpConfiguration(conf, outWriter);
+    String jsonStr = outWriter.toString();
+    ObjectMapper mapper = new ObjectMapper();
+    JsonConfiguration jconf = 
+      mapper.readValue(jsonStr, JsonConfiguration.class);
+    int defaultLength = jconf.getProperties().length;
+    
+    // add 3 keys to the existing configuration properties
+    out=new BufferedWriter(new FileWriter(CONFIG));
+    startConfig();
+    appendProperty("test.key1", "value1");
+    appendProperty("test.key2", "value2",true);
+    appendProperty("test.key3", "value3");
+    endConfig();
+    Path fileResource = new Path(CONFIG);
+    conf.addResource(fileResource);
+    out.close();
+    
+    outWriter = new StringWriter();
+    Configuration.dumpConfiguration(conf, outWriter);
+    jsonStr = outWriter.toString();
+    mapper = new ObjectMapper();
+    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+    int length = jconf.getProperties().length;
+    // check for consistency in the number of properties parsed in Json format.
+    assertEquals(length, defaultLength+3);
+    
+    //change few keys in another resource file
+    out=new BufferedWriter(new FileWriter(CONFIG2));
+    startConfig();
+    appendProperty("test.key1", "newValue1");
+    appendProperty("test.key2", "newValue2");
+    endConfig();
+    Path fileResource1 = new Path(CONFIG2);
+    conf.addResource(fileResource1);
+    out.close();
+    
+    outWriter = new StringWriter();
+    Configuration.dumpConfiguration(conf, outWriter);
+    jsonStr = outWriter.toString();
+    mapper = new ObjectMapper();
+    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+    
+    // put the keys and their corresponding attributes into a hashmap for their 
+    // efficient retrieval
+    HashMap<String,JsonProperty> confDump = new HashMap<String,JsonProperty>();
+    for(JsonProperty prop : jconf.getProperties()) {
+      confDump.put(prop.getKey(), prop);
+    }
+    // check if the value and resource of test.key1 is changed
+    assertEquals("newValue1", confDump.get("test.key1").getValue());
+    assertEquals(false, confDump.get("test.key1").getIsFinal());
+    assertEquals(fileResource1.toString(),
+        confDump.get("test.key1").getResource());
+    // check if final parameter test.key2 is not changed, since it is first 
+    // loaded as final parameter
+    assertEquals("value2", confDump.get("test.key2").getValue());
+    assertEquals(true, confDump.get("test.key2").getIsFinal());
+    assertEquals(fileResource.toString(),
+        confDump.get("test.key2").getResource());
+    // check for other keys which are not modified later
+    assertEquals("value3", confDump.get("test.key3").getValue());
+    assertEquals(false, confDump.get("test.key3").getIsFinal());
+    assertEquals(fileResource.toString(),
+        confDump.get("test.key3").getResource());
+    // check for resource to be "Unknown" for keys which are loaded using 'set' 
+    // and expansion of properties
+    conf.set("test.key4", "value4");
+    conf.set("test.key5", "value5");
+    conf.set("test.key6", "${test.key5}");
+    outWriter = new StringWriter();
+    Configuration.dumpConfiguration(conf, outWriter);
+    jsonStr = outWriter.toString();
+    mapper = new ObjectMapper();
+    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+    confDump = new HashMap<String, JsonProperty>();
+    for(JsonProperty prop : jconf.getProperties()) {
+      confDump.put(prop.getKey(), prop);
+    }
+    assertEquals("value5",confDump.get("test.key6").getValue());
+    assertEquals("Unknown", confDump.get("test.key4").getResource());
+    outWriter.close();
+  }
+  
+  public void testDumpConfiguratioWithoutDefaults() throws IOException {
+    // check for case when default resources are not loaded
+    Configuration config = new Configuration(false);
+    StringWriter outWriter = new StringWriter();
+    Configuration.dumpConfiguration(config, outWriter);
+    String jsonStr = outWriter.toString();
+    ObjectMapper mapper = new ObjectMapper();
+    JsonConfiguration jconf = 
+      mapper.readValue(jsonStr, JsonConfiguration.class);
+    
+    //ensure that no properties are loaded.
+    assertEquals(0, jconf.getProperties().length);
+    
+    // add 2 keys
+    out=new BufferedWriter(new FileWriter(CONFIG));
+    startConfig();
+    appendProperty("test.key1", "value1");
+    appendProperty("test.key2", "value2",true);
+    endConfig();
+    Path fileResource = new Path(CONFIG);
+    config.addResource(fileResource);
+    out.close();
+    
+    outWriter = new StringWriter();
+    Configuration.dumpConfiguration(config, outWriter);
+    jsonStr = outWriter.toString();
+    mapper = new ObjectMapper();
+    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+    
+    HashMap<String, JsonProperty>confDump = new HashMap<String, JsonProperty>();
+    for (JsonProperty prop : jconf.getProperties()) {
+      confDump.put(prop.getKey(), prop);
+    }
+    //ensure only 2 keys are loaded
+    assertEquals(2,jconf.getProperties().length);
+    //ensure the values are consistent
+    assertEquals(confDump.get("test.key1").getValue(),"value1");
+    assertEquals(confDump.get("test.key2").getValue(),"value2");
+    //check the final tag
+    assertEquals(false, confDump.get("test.key1").getIsFinal());
+    assertEquals(true, confDump.get("test.key2").getIsFinal());
+    //check the resource for each property
+    for (JsonProperty prop : jconf.getProperties()) {
+      assertEquals(fileResource.toString(),prop.getResource());
+    }
+  }
+  
   public static void main(String[] argv) throws Exception {
     junit.textui.TestRunner.main(new String[]{
       TestConfiguration.class.getName()