From 00160f71b6d98244fcb1cb58b2db9fc24f1cd672 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Mon, 3 Oct 2016 22:12:17 -0400 Subject: [PATCH] HADOOP-13628. Support to retrieve specific property from configuration via REST API. Contributed by Weiwei Yang --- .../org/apache/hadoop/conf/ConfServlet.java | 19 +- .../org/apache/hadoop/conf/Configuration.java | 284 ++++++++++++++---- .../apache/hadoop/conf/TestConfServlet.java | 122 +++++++- .../apache/hadoop/conf/TestConfiguration.java | 140 ++++++++- 4 files changed, 491 insertions(+), 74 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java index 700487118d9..cdc95813087 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java @@ -70,11 +70,14 @@ public class ConfServlet extends HttpServlet { response.setContentType("application/json; charset=utf-8"); } + String name = request.getParameter("name"); Writer out = response.getWriter(); try { - writeResponse(getConfFromContext(), out, format); + writeResponse(getConfFromContext(), out, format, name); } catch (BadFormatException bfe) { response.sendError(HttpServletResponse.SC_BAD_REQUEST, bfe.getMessage()); + } catch (IllegalArgumentException iae) { + response.sendError(HttpServletResponse.SC_NOT_FOUND, iae.getMessage()); } out.close(); } @@ -89,17 +92,23 @@ public class ConfServlet extends HttpServlet { /** * Guts of the servlet - extracted for easy testing. */ - static void writeResponse(Configuration conf, Writer out, String format) - throws IOException, BadFormatException { + static void writeResponse(Configuration conf, + Writer out, String format, String propertyName) + throws IOException, IllegalArgumentException, BadFormatException { if (FORMAT_JSON.equals(format)) { - Configuration.dumpConfiguration(conf, out); + Configuration.dumpConfiguration(conf, propertyName, out); } else if (FORMAT_XML.equals(format)) { - conf.writeXml(out); + conf.writeXml(propertyName, out); } else { throw new BadFormatException("Bad format: " + format); } } + static void writeResponse(Configuration conf, Writer out, String format) + throws IOException, BadFormatException { + writeResponse(conf, out, format, null); + } + public static class BadFormatException extends Exception { private static final long serialVersionUID = 1L; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 66734b6519f..1e8ed503c39 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -103,8 +103,9 @@ import org.w3c.dom.Text; import org.xml.sax.SAXException; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; -/** +/** * Provides access to configuration parameters. * *

Resources

@@ -2834,14 +2835,37 @@ public class Configuration implements Iterable>, writeXml(new OutputStreamWriter(out, "UTF-8")); } - /** - * Write out the non-default properties in this configuration to the given - * {@link Writer}. - * + public void writeXml(Writer out) throws IOException { + writeXml(null, out); + } + + /** + * Write out the non-default properties in this configuration to the + * given {@link Writer}. + * + *
  • + * When property name is not empty and the property exists in the + * configuration, this method writes the property and its attributes + * to the {@link Writer}. + *
    • + *

    + * + *

  • + * When property name is null or empty, this method writes all the + * configuration properties and their attributes to the {@link Writer}. + *
    • + *

    + * + *

  • + * When property name is not empty but the property doesn't exist in + * the configuration, this method throws an {@link IllegalArgumentException}. + *
    • + *

    * @param out the writer to write to. */ - public void writeXml(Writer out) throws IOException { - Document doc = asXmlDocument(); + public void writeXml(String propertyName, Writer out) + throws IOException, IllegalArgumentException { + Document doc = asXmlDocument(propertyName); try { DOMSource source = new DOMSource(doc); @@ -2861,62 +2885,180 @@ public class Configuration implements Iterable>, /** * Return the XML DOM corresponding to this Configuration. */ - private synchronized Document asXmlDocument() throws IOException { + private synchronized Document asXmlDocument(String propertyName) + throws IOException, IllegalArgumentException { Document doc; try { - doc = - DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); + doc = DocumentBuilderFactory + .newInstance() + .newDocumentBuilder() + .newDocument(); } catch (ParserConfigurationException pe) { throw new IOException(pe); } + Element conf = doc.createElement("configuration"); doc.appendChild(conf); conf.appendChild(doc.createTextNode("\n")); handleDeprecation(); //ensure properties is set and deprecation is handled - for (Enumeration e = properties.keys(); e.hasMoreElements();) { - String name = (String)e.nextElement(); - Object object = properties.get(name); - String value = null; - if (object instanceof String) { - value = (String) object; - }else { - continue; + + if(!Strings.isNullOrEmpty(propertyName)) { + if (!properties.containsKey(propertyName)) { + // given property not found, illegal argument + throw new IllegalArgumentException("Property " + + propertyName + " not found"); + } else { + // given property is found, write single property + appendXMLProperty(doc, conf, propertyName); + conf.appendChild(doc.createTextNode("\n")); } - Element propNode = doc.createElement("property"); - conf.appendChild(propNode); - - Element nameNode = doc.createElement("name"); - nameNode.appendChild(doc.createTextNode(name)); - propNode.appendChild(nameNode); - - Element valueNode = doc.createElement("value"); - valueNode.appendChild(doc.createTextNode(value)); - propNode.appendChild(valueNode); - - if (updatingResource != null) { - String[] sources = updatingResource.get(name); - if(sources != null) { - for(String s : sources) { - Element sourceNode = doc.createElement("source"); - sourceNode.appendChild(doc.createTextNode(s)); - propNode.appendChild(sourceNode); - } - } + } else { + // append all elements + for (Enumeration e = properties.keys(); e.hasMoreElements();) { + appendXMLProperty(doc, conf, (String)e.nextElement()); + conf.appendChild(doc.createTextNode("\n")); } - - conf.appendChild(doc.createTextNode("\n")); } return doc; } /** - * Writes out all the parameters and their properties (final and resource) to - * the given {@link Writer} - * The format of the output would be - * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2, - * key2.isFinal,key2.resource}... ] } - * It does not output the parameters of the configuration object which is - * loaded from an input stream. + * Append a property with its attributes to a given {#link Document} + * if the property is found in configuration. + * + * @param doc + * @param conf + * @param propertyName + */ + private synchronized void appendXMLProperty(Document doc, Element conf, + String propertyName) { + // skip writing if given property name is empty or null + if (!Strings.isNullOrEmpty(propertyName)) { + String value = properties.getProperty(propertyName); + if (value != null) { + Element propNode = doc.createElement("property"); + conf.appendChild(propNode); + + Element nameNode = doc.createElement("name"); + nameNode.appendChild(doc.createTextNode(propertyName)); + propNode.appendChild(nameNode); + + Element valueNode = doc.createElement("value"); + valueNode.appendChild(doc.createTextNode( + properties.getProperty(propertyName))); + propNode.appendChild(valueNode); + + Element finalNode = doc.createElement("final"); + finalNode.appendChild(doc.createTextNode( + String.valueOf(finalParameters.contains(propertyName)))); + propNode.appendChild(finalNode); + + if (updatingResource != null) { + String[] sources = updatingResource.get(propertyName); + if(sources != null) { + for(String s : sources) { + Element sourceNode = doc.createElement("source"); + sourceNode.appendChild(doc.createTextNode(s)); + propNode.appendChild(sourceNode); + } + } + } + } + } + } + + /** + * Writes properties and their attributes (final and resource) + * to the given {@link Writer}. + * + *
  • + * When propertyName is not empty, and the property exists + * in the configuration, the format of the output would be, + * 
    +   *  {
+   *    "property": {
+   *      "key" : "key1",
+   *      "value" : "value1",
+   *      "isFinal" : "key1.isFinal",
+   *      "resource" : "key1.resource"
+   *    }
+   *  }
+   *
    + *
    • + * + *
  • + * When propertyName is null or empty, it behaves same as + * {@link #dumpConfiguration(Configuration, Writer)}, the + * output would be, + * 
    +   *  { "properties" :
+   *      [ { key : "key1",
+   *          value : "value1",
+   *          isFinal : "key1.isFinal",
+   *          resource : "key1.resource" },
+   *        { key : "key2",
+   *          value : "value2",
+   *          isFinal : "ke2.isFinal",
+   *          resource : "key2.resource" }
+   *       ]
+   *   }
+   *
    + *
    • + * + *
  • + * When propertyName is not empty, and the property is not + * found in the configuration, this method will throw an + * {@link IllegalArgumentException}. + *
    • + *

    + * @param config the configuration + * @param propertyName property name + * @param out the Writer to write to + * @throws IOException + * @throws IllegalArgumentException when property name is not + * empty and the property is not found in configuration + **/ + public static void dumpConfiguration(Configuration config, + String propertyName, Writer out) throws IOException { + if(Strings.isNullOrEmpty(propertyName)) { + dumpConfiguration(config, out); + } else if (Strings.isNullOrEmpty(config.get(propertyName))) { + throw new IllegalArgumentException("Property " + + propertyName + " not found"); + } else { + JsonFactory dumpFactory = new JsonFactory(); + JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out); + dumpGenerator.writeStartObject(); + dumpGenerator.writeFieldName("property"); + appendJSONProperty(dumpGenerator, config, propertyName); + dumpGenerator.writeEndObject(); + dumpGenerator.flush(); + } + } + + /** + * Writes out all properties and their attributes (final and resource) to + * the given {@link Writer}, the format of the output would be, + * + * 

    +   *  { "properties" :
+   *      [ { key : "key1",
+   *          value : "value1",
+   *          isFinal : "key1.isFinal",
+   *          resource : "key1.resource" },
+   *        { key : "key2",
+   *          value : "value2",
+   *          isFinal : "ke2.isFinal",
+   *          resource : "key2.resource" }
+   *       ]
+   *   }
+   *
    + * + * It does not output the properties of the configuration object which + * is loaded from an input stream. + *

    + * + * @param config the configuration * @param out the Writer to write to * @throws IOException */ @@ -2930,29 +3072,47 @@ public class Configuration implements Iterable>, dumpGenerator.flush(); synchronized (config) { for (Map.Entry item: config.getProps().entrySet()) { - dumpGenerator.writeStartObject(); - dumpGenerator.writeStringField("key", (String) item.getKey()); - dumpGenerator.writeStringField("value", - config.get((String) item.getKey())); - dumpGenerator.writeBooleanField("isFinal", - config.finalParameters.contains(item.getKey())); - String[] resources = config.updatingResource.get(item.getKey()); - String resource = UNKNOWN_RESOURCE; - if(resources != null && resources.length > 0) { - resource = resources[0]; - } - dumpGenerator.writeStringField("resource", resource); - dumpGenerator.writeEndObject(); + appendJSONProperty(dumpGenerator, + config, + item.getKey().toString()); } } dumpGenerator.writeEndArray(); dumpGenerator.writeEndObject(); dumpGenerator.flush(); } - + + /** + * Write property and its attributes as json format to given + * {@link JsonGenerator}. + * + * @param jsonGen json writer + * @param config configuration + * @param name property name + * @throws IOException + */ + private static void appendJSONProperty(JsonGenerator jsonGen, + Configuration config, String name) throws IOException { + // skip writing if given property name is empty or null + if(!Strings.isNullOrEmpty(name) && jsonGen != null) { + jsonGen.writeStartObject(); + jsonGen.writeStringField("key", name); + jsonGen.writeStringField("value", config.get(name)); + jsonGen.writeBooleanField("isFinal", + config.finalParameters.contains(name)); + String[] resources = config.updatingResource.get(name); + String resource = UNKNOWN_RESOURCE; + if(resources != null && resources.length > 0) { + resource = resources[0]; + } + jsonGen.writeStringField("resource", resource); + jsonGen.writeEndObject(); + } + } + /** * Get the {@link ClassLoader} for this job. - * + * * @return the correct class loader. */ public ClassLoader getClassLoader() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java index ad3d5c5784f..60035be94dc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java @@ -18,11 +18,15 @@ package org.apache.hadoop.conf; import java.io.StringWriter; +import java.io.PrintWriter; import java.io.StringReader; import java.util.HashMap; import java.util.Map; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.ServletConfig; +import javax.servlet.ServletContext; import javax.ws.rs.core.HttpHeaders; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -34,17 +38,36 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; -import junit.framework.TestCase; +import com.google.common.base.Strings; + +import org.apache.hadoop.http.HttpServer2; +import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.mock; +import static org.junit.Assert.*; /** * Basic test case that the ConfServlet can write configuration * to its output in XML and JSON format. */ -public class TestConfServlet extends TestCase { +public class TestConfServlet { private static final String TEST_KEY = "testconfservlet.key"; private static final String TEST_VAL = "testval"; + private static final Map TEST_PROPERTIES = + new HashMap(); + private static final Map TEST_FORMATS = + new HashMap(); + + @BeforeClass + public static void initTestProperties() { + TEST_PROPERTIES.put("test.key1", "value1"); + TEST_PROPERTIES.put("test.key2", "value2"); + TEST_PROPERTIES.put("test.key3", "value3"); + TEST_FORMATS.put(ConfServlet.FORMAT_XML, "application/xml"); + TEST_FORMATS.put(ConfServlet.FORMAT_JSON, "application/json"); + } private Configuration getTestConf() { Configuration testConf = new Configuration(); @@ -52,6 +75,14 @@ public class TestConfServlet extends TestCase { return testConf; } + private Configuration getMultiPropertiesConf() { + Configuration testConf = new Configuration(false); + for(String key : TEST_PROPERTIES.keySet()) { + testConf.set(key, TEST_PROPERTIES.get(key)); + } + return testConf; + } + @Test public void testParseHeaders() throws Exception { HashMap verifyMap = new HashMap(); @@ -71,6 +102,92 @@ public class TestConfServlet extends TestCase { } } + private void verifyGetProperty(Configuration conf, String format, + String propertyName) throws Exception { + StringWriter sw = null; + PrintWriter pw = null; + ConfServlet service = null; + try { + service = new ConfServlet(); + ServletConfig servletConf = mock(ServletConfig.class); + ServletContext context = mock(ServletContext.class); + service.init(servletConf); + when(context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE)) + .thenReturn(conf); + when(service.getServletContext()) + .thenReturn(context); + + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getHeader(HttpHeaders.ACCEPT)) + .thenReturn(TEST_FORMATS.get(format)); + when(request.getParameter("name")) + .thenReturn(propertyName); + + HttpServletResponse response = mock(HttpServletResponse.class); + sw = new StringWriter(); + pw = new PrintWriter(sw); + when(response.getWriter()).thenReturn(pw); + + // response request + service.doGet(request, response); + String result = sw.toString().trim(); + + // if property name is null or empty, expect all properties + // in the response + if (Strings.isNullOrEmpty(propertyName)) { + for(String key : TEST_PROPERTIES.keySet()) { + assertTrue(result.contains(key) && + result.contains(TEST_PROPERTIES.get(key))); + } + } else { + if(conf.get(propertyName) != null) { + // if property name is not empty and property is found + assertTrue(result.contains(propertyName)); + for(String key : TEST_PROPERTIES.keySet()) { + if(!key.equals(propertyName)) { + assertFalse(result.contains(key)); + } + } + } else { + // if property name is not empty, and it's not in configuration + // expect proper error code and error message is set to the response + Mockito.verify(response).sendError( + Mockito.eq(HttpServletResponse.SC_NOT_FOUND), + Mockito.eq("Property " + propertyName + " not found")); + } + } + } finally { + if (sw != null) { + sw.close(); + } + if (pw != null) { + pw.close(); + } + if (service != null) { + service.destroy(); + } + } + } + + @Test + public void testGetProperty() throws Exception { + Configuration configurations = getMultiPropertiesConf(); + // list various of property names + String[] testKeys = new String[] { + "test.key1", + "test.unknown.key", + "", + "test.key2", + null + }; + + for(String format : TEST_FORMATS.keySet()) { + for(String key : testKeys) { + verifyGetProperty(configurations, format, key); + } + } + } + @Test @SuppressWarnings("unchecked") public void testWriteJson() throws Exception { @@ -109,7 +226,6 @@ public class TestConfServlet extends TestCase { for (int i = 0; i < nameNodes.getLength(); i++) { Node nameNode = nameNodes.item(i); String key = nameNode.getTextContent(); - System.err.println("xml key: " + key); if (TEST_KEY.equals(key)) { foundSetting = true; Element propertyElem = (Element)nameNode.getParentNode(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index e7ebea148f4..917ccbce29c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -42,7 +42,6 @@ import static java.util.concurrent.TimeUnit.*; import junit.framework.TestCase; import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.fail; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration.IntegerRanges; @@ -1140,7 +1139,19 @@ public class TestConfiguration extends TestCase { this.properties = properties; } } - + + static class SingleJsonConfiguration { + private JsonProperty property; + + public JsonProperty getProperty() { + return property; + } + + public void setProperty(JsonProperty property) { + this.property = property; + } + } + static class JsonProperty { String key; public String getKey() { @@ -1171,7 +1182,14 @@ public class TestConfiguration extends TestCase { boolean isFinal; String resource; } - + + private Configuration getActualConf(String xmlStr) { + Configuration ac = new Configuration(false); + InputStream in = new ByteArrayInputStream(xmlStr.getBytes()); + ac.addResource(in); + return ac; + } + public void testGetSetTrimmedNames() throws IOException { Configuration conf = new Configuration(false); conf.set(" name", "value"); @@ -1180,7 +1198,121 @@ public class TestConfiguration extends TestCase { assertEquals("value", conf.getRaw(" name ")); } - public void testDumpConfiguration () throws IOException { + public void testDumpProperty() throws IOException { + StringWriter outWriter = new StringWriter(); + ObjectMapper mapper = new ObjectMapper(); + String jsonStr = null; + String xmlStr = null; + try { + Configuration testConf = new Configuration(false); + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.key1", "value1"); + appendProperty("test.key2", "value2", true); + appendProperty("test.key3", "value3"); + endConfig(); + Path fileResource = new Path(CONFIG); + testConf.addResource(fileResource); + out.close(); + + // case 1: dump an existing property + // test json format + outWriter = new StringWriter(); + Configuration.dumpConfiguration(testConf, "test.key2", outWriter); + jsonStr = outWriter.toString(); + outWriter.close(); + mapper = new ObjectMapper(); + SingleJsonConfiguration jconf1 = + mapper.readValue(jsonStr, SingleJsonConfiguration.class); + JsonProperty jp1 = jconf1.getProperty(); + assertEquals("test.key2", jp1.getKey()); + assertEquals("value2", jp1.getValue()); + assertEquals(true, jp1.isFinal); + assertEquals(fileResource.toUri().getPath(), jp1.getResource()); + + // test xml format + outWriter = new StringWriter(); + testConf.writeXml("test.key2", outWriter); + xmlStr = outWriter.toString(); + outWriter.close(); + Configuration actualConf1 = getActualConf(xmlStr); + assertEquals(1, actualConf1.size()); + assertEquals("value2", actualConf1.get("test.key2")); + assertTrue(actualConf1.getFinalParameters().contains("test.key2")); + assertEquals(fileResource.toUri().getPath(), + actualConf1.getPropertySources("test.key2")[0]); + + // case 2: dump an non existing property + // test json format + try { + outWriter = new StringWriter(); + Configuration.dumpConfiguration(testConf, + "test.unknown.key", outWriter); + outWriter.close(); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + assertTrue(e.getMessage().contains("test.unknown.key") && + e.getMessage().contains("not found")); + } + // test xml format + try { + outWriter = new StringWriter(); + testConf.writeXml("test.unknown.key", outWriter); + outWriter.close(); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + assertTrue(e.getMessage().contains("test.unknown.key") && + e.getMessage().contains("not found")); + } + + // case 3: specify a null property, ensure all configurations are dumped + outWriter = new StringWriter(); + Configuration.dumpConfiguration(testConf, null, outWriter); + jsonStr = outWriter.toString(); + mapper = new ObjectMapper(); + JsonConfiguration jconf3 = + mapper.readValue(jsonStr, JsonConfiguration.class); + assertEquals(3, jconf3.getProperties().length); + + outWriter = new StringWriter(); + testConf.writeXml(null, outWriter); + xmlStr = outWriter.toString(); + outWriter.close(); + Configuration actualConf3 = getActualConf(xmlStr); + assertEquals(3, actualConf3.size()); + assertTrue(actualConf3.getProps().containsKey("test.key1") && + actualConf3.getProps().containsKey("test.key2") && + actualConf3.getProps().containsKey("test.key3")); + + // case 4: specify an empty property, ensure all configurations are dumped + outWriter = new StringWriter(); + Configuration.dumpConfiguration(testConf, "", outWriter); + jsonStr = outWriter.toString(); + mapper = new ObjectMapper(); + JsonConfiguration jconf4 = + mapper.readValue(jsonStr, JsonConfiguration.class); + assertEquals(3, jconf4.getProperties().length); + + outWriter = new StringWriter(); + testConf.writeXml("", outWriter); + xmlStr = outWriter.toString(); + outWriter.close(); + Configuration actualConf4 = getActualConf(xmlStr); + assertEquals(3, actualConf4.size()); + assertTrue(actualConf4.getProps().containsKey("test.key1") && + actualConf4.getProps().containsKey("test.key2") && + actualConf4.getProps().containsKey("test.key3")); + } finally { + if(outWriter != null) { + outWriter.close(); + } + if(out != null) { + out.close(); + } + } + } + + public void testDumpConfiguration() throws IOException { StringWriter outWriter = new StringWriter(); Configuration.dumpConfiguration(conf, outWriter); String jsonStr = outWriter.toString();