Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13628. Support to retrieve specific property from configuration via REST API. Contributed by Weiwei Yang

This commit is contained in:
Mingliang Liu 2016-10-03 22:12:17 -04:00
parent d65b957776
commit 00160f71b6
4 changed files with 491 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
View File

@ -70,11 +70,14 @@ public class ConfServlet extends HttpServlet {
response.setContentType("application/json; charset=utf-8");
}
String name = request.getParameter("name");
Writer out = response.getWriter();
try {
writeResponse(getConfFromContext(), out, format);
writeResponse(getConfFromContext(), out, format, name);
} catch (BadFormatException bfe) {
response.sendError(HttpServletResponse.SC_BAD_REQUEST, bfe.getMessage());
} catch (IllegalArgumentException iae) {
response.sendError(HttpServletResponse.SC_NOT_FOUND, iae.getMessage());
}
out.close();
}
@ -89,17 +92,23 @@ public class ConfServlet extends HttpServlet {
/**
* Guts of the servlet - extracted for easy testing.
*/
static void writeResponse(Configuration conf, Writer out, String format)
throws IOException, BadFormatException {
static void writeResponse(Configuration conf,
Writer out, String format, String propertyName)
throws IOException, IllegalArgumentException, BadFormatException {
if (FORMAT_JSON.equals(format)) {
Configuration.dumpConfiguration(conf, out);
Configuration.dumpConfiguration(conf, propertyName, out);
} else if (FORMAT_XML.equals(format)) {
conf.writeXml(out);
conf.writeXml(propertyName, out);
} else {
throw new BadFormatException("Bad format: " + format);
}
}
static void writeResponse(Configuration conf, Writer out, String format)
throws IOException, BadFormatException {
writeResponse(conf, out, format, null);
}
public static class BadFormatException extends Exception {
private static final long serialVersionUID = 1L;

284
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -103,8 +103,9 @@ import org.w3c.dom.Text;
import org.xml.sax.SAXException;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
/**
/**
* Provides access to configuration parameters.
*
* <h4 id="Resources">Resources</h4>
@ -2834,14 +2835,37 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
writeXml(new OutputStreamWriter(out, "UTF-8"));
}
/**
* Write out the non-default properties in this configuration to the given
* {@link Writer}.
*
public void writeXml(Writer out) throws IOException {
writeXml(null, out);
}
/**
* Write out the non-default properties in this configuration to the
* given {@link Writer}.
*
* <li>
* When property name is not empty and the property exists in the
* configuration, this method writes the property and its attributes
* to the {@link Writer}.
* </li>
* <p>
*
* <li>
* When property name is null or empty, this method writes all the
* configuration properties and their attributes to the {@link Writer}.
* </li>
* <p>
*
* <li>
* When property name is not empty but the property doesn't exist in
* the configuration, this method throws an {@link IllegalArgumentException}.
* </li>
* <p>
* @param out the writer to write to.
*/
public void writeXml(Writer out) throws IOException {
Document doc = asXmlDocument();
public void writeXml(String propertyName, Writer out)
throws IOException, IllegalArgumentException {
Document doc = asXmlDocument(propertyName);
try {
DOMSource source = new DOMSource(doc);
@ -2861,62 +2885,180 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* Return the XML DOM corresponding to this Configuration.
*/
private synchronized Document asXmlDocument() throws IOException {
private synchronized Document asXmlDocument(String propertyName)
throws IOException, IllegalArgumentException {
Document doc;
try {
doc =
DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
doc = DocumentBuilderFactory
.newInstance()
.newDocumentBuilder()
.newDocument();
} catch (ParserConfigurationException pe) {
throw new IOException(pe);
}
Element conf = doc.createElement("configuration");
doc.appendChild(conf);
conf.appendChild(doc.createTextNode("\n"));
handleDeprecation(); //ensure properties is set and deprecation is handled
for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
String name = (String)e.nextElement();
Object object = properties.get(name);
String value = null;
if (object instanceof String) {
value = (String) object;
}else {
continue;
if(!Strings.isNullOrEmpty(propertyName)) {
if (!properties.containsKey(propertyName)) {
// given property not found, illegal argument
throw new IllegalArgumentException("Property " +
propertyName + " not found");
} else {
// given property is found, write single property
appendXMLProperty(doc, conf, propertyName);
conf.appendChild(doc.createTextNode("\n"));
}
Element propNode = doc.createElement("property");
conf.appendChild(propNode);
Element nameNode = doc.createElement("name");
nameNode.appendChild(doc.createTextNode(name));
propNode.appendChild(nameNode);
Element valueNode = doc.createElement("value");
valueNode.appendChild(doc.createTextNode(value));
propNode.appendChild(valueNode);
if (updatingResource != null) {
String[] sources = updatingResource.get(name);
if(sources != null) {
for(String s : sources) {
Element sourceNode = doc.createElement("source");
sourceNode.appendChild(doc.createTextNode(s));
propNode.appendChild(sourceNode);
}
}
} else {
// append all elements
for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
appendXMLProperty(doc, conf, (String)e.nextElement());
conf.appendChild(doc.createTextNode("\n"));
}
conf.appendChild(doc.createTextNode("\n"));
}
return doc;
}
/**
* Writes out all the parameters and their properties (final and resource) to
* the given {@link Writer}
* The format of the output would be
* { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
* key2.isFinal,key2.resource}... ] }
* It does not output the parameters of the configuration object which is
* loaded from an input stream.
* Append a property with its attributes to a given {#link Document}
* if the property is found in configuration.
*
* @param doc
* @param conf
* @param propertyName
*/
private synchronized void appendXMLProperty(Document doc, Element conf,
String propertyName) {
// skip writing if given property name is empty or null
if (!Strings.isNullOrEmpty(propertyName)) {
String value = properties.getProperty(propertyName);
if (value != null) {
Element propNode = doc.createElement("property");
conf.appendChild(propNode);
Element nameNode = doc.createElement("name");
nameNode.appendChild(doc.createTextNode(propertyName));
propNode.appendChild(nameNode);
Element valueNode = doc.createElement("value");
valueNode.appendChild(doc.createTextNode(
properties.getProperty(propertyName)));
propNode.appendChild(valueNode);
Element finalNode = doc.createElement("final");
finalNode.appendChild(doc.createTextNode(
String.valueOf(finalParameters.contains(propertyName))));
propNode.appendChild(finalNode);
if (updatingResource != null) {
String[] sources = updatingResource.get(propertyName);
if(sources != null) {
for(String s : sources) {
Element sourceNode = doc.createElement("source");
sourceNode.appendChild(doc.createTextNode(s));
propNode.appendChild(sourceNode);
}
}
}
}
}
}
/**
* Writes properties and their attributes (final and resource)
* to the given {@link Writer}.
*
* <li>
* When propertyName is not empty, and the property exists
* in the configuration, the format of the output would be,
* <pre>
* {
* "property": {
* "key" : "key1",
* "value" : "value1",
* "isFinal" : "key1.isFinal",
* "resource" : "key1.resource"
* }
* }
* </pre>
* </li>
*
* <li>
* When propertyName is null or empty, it behaves same as
* {@link #dumpConfiguration(Configuration, Writer)}, the
* output would be,
* <pre>
* { "properties" :
* [ { key : "key1",
* value : "value1",
* isFinal : "key1.isFinal",
* resource : "key1.resource" },
* { key : "key2",
* value : "value2",
* isFinal : "ke2.isFinal",
* resource : "key2.resource" }
* ]
* }
* </pre>
* </li>
*
* <li>
* When propertyName is not empty, and the property is not
* found in the configuration, this method will throw an
* {@link IllegalArgumentException}.
* </li>
* <p>
* @param config the configuration
* @param propertyName property name
* @param out the Writer to write to
* @throws IOException
* @throws IllegalArgumentException when property name is not
* empty and the property is not found in configuration
**/
public static void dumpConfiguration(Configuration config,
String propertyName, Writer out) throws IOException {
if(Strings.isNullOrEmpty(propertyName)) {
dumpConfiguration(config, out);
} else if (Strings.isNullOrEmpty(config.get(propertyName))) {
throw new IllegalArgumentException("Property " +
propertyName + " not found");
} else {
JsonFactory dumpFactory = new JsonFactory();
JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("property");
appendJSONProperty(dumpGenerator, config, propertyName);
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
}
/**
* Writes out all properties and their attributes (final and resource) to
* the given {@link Writer}, the format of the output would be,
*
* <pre>
* { "properties" :
* [ { key : "key1",
* value : "value1",
* isFinal : "key1.isFinal",
* resource : "key1.resource" },
* { key : "key2",
* value : "value2",
* isFinal : "ke2.isFinal",
* resource : "key2.resource" }
* ]
* }
* </pre>
*
* It does not output the properties of the configuration object which
* is loaded from an input stream.
* <p>
*
* @param config the configuration
* @param out the Writer to write to
* @throws IOException
*/
@ -2930,29 +3072,47 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
dumpGenerator.flush();
synchronized (config) {
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
dumpGenerator.writeStartObject();
dumpGenerator.writeStringField("key", (String) item.getKey());
dumpGenerator.writeStringField("value",
config.get((String) item.getKey()));
dumpGenerator.writeBooleanField("isFinal",
config.finalParameters.contains(item.getKey()));
String[] resources = config.updatingResource.get(item.getKey());
String resource = UNKNOWN_RESOURCE;
if(resources != null && resources.length > 0) {
resource = resources[0];
}
dumpGenerator.writeStringField("resource", resource);
dumpGenerator.writeEndObject();
appendJSONProperty(dumpGenerator,
config,
item.getKey().toString());
}
}
dumpGenerator.writeEndArray();
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
/**
* Write property and its attributes as json format to given
* {@link JsonGenerator}.
*
* @param jsonGen json writer
* @param config configuration
* @param name property name
* @throws IOException
*/
private static void appendJSONProperty(JsonGenerator jsonGen,
Configuration config, String name) throws IOException {
// skip writing if given property name is empty or null
if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
jsonGen.writeStartObject();
jsonGen.writeStringField("key", name);
jsonGen.writeStringField("value", config.get(name));
jsonGen.writeBooleanField("isFinal",
config.finalParameters.contains(name));
String[] resources = config.updatingResource.get(name);
String resource = UNKNOWN_RESOURCE;
if(resources != null && resources.length > 0) {
resource = resources[0];
}
jsonGen.writeStringField("resource", resource);
jsonGen.writeEndObject();
}
}
/**
* Get the {@link ClassLoader} for this job.
*
*
* @return the correct class loader.
*/
public ClassLoader getClassLoader() {

122
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
View File

@ -18,11 +18,15 @@
package org.apache.hadoop.conf;
import java.io.StringWriter;
import java.io.PrintWriter;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.ServletConfig;
import javax.servlet.ServletContext;
import javax.ws.rs.core.HttpHeaders;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@ -34,17 +38,36 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import junit.framework.TestCase;
import com.google.common.base.Strings;
import org.apache.hadoop.http.HttpServer2;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.mock;
import static org.junit.Assert.*;
/**
* Basic test case that the ConfServlet can write configuration
* to its output in XML and JSON format.
*/
public class TestConfServlet extends TestCase {
public class TestConfServlet {
private static final String TEST_KEY = "testconfservlet.key";
private static final String TEST_VAL = "testval";
private static final Map<String, String> TEST_PROPERTIES =
new HashMap<String, String>();
private static final Map<String, String> TEST_FORMATS =
new HashMap<String, String>();
@BeforeClass
public static void initTestProperties() {
TEST_PROPERTIES.put("test.key1", "value1");
TEST_PROPERTIES.put("test.key2", "value2");
TEST_PROPERTIES.put("test.key3", "value3");
TEST_FORMATS.put(ConfServlet.FORMAT_XML, "application/xml");
TEST_FORMATS.put(ConfServlet.FORMAT_JSON, "application/json");
}
private Configuration getTestConf() {
Configuration testConf = new Configuration();
@ -52,6 +75,14 @@ public class TestConfServlet extends TestCase {
return testConf;
}
private Configuration getMultiPropertiesConf() {
Configuration testConf = new Configuration(false);
for(String key : TEST_PROPERTIES.keySet()) {
testConf.set(key, TEST_PROPERTIES.get(key));
}
return testConf;
}
@Test
public void testParseHeaders() throws Exception {
HashMap<String, String> verifyMap = new HashMap<String, String>();
@ -71,6 +102,92 @@ public class TestConfServlet extends TestCase {
}
}
private void verifyGetProperty(Configuration conf, String format,
String propertyName) throws Exception {
StringWriter sw = null;
PrintWriter pw = null;
ConfServlet service = null;
try {
service = new ConfServlet();
ServletConfig servletConf = mock(ServletConfig.class);
ServletContext context = mock(ServletContext.class);
service.init(servletConf);
when(context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE))
.thenReturn(conf);
when(service.getServletContext())
.thenReturn(context);
HttpServletRequest request = mock(HttpServletRequest.class);
when(request.getHeader(HttpHeaders.ACCEPT))
.thenReturn(TEST_FORMATS.get(format));
when(request.getParameter("name"))
.thenReturn(propertyName);
HttpServletResponse response = mock(HttpServletResponse.class);
sw = new StringWriter();
pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw);
// response request
service.doGet(request, response);
String result = sw.toString().trim();
// if property name is null or empty, expect all properties
// in the response
if (Strings.isNullOrEmpty(propertyName)) {
for(String key : TEST_PROPERTIES.keySet()) {
assertTrue(result.contains(key) &&
result.contains(TEST_PROPERTIES.get(key)));
}
} else {
if(conf.get(propertyName) != null) {
// if property name is not empty and property is found
assertTrue(result.contains(propertyName));
for(String key : TEST_PROPERTIES.keySet()) {
if(!key.equals(propertyName)) {
assertFalse(result.contains(key));
}
}
} else {
// if property name is not empty, and it's not in configuration
// expect proper error code and error message is set to the response
Mockito.verify(response).sendError(
Mockito.eq(HttpServletResponse.SC_NOT_FOUND),
Mockito.eq("Property " + propertyName + " not found"));
}
}
} finally {
if (sw != null) {
sw.close();
}
if (pw != null) {
pw.close();
}
if (service != null) {
service.destroy();
}
}
}
@Test
public void testGetProperty() throws Exception {
Configuration configurations = getMultiPropertiesConf();
// list various of property names
String[] testKeys = new String[] {
"test.key1",
"test.unknown.key",
"",
"test.key2",
null
};
for(String format : TEST_FORMATS.keySet()) {
for(String key : testKeys) {
verifyGetProperty(configurations, format, key);
}
}
}
@Test
@SuppressWarnings("unchecked")
public void testWriteJson() throws Exception {
@ -109,7 +226,6 @@ public class TestConfServlet extends TestCase {
for (int i = 0; i < nameNodes.getLength(); i++) {
Node nameNode = nameNodes.item(i);
String key = nameNode.getTextContent();
System.err.println("xml key: " + key);
if (TEST_KEY.equals(key)) {
foundSetting = true;
Element propertyElem = (Element)nameNode.getParentNode();

140
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -42,7 +42,6 @@ import static java.util.concurrent.TimeUnit.*;
import junit.framework.TestCase;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.fail;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
@ -1140,7 +1139,19 @@ public class TestConfiguration extends TestCase {
this.properties = properties;
}
}
static class SingleJsonConfiguration {
private JsonProperty property;
public JsonProperty getProperty() {
return property;
}
public void setProperty(JsonProperty property) {
this.property = property;
}
}
static class JsonProperty {
String key;
public String getKey() {
@ -1171,7 +1182,14 @@ public class TestConfiguration extends TestCase {
boolean isFinal;
String resource;
}
private Configuration getActualConf(String xmlStr) {
Configuration ac = new Configuration(false);
InputStream in = new ByteArrayInputStream(xmlStr.getBytes());
ac.addResource(in);
return ac;
}
public void testGetSetTrimmedNames() throws IOException {
Configuration conf = new Configuration(false);
conf.set(" name", "value");
@ -1180,7 +1198,121 @@ public class TestConfiguration extends TestCase {
assertEquals("value", conf.getRaw(" name "));
}
public void testDumpConfiguration () throws IOException {
public void testDumpProperty() throws IOException {
StringWriter outWriter = new StringWriter();
ObjectMapper mapper = new ObjectMapper();
String jsonStr = null;
String xmlStr = null;
try {
Configuration testConf = new Configuration(false);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.key1", "value1");
appendProperty("test.key2", "value2", true);
appendProperty("test.key3", "value3");
endConfig();
Path fileResource = new Path(CONFIG);
testConf.addResource(fileResource);
out.close();
// case 1: dump an existing property
// test json format
outWriter = new StringWriter();
Configuration.dumpConfiguration(testConf, "test.key2", outWriter);
jsonStr = outWriter.toString();
outWriter.close();
mapper = new ObjectMapper();
SingleJsonConfiguration jconf1 =
mapper.readValue(jsonStr, SingleJsonConfiguration.class);
JsonProperty jp1 = jconf1.getProperty();
assertEquals("test.key2", jp1.getKey());
assertEquals("value2", jp1.getValue());
assertEquals(true, jp1.isFinal);
assertEquals(fileResource.toUri().getPath(), jp1.getResource());
// test xml format
outWriter = new StringWriter();
testConf.writeXml("test.key2", outWriter);
xmlStr = outWriter.toString();
outWriter.close();
Configuration actualConf1 = getActualConf(xmlStr);
assertEquals(1, actualConf1.size());
assertEquals("value2", actualConf1.get("test.key2"));
assertTrue(actualConf1.getFinalParameters().contains("test.key2"));
assertEquals(fileResource.toUri().getPath(),
actualConf1.getPropertySources("test.key2")[0]);
// case 2: dump an non existing property
// test json format
try {
outWriter = new StringWriter();
Configuration.dumpConfiguration(testConf,
"test.unknown.key", outWriter);
outWriter.close();
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
assertTrue(e.getMessage().contains("test.unknown.key") &&
e.getMessage().contains("not found"));
}
// test xml format
try {
outWriter = new StringWriter();
testConf.writeXml("test.unknown.key", outWriter);
outWriter.close();
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
assertTrue(e.getMessage().contains("test.unknown.key") &&
e.getMessage().contains("not found"));
}
// case 3: specify a null property, ensure all configurations are dumped
outWriter = new StringWriter();
Configuration.dumpConfiguration(testConf, null, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
JsonConfiguration jconf3 =
mapper.readValue(jsonStr, JsonConfiguration.class);
assertEquals(3, jconf3.getProperties().length);
outWriter = new StringWriter();
testConf.writeXml(null, outWriter);
xmlStr = outWriter.toString();
outWriter.close();
Configuration actualConf3 = getActualConf(xmlStr);
assertEquals(3, actualConf3.size());
assertTrue(actualConf3.getProps().containsKey("test.key1") &&
actualConf3.getProps().containsKey("test.key2") &&
actualConf3.getProps().containsKey("test.key3"));
// case 4: specify an empty property, ensure all configurations are dumped
outWriter = new StringWriter();
Configuration.dumpConfiguration(testConf, "", outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
JsonConfiguration jconf4 =
mapper.readValue(jsonStr, JsonConfiguration.class);
assertEquals(3, jconf4.getProperties().length);
outWriter = new StringWriter();
testConf.writeXml("", outWriter);
xmlStr = outWriter.toString();
outWriter.close();
Configuration actualConf4 = getActualConf(xmlStr);
assertEquals(3, actualConf4.size());
assertTrue(actualConf4.getProps().containsKey("test.key1") &&
actualConf4.getProps().containsKey("test.key2") &&
actualConf4.getProps().containsKey("test.key3"));
} finally {
if(outWriter != null) {
outWriter.close();
}
if(out != null) {
out.close();
}
}
}
public void testDumpConfiguration() throws IOException {
StringWriter outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
String jsonStr = outWriter.toString();