HADOOP-8525. Provide Improved Traceability for Configuration (bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1359775 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-07-10 16:49:24 +00:00
parent 6804ef32fc
commit 0b7139d6bc
5 changed files with 205 additions and 75 deletions

View File

@ -15,6 +15,8 @@ Trunk (unreleased changes)
HADOOP-8470. Add NetworkTopologyWithNodeGroup, a 4-layer implementation HADOOP-8470. Add NetworkTopologyWithNodeGroup, a 4-layer implementation
of NetworkTopology. (Junping Du via szetszwo) of NetworkTopology. (Junping Du via szetszwo)
HADOOP-8525. Provide Improved Traceability for Configuration (bobby)
IMPROVEMENTS IMPROVEMENTS
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution

View File

@ -40,6 +40,7 @@ import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.ListIterator; import java.util.ListIterator;
import java.util.Map; import java.util.Map;
@ -75,7 +76,6 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.JsonGenerator;
import org.w3c.dom.Comment;
import org.w3c.dom.DOMException; import org.w3c.dom.DOMException;
import org.w3c.dom.Document; import org.w3c.dom.Document;
import org.w3c.dom.Element; import org.w3c.dom.Element;
@ -158,17 +158,45 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private boolean quietmode = true; private boolean quietmode = true;
private static class Resource {
private final Object resource;
private final String name;
public Resource(Object resource) {
this(resource, resource.toString());
}
public Resource(Object resource, String name) {
this.resource = resource;
this.name = name;
}
public String getName(){
return name;
}
public Object getResource() {
return resource;
}
@Override
public String toString() {
return name;
}
}
/** /**
* List of configuration resources. * List of configuration resources.
*/ */
private ArrayList<Object> resources = new ArrayList<Object>(); private ArrayList<Resource> resources = new ArrayList<Resource>();
/** /**
* The value reported as the setting resource when a key is set * The value reported as the setting resource when a key is set
* by code rather than a file resource. * by code rather than a file resource by dumpConfiguration.
*/ */
static final String UNKNOWN_RESOURCE = "Unknown"; static final String UNKNOWN_RESOURCE = "Unknown";
/** /**
* List of configuration parameters marked <b>final</b>. * List of configuration parameters marked <b>final</b>.
*/ */
@ -202,7 +230,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* Stores the mapping of key to the resource which modifies or loads * Stores the mapping of key to the resource which modifies or loads
* the key most recently * the key most recently
*/ */
private HashMap<String, String> updatingResource; private HashMap<String, String[]> updatingResource;
/** /**
* Class to keep the information about the keys which replace the deprecated * Class to keep the information about the keys which replace the deprecated
@ -369,7 +397,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @return alternate name. * @return alternate name.
*/ */
private String[] getAlternateNames(String name) { private String[] getAlternateNames(String name) {
String oldName, altNames[] = null; String altNames[] = null;
DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name); DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
if (keyInfo == null) { if (keyInfo == null) {
altNames = (reverseDeprecatedKeyMap.get(name) != null ) ? altNames = (reverseDeprecatedKeyMap.get(name) != null ) ?
@ -485,7 +513,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
*/ */
public Configuration(boolean loadDefaults) { public Configuration(boolean loadDefaults) {
this.loadDefaults = loadDefaults; this.loadDefaults = loadDefaults;
updatingResource = new HashMap<String, String>(); updatingResource = new HashMap<String, String[]>();
synchronized(Configuration.class) { synchronized(Configuration.class) {
REGISTRY.put(this, null); REGISTRY.put(this, null);
} }
@ -498,7 +526,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Configuration(Configuration other) { public Configuration(Configuration other) {
this.resources = (ArrayList)other.resources.clone(); this.resources = (ArrayList<Resource>) other.resources.clone();
synchronized(other) { synchronized(other) {
if (other.properties != null) { if (other.properties != null) {
this.properties = (Properties)other.properties.clone(); this.properties = (Properties)other.properties.clone();
@ -508,7 +536,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
this.overlay = (Properties)other.overlay.clone(); this.overlay = (Properties)other.overlay.clone();
} }
this.updatingResource = new HashMap<String, String>(other.updatingResource); this.updatingResource = new HashMap<String, String[]>(other.updatingResource);
} }
this.finalParameters = new HashSet<String>(other.finalParameters); this.finalParameters = new HashSet<String>(other.finalParameters);
@ -546,7 +574,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* with that name. * with that name.
*/ */
public void addResource(String name) { public void addResource(String name) {
addResourceObject(name); addResourceObject(new Resource(name));
} }
/** /**
@ -560,7 +588,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* the classpath. * the classpath.
*/ */
public void addResource(URL url) { public void addResource(URL url) {
addResourceObject(url); addResourceObject(new Resource(url));
} }
/** /**
@ -574,7 +602,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* the classpath. * the classpath.
*/ */
public void addResource(Path file) { public void addResource(Path file) {
addResourceObject(file); addResourceObject(new Resource(file));
} }
/** /**
@ -586,7 +614,21 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @param in InputStream to deserialize the object from. * @param in InputStream to deserialize the object from.
*/ */
public void addResource(InputStream in) { public void addResource(InputStream in) {
addResourceObject(in); addResourceObject(new Resource(in));
}
/**
* Add a configuration resource.
*
* The properties of this resource will override properties of previously
* added resources, unless they were marked <a href="#Final">final</a>.
*
* @param in InputStream to deserialize the object from.
* @param name the name of the resource because InputStream.toString is not
* very descriptive some times.
*/
public void addResource(InputStream in, String name) {
addResourceObject(new Resource(in, name));
} }
@ -603,7 +645,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
finalParameters.clear(); // clear site-limits finalParameters.clear(); // clear site-limits
} }
private synchronized void addResourceObject(Object resource) { private synchronized void addResourceObject(Resource resource) {
resources.add(resource); // add to resources resources.add(resource); // add to resources
reloadConfiguration(); reloadConfiguration();
} }
@ -721,17 +763,39 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @param value property value. * @param value property value.
*/ */
public void set(String name, String value) { public void set(String name, String value) {
set(name, value, null);
}
/**
* Set the <code>value</code> of the <code>name</code> property. If
* <code>name</code> is deprecated or there is a deprecated name associated to it,
* it sets the value to both names.
*
* @param name property name.
* @param value property value.
* @param source the place that this configuration value came from
* (For debugging).
*/
public void set(String name, String value, String source) {
if (deprecatedKeyMap.isEmpty()) { if (deprecatedKeyMap.isEmpty()) {
getProps(); getProps();
} }
getOverlay().setProperty(name, value); getOverlay().setProperty(name, value);
getProps().setProperty(name, value); getProps().setProperty(name, value);
updatingResource.put(name, UNKNOWN_RESOURCE); if(source == null) {
updatingResource.put(name, new String[] {"programatically"});
} else {
updatingResource.put(name, new String[] {source});
}
String[] altNames = getAlternateNames(name); String[] altNames = getAlternateNames(name);
if (altNames != null && altNames.length > 0) { if (altNames != null && altNames.length > 0) {
String altSource = "because " + name + " is deprecated";
for(String altName : altNames) { for(String altName : altNames) {
getOverlay().setProperty(altName, value); if(!altName.equals(name)) {
getProps().setProperty(altName, value); getOverlay().setProperty(altName, value);
getProps().setProperty(altName, value);
updatingResource.put(altName, new String[] {altSource});
}
} }
} }
warnOnceIfDeprecated(name); warnOnceIfDeprecated(name);
@ -1071,17 +1135,22 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
} }
/** /**
* Gets the absolute path to the resource object (file, URL, etc.), for a given * Gets information about why a property was set. Typically this is the
* property name. * path to the resource objects (file, URL, etc.) the property came from, but
* it can also indicate that it was set programatically, or because of the
* command line.
* *
* @param name - The property name to get the source of. * @param name - The property name to get the source of.
* @return null - If the property or its source wasn't found or if the property * @return null - If the property or its source wasn't found. Otherwise,
* was defined in code (i.e. in a Configuration instance, not from a physical * returns a list of the sources of the resource. The older sources are
* resource). Otherwise, returns the absolute path of the resource that loaded * the first ones in the list. So for example if a configuration is set from
* the property name, as a String. * the command line, and then written out to a file that is read back in the
* first entry would indicate that it was set from the command line, while
* the second one would indicate the file that the new configuration was read
* in from.
*/ */
@InterfaceStability.Unstable @InterfaceStability.Unstable
public synchronized String getPropertySource(String name) { public synchronized String[] getPropertySources(String name) {
if (properties == null) { if (properties == null) {
// If properties is null, it means a resource was newly added // If properties is null, it means a resource was newly added
// but the props were cleared so as to load it upon future // but the props were cleared so as to load it upon future
@ -1093,11 +1162,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
if (properties == null || updatingResource == null) { if (properties == null || updatingResource == null) {
return null; return null;
} else { } else {
String source = updatingResource.get(name); String[] source = updatingResource.get(name);
if (source == null || source.equals(UNKNOWN_RESOURCE)) { if(source == null) {
return null; return null;
} else { } else {
return source; return Arrays.copyOf(source, source.length);
} }
} }
} }
@ -1702,11 +1771,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
protected synchronized Properties getProps() { protected synchronized Properties getProps() {
if (properties == null) { if (properties == null) {
properties = new Properties(); properties = new Properties();
HashMap<String, String[]> backup =
new HashMap<String, String[]>(updatingResource);
loadResources(properties, resources, quietmode); loadResources(properties, resources, quietmode);
if (overlay!= null) { if (overlay!= null) {
properties.putAll(overlay); properties.putAll(overlay);
for (Map.Entry<Object,Object> item: overlay.entrySet()) { for (Map.Entry<Object,Object> item: overlay.entrySet()) {
updatingResource.put((String) item.getKey(), UNKNOWN_RESOURCE); String key = (String)item.getKey();
updatingResource.put(key, backup.get(key));
} }
} }
} }
@ -1752,25 +1824,25 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
} }
private void loadResources(Properties properties, private void loadResources(Properties properties,
ArrayList resources, ArrayList<Resource> resources,
boolean quiet) { boolean quiet) {
if(loadDefaults) { if(loadDefaults) {
for (String resource : defaultResources) { for (String resource : defaultResources) {
loadResource(properties, resource, quiet); loadResource(properties, new Resource(resource), quiet);
} }
//support the hadoop-site.xml as a deprecated case //support the hadoop-site.xml as a deprecated case
if(getResource("hadoop-site.xml")!=null) { if(getResource("hadoop-site.xml")!=null) {
loadResource(properties, "hadoop-site.xml", quiet); loadResource(properties, new Resource("hadoop-site.xml"), quiet);
} }
} }
for (Object resource : resources) { for (Resource resource : resources) {
loadResource(properties, resource, quiet); loadResource(properties, resource, quiet);
} }
} }
private void loadResource(Properties properties, Object name, boolean quiet) { private void loadResource(Properties properties, Resource wrapper, boolean quiet) {
try { try {
DocumentBuilderFactory docBuilderFactory DocumentBuilderFactory docBuilderFactory
= DocumentBuilderFactory.newInstance(); = DocumentBuilderFactory.newInstance();
@ -1791,26 +1863,29 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
Document doc = null; Document doc = null;
Element root = null; Element root = null;
if (name instanceof URL) { // an URL resource Object resource = wrapper.getResource();
URL url = (URL)name; String name = wrapper.getName();
if (resource instanceof URL) { // an URL resource
URL url = (URL)resource;
if (url != null) { if (url != null) {
if (!quiet) { if (!quiet) {
LOG.info("parsing " + url); LOG.info("parsing " + url);
} }
doc = builder.parse(url.toString()); doc = builder.parse(url.toString());
} }
} else if (name instanceof String) { // a CLASSPATH resource } else if (resource instanceof String) { // a CLASSPATH resource
URL url = getResource((String)name); URL url = getResource((String)resource);
if (url != null) { if (url != null) {
if (!quiet) { if (!quiet) {
LOG.info("parsing " + url); LOG.info("parsing " + url);
} }
doc = builder.parse(url.toString()); doc = builder.parse(url.toString());
} }
} else if (name instanceof Path) { // a file resource } else if (resource instanceof Path) { // a file resource
// Can't use FileSystem API or we get an infinite loop // Can't use FileSystem API or we get an infinite loop
// since FileSystem uses Configuration API. Use java.io.File instead. // since FileSystem uses Configuration API. Use java.io.File instead.
File file = new File(((Path)name).toUri().getPath()) File file = new File(((Path)resource).toUri().getPath())
.getAbsoluteFile(); .getAbsoluteFile();
if (file.exists()) { if (file.exists()) {
if (!quiet) { if (!quiet) {
@ -1823,20 +1898,20 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
in.close(); in.close();
} }
} }
} else if (name instanceof InputStream) { } else if (resource instanceof InputStream) {
try { try {
doc = builder.parse((InputStream)name); doc = builder.parse((InputStream)resource);
} finally { } finally {
((InputStream)name).close(); ((InputStream)resource).close();
} }
} else if (name instanceof Element) { } else if (resource instanceof Element) {
root = (Element)name; root = (Element)resource;
} }
if (doc == null && root == null) { if (doc == null && root == null) {
if (quiet) if (quiet)
return; return;
throw new RuntimeException(name + " not found"); throw new RuntimeException(resource + " not found");
} }
if (root == null) { if (root == null) {
@ -1851,7 +1926,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
continue; continue;
Element prop = (Element)propNode; Element prop = (Element)propNode;
if ("configuration".equals(prop.getTagName())) { if ("configuration".equals(prop.getTagName())) {
loadResource(properties, prop, quiet); loadResource(properties, new Resource(prop, name), quiet);
continue; continue;
} }
if (!"property".equals(prop.getTagName())) if (!"property".equals(prop.getTagName()))
@ -1860,6 +1935,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
String attr = null; String attr = null;
String value = null; String value = null;
boolean finalParameter = false; boolean finalParameter = false;
LinkedList<String> source = new LinkedList<String>();
for (int j = 0; j < fields.getLength(); j++) { for (int j = 0; j < fields.getLength(); j++) {
Node fieldNode = fields.item(j); Node fieldNode = fields.item(j);
if (!(fieldNode instanceof Element)) if (!(fieldNode instanceof Element))
@ -1871,7 +1947,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
value = ((Text)field.getFirstChild()).getData(); value = ((Text)field.getFirstChild()).getData();
if ("final".equals(field.getTagName()) && field.hasChildNodes()) if ("final".equals(field.getTagName()) && field.hasChildNodes())
finalParameter = "true".equals(((Text)field.getFirstChild()).getData()); finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
if ("source".equals(field.getTagName()) && field.hasChildNodes())
source.add(((Text)field.getFirstChild()).getData());
} }
source.add(name);
// Ignore this parameter if it has already been marked as 'final' // Ignore this parameter if it has already been marked as 'final'
if (attr != null) { if (attr != null) {
@ -1880,11 +1959,13 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
keyInfo.accessed = false; keyInfo.accessed = false;
for (String key:keyInfo.newKeys) { for (String key:keyInfo.newKeys) {
// update new keys with deprecated key's value // update new keys with deprecated key's value
loadProperty(properties, name, key, value, finalParameter); loadProperty(properties, name, key, value, finalParameter,
source.toArray(new String[source.size()]));
} }
} }
else { else {
loadProperty(properties, name, attr, value, finalParameter); loadProperty(properties, name, attr, value, finalParameter,
source.toArray(new String[source.size()]));
} }
} }
} }
@ -1904,12 +1985,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
} }
} }
private void loadProperty(Properties properties, Object name, String attr, private void loadProperty(Properties properties, String name, String attr,
String value, boolean finalParameter) { String value, boolean finalParameter, String[] source) {
if (value != null) { if (value != null) {
if (!finalParameters.contains(attr)) { if (!finalParameters.contains(attr)) {
properties.setProperty(attr, value); properties.setProperty(attr, value);
updatingResource.put(attr, name.toString()); updatingResource.put(attr, source);
} else if (!value.equals(properties.getProperty(attr))) { } else if (!value.equals(properties.getProperty(attr))) {
LOG.warn(name+":an attempt to override final parameter: "+attr LOG.warn(name+":an attempt to override final parameter: "+attr
+"; Ignoring."); +"; Ignoring.");
@ -1981,11 +2062,6 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
Element propNode = doc.createElement("property"); Element propNode = doc.createElement("property");
conf.appendChild(propNode); conf.appendChild(propNode);
if (updatingResource != null) {
Comment commentNode = doc.createComment(
"Loaded from " + updatingResource.get(name));
propNode.appendChild(commentNode);
}
Element nameNode = doc.createElement("name"); Element nameNode = doc.createElement("name");
nameNode.appendChild(doc.createTextNode(name)); nameNode.appendChild(doc.createTextNode(name));
propNode.appendChild(nameNode); propNode.appendChild(nameNode);
@ -1994,6 +2070,17 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
valueNode.appendChild(doc.createTextNode(value)); valueNode.appendChild(doc.createTextNode(value));
propNode.appendChild(valueNode); propNode.appendChild(valueNode);
if (updatingResource != null) {
String[] sources = updatingResource.get(name);
if(sources != null) {
for(String s : sources) {
Element sourceNode = doc.createElement("source");
sourceNode.appendChild(doc.createTextNode(s));
propNode.appendChild(sourceNode);
}
}
}
conf.appendChild(doc.createTextNode("\n")); conf.appendChild(doc.createTextNode("\n"));
} }
return doc; return doc;
@ -2026,8 +2113,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
config.get((String) item.getKey())); config.get((String) item.getKey()));
dumpGenerator.writeBooleanField("isFinal", dumpGenerator.writeBooleanField("isFinal",
config.finalParameters.contains(item.getKey())); config.finalParameters.contains(item.getKey()));
dumpGenerator.writeStringField("resource", String[] resources = config.updatingResource.get(item.getKey());
config.updatingResource.get(item.getKey())); String resource = UNKNOWN_RESOURCE;
if(resources != null && resources.length > 0) {
resource = resources[0];
}
dumpGenerator.writeStringField("resource", resource);
dumpGenerator.writeEndObject(); dumpGenerator.writeEndObject();
} }
} }
@ -2067,7 +2158,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
toString(resources, sb); toString(resources, sb);
return sb.toString(); return sb.toString();
} }
private <T> void toString(List<T> resources, StringBuilder sb) { private <T> void toString(List<T> resources, StringBuilder sb) {
ListIterator<T> i = resources.listIterator(); ListIterator<T> i = resources.listIterator();
while (i.hasNext()) { while (i.hasNext()) {
@ -2104,8 +2195,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
clear(); clear();
int size = WritableUtils.readVInt(in); int size = WritableUtils.readVInt(in);
for(int i=0; i < size; ++i) { for(int i=0; i < size; ++i) {
set(org.apache.hadoop.io.Text.readString(in), String key = org.apache.hadoop.io.Text.readString(in);
org.apache.hadoop.io.Text.readString(in)); String value = org.apache.hadoop.io.Text.readString(in);
set(key, value);
String sources[] = WritableUtils.readCompressedStringArray(in);
updatingResource.put(key, sources);
} }
} }
@ -2116,6 +2210,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
for(Map.Entry<Object, Object> item: props.entrySet()) { for(Map.Entry<Object, Object> item: props.entrySet()) {
org.apache.hadoop.io.Text.writeString(out, (String) item.getKey()); org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
org.apache.hadoop.io.Text.writeString(out, (String) item.getValue()); org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
WritableUtils.writeCompressedStringArray(out,
updatingResource.get(item.getKey()));
} }
} }

View File

@ -268,7 +268,8 @@ public class GenericOptionsParser {
} }
if (line.hasOption("jt")) { if (line.hasOption("jt")) {
conf.set("mapred.job.tracker", line.getOptionValue("jt")); conf.set("mapred.job.tracker", line.getOptionValue("jt"),
"from -jt command line option");
} }
if (line.hasOption("conf")) { if (line.hasOption("conf")) {
String[] values = line.getOptionValues("conf"); String[] values = line.getOptionValues("conf");
@ -278,7 +279,8 @@ public class GenericOptionsParser {
} }
if (line.hasOption("libjars")) { if (line.hasOption("libjars")) {
conf.set("tmpjars", conf.set("tmpjars",
validateFiles(line.getOptionValue("libjars"), conf)); validateFiles(line.getOptionValue("libjars"), conf),
"from -libjars command line option");
//setting libjars in client classpath //setting libjars in client classpath
URL[] libjars = getLibJars(conf); URL[] libjars = getLibJars(conf);
if(libjars!=null && libjars.length>0) { if(libjars!=null && libjars.length>0) {
@ -290,18 +292,20 @@ public class GenericOptionsParser {
} }
if (line.hasOption("files")) { if (line.hasOption("files")) {
conf.set("tmpfiles", conf.set("tmpfiles",
validateFiles(line.getOptionValue("files"), conf)); validateFiles(line.getOptionValue("files"), conf),
"from -files command line option");
} }
if (line.hasOption("archives")) { if (line.hasOption("archives")) {
conf.set("tmparchives", conf.set("tmparchives",
validateFiles(line.getOptionValue("archives"), conf)); validateFiles(line.getOptionValue("archives"), conf),
"from -archives command line option");
} }
if (line.hasOption('D')) { if (line.hasOption('D')) {
String[] property = line.getOptionValues('D'); String[] property = line.getOptionValues('D');
for(String prop : property) { for(String prop : property) {
String[] keyval = prop.split("=", 2); String[] keyval = prop.split("=", 2);
if (keyval.length == 2) { if (keyval.length == 2) {
conf.set(keyval[0], keyval[1]); conf.set(keyval[0], keyval[1], "from command line");
} }
} }
} }
@ -320,7 +324,7 @@ public class GenericOptionsParser {
LOG.debug("setting conf tokensFile: " + fileName); LOG.debug("setting conf tokensFile: " + fileName);
} }
conf.set("mapreduce.job.credentials.json", localFs.makeQualified(p) conf.set("mapreduce.job.credentials.json", localFs.makeQualified(p)
.toString()); .toString(), "from -tokenCacheFile command line option");
} }
} }

View File

@ -64,7 +64,7 @@ public class TestConfServlet extends TestCase {
String resource = (String)propertyInfo.get("resource"); String resource = (String)propertyInfo.get("resource");
System.err.println("k: " + key + " v: " + val + " r: " + resource); System.err.println("k: " + key + " v: " + val + " r: " + resource);
if (TEST_KEY.equals(key) && TEST_VAL.equals(val) if (TEST_KEY.equals(key) && TEST_VAL.equals(val)
&& Configuration.UNKNOWN_RESOURCE.equals(resource)) { && "programatically".equals(resource)) {
foundSetting = true; foundSetting = true;
} }
} }

View File

@ -168,7 +168,8 @@ public class TestConfiguration extends TestCase {
appendProperty(name, val, false); appendProperty(name, val, false);
} }
void appendProperty(String name, String val, boolean isFinal) void appendProperty(String name, String val, boolean isFinal,
String ... sources)
throws IOException { throws IOException {
out.write("<property>"); out.write("<property>");
out.write("<name>"); out.write("<name>");
@ -180,6 +181,11 @@ public class TestConfiguration extends TestCase {
if (isFinal) { if (isFinal) {
out.write("<final>true</final>"); out.write("<final>true</final>");
} }
for(String s : sources) {
out.write("<source>");
out.write(s);
out.write("</source>");
}
out.write("</property>\n"); out.write("</property>\n");
} }
@ -671,16 +677,38 @@ public class TestConfiguration extends TestCase {
Path fileResource = new Path(CONFIG); Path fileResource = new Path(CONFIG);
conf.addResource(fileResource); conf.addResource(fileResource);
conf.set("fs.defaultFS", "value"); conf.set("fs.defaultFS", "value");
String [] sources = conf.getPropertySources("test.foo");
assertEquals(1, sources.length);
assertEquals( assertEquals(
"Resource string returned for a file-loaded property" + "Resource string returned for a file-loaded property" +
" must be a proper absolute path", " must be a proper absolute path",
fileResource, fileResource,
new Path(conf.getPropertySource("test.foo"))); new Path(sources[0]));
assertEquals("Resource string returned for a set() property must be null", assertArrayEquals("Resource string returned for a set() property must be " +
null, "\"programatically\"",
conf.getPropertySource("fs.defaultFS")); new String[]{"programatically"},
conf.getPropertySources("fs.defaultFS"));
assertEquals("Resource string returned for an unset property must be null", assertEquals("Resource string returned for an unset property must be null",
null, conf.getPropertySource("fs.defaultFoo")); null, conf.getPropertySources("fs.defaultFoo"));
}
public void testMultiplePropertySource() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.foo", "bar", false, "a", "b", "c");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
String [] sources = conf.getPropertySources("test.foo");
assertEquals(4, sources.length);
assertEquals("a", sources[0]);
assertEquals("b", sources[1]);
assertEquals("c", sources[2]);
assertEquals(
"Resource string returned for a file-loaded property" +
" must be a proper absolute path",
fileResource,
new Path(sources[3]));
} }
public void testSocketAddress() { public void testSocketAddress() {
@ -929,7 +957,7 @@ public class TestConfiguration extends TestCase {
confDump.put(prop.getKey(), prop); confDump.put(prop.getKey(), prop);
} }
assertEquals("value5",confDump.get("test.key6").getValue()); assertEquals("value5",confDump.get("test.key6").getValue());
assertEquals("Unknown", confDump.get("test.key4").getResource()); assertEquals("programatically", confDump.get("test.key4").getResource());
outWriter.close(); outWriter.close();
} }