diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index f4bf7bbafe4..d1f64067be5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -381,6 +381,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4267. mavenize pipes (tgraves via bobby) + MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby + via tgraves) + OPTIMIZATIONS MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 10eb68dbf85..982c89c4c94 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -1561,7 +1561,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, Path confPath = getConfFile(); FileContext fc = FileContext.getFileContext(confPath.toUri(), conf); Configuration jobConf = new Configuration(false); - jobConf.addResource(fc.open(confPath)); + jobConf.addResource(fc.open(confPath), confPath.toString()); return jobConf; } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java index 36f202e66bf..fd2edb45701 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java @@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock { tr(). th(_TH, "key"). th(_TH, "value"). + th(_TH, "source chain"). _(). _(). tbody(); for (ConfEntryInfo entry : info.getProperties()) { + StringBuffer buffer = new StringBuffer(); + String[] sources = entry.getSource(); + //Skip the last entry, because it is always the same HDFS file, and + // output them in reverse order so most recent is output first + boolean first = true; + for(int i = (sources.length - 2); i >= 0; i--) { + if(!first) { + // \u2B05 is an arrow <-- + buffer.append(" \u2B05 "); + } + first = false; + buffer.append(sources[i]); + } tbody. tr(). td(entry.getName()). td(entry.getValue()). + td(buffer.toString()). _(); } tbody._(). @@ -93,6 +108,7 @@ public class ConfBlock extends HtmlBlock { tr(). th().input("search_init").$type(InputType.text).$name("key").$value("key")._()._(). th().input("search_init").$type(InputType.text).$name("value").$value("value")._()._(). + th().input("search_init").$type(InputType.text).$name("source chain").$value("source chain")._()._(). _(). _(). _(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java index 5cfa40aa1aa..bfd50ef0db4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java @@ -27,13 +27,19 @@ public class ConfEntryInfo { protected String name; protected String value; + protected String[] source; public ConfEntryInfo() { } public ConfEntryInfo(String key, String value) { + this(key, value, null); + } + + public ConfEntryInfo(String key, String value, String[] source) { this.name = key; this.value = value; + this.source = source; } public String getName() { @@ -43,4 +49,8 @@ public class ConfEntryInfo { public String getValue() { return this.value; } + + public String[] getSource() { + return source; + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java index fa971a749e4..a05c3176f9c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java @@ -46,7 +46,8 @@ public class ConfInfo { Configuration jobConf = job.loadConfFile(); this.path = job.getConfFile().toString(); for (Map.Entry entry : jobConf) { - this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue())); + this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue(), + jobConf.getPropertySources(entry.getKey()))); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index 7f1d8200327..a9e4e4910c0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -603,7 +603,7 @@ public class MockJobs extends MockApps { public Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(configFile.toUri(), conf); Configuration jobConf = new Configuration(false); - jobConf.addResource(fc.open(configFile)); + jobConf.addResource(fc.open(configFile), configFile.toString()); return jobConf; } }; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 6447109916b..743dc0920a9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -336,7 +336,7 @@ public class HistoryFileManager extends AbstractService { public synchronized Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(confFile.toUri(), conf); Configuration jobConf = new Configuration(false); - jobConf.addResource(fc.open(confFile)); + jobConf.addResource(fc.open(confFile), confFile.toString()); return jobConf; } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm index de51c8ad3fb..5b11736b8fe 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm @@ -1261,6 +1261,9 @@ History Server REST API's. *---------------+--------------+-------------------------------+ | value | string | The value of the configuration property | *---------------+--------------+-------------------------------+ +| source | string | The location this configuration object came from. If there is more then one of these it shows the history with the latest source at the end of the list. | +*---------------+--------------+-------------------------------+ + *** Response Examples @@ -1293,14 +1296,17 @@ History Server REST API's. { "value" : "/home/hadoop/hdfs/data", "name" : "dfs.datanode.data.dir" + "source" : ["hdfs-site.xml", "job.xml"] }, { "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer", "name" : "hadoop.http.filter.initializers" + "source" : ["programatically", "job.xml"] }, { "value" : "/home/hadoop/tmp", "name" : "mapreduce.cluster.temp.dir" + "source" : ["mapred-site.xml"] }, ... ] @@ -1335,14 +1341,19 @@ History Server REST API's. dfs.datanode.data.dir /home/hadoop/hdfs/data + hdfs-site.xml + job.xml hadoop.http.filter.initializers org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer + programatically + job.xml mapreduce.cluster.temp.dir /home/hadoop/tmp + mapred-site.xml ... diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm index ffd25c4763f..de0093c269c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm @@ -1296,6 +1296,8 @@ MapReduce Application Master REST API's. *---------------+--------------+-------------------------------+ | value | string | The value of the configuration property | *---------------+--------------+-------------------------------+ +| source | string | The location this configuration object came from. If there is more then one of these it shows the history with the latest source at the end of the list. | +*---------------+--------------+-------------------------------+ ** Response Examples @@ -1327,15 +1329,18 @@ MapReduce Application Master REST API's. "property" : [ { "value" : "/home/hadoop/hdfs/data", - "name" : "dfs.datanode.data.dir" + "name" : "dfs.datanode.data.dir", + "source" : ["hdfs-site.xml", "job.xml"] }, { "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer", "name" : "hadoop.http.filter.initializers" + "source" : ["programatically", "job.xml"] }, { "value" : "/home/hadoop/tmp", "name" : "mapreduce.cluster.temp.dir" + "source" : ["mapred-site.xml"] }, ... ] @@ -1370,14 +1375,19 @@ MapReduce Application Master REST API's. dfs.datanode.data.dir /home/hadoop/hdfs/data + hdfs-site.xml + job.xml hadoop.http.filter.initializers org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer + programatically + job.xml mapreduce.cluster.temp.dir /home/hadoop/tmp + mapred-site.xml ...