MAPREDUCE-6022. map_input_file is missing from streaming job

environment. Contributed by Jason Lowe.
(cherry picked from commit b056048114)
This commit is contained in:
Kihwal Lee 2014-10-29 12:30:08 -05:00
parent e88832dfb3
commit 45b6dea95f
4 changed files with 35 additions and 1 deletions

View File

@ -558,6 +558,32 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
}
/**
* Sets all deprecated properties that are not currently set but have a
* corresponding new property that is set. Useful for iterating the
* properties when all deprecated properties for currently set properties
* need to be present.
*/
public void setDeprecatedProperties() {
DeprecationContext deprecations = deprecationContext.get();
Properties props = getProps();
Properties overlay = getOverlay();
for (Map.Entry<String, DeprecatedKeyInfo> entry :
deprecations.getDeprecatedKeyMap().entrySet()) {
String depKey = entry.getKey();
if (!overlay.contains(depKey)) {
for (String newKey : entry.getValue().newKeys) {
String val = overlay.getProperty(newKey);
if (val != null) {
props.setProperty(depKey, val);
overlay.setProperty(depKey, val);
break;
}
}
}
}
}
/**
* Checks for the presence of the property <code>name</code> in the
* deprecation map. Returns the first of the list of new keys if present

View File

@ -231,6 +231,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6142. Fixed test failures in TestJobHistoryEventHandler and
TestMRTimelineEventHandling. (Zhijie Shen via vinodkv)
MAPREDUCE-6022. map_input_file is missing from streaming job environment.
(jlowe via kihwal)
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES

View File

@ -235,7 +235,9 @@ public abstract class PipeMapRed {
}
}
void addJobConfToEnvironment(JobConf conf, Properties env) {
void addJobConfToEnvironment(JobConf jobconf, Properties env) {
JobConf conf = new JobConf(jobconf);
conf.setDeprecatedProperties();
Iterator it = conf.iterator();
while (it.hasNext()) {
Map.Entry en = (Map.Entry) it.next();

View File

@ -57,6 +57,9 @@ public class TrApp
// the FileSplit context properties are not available in local hadoop..
// so can't check them in this test.
// verify some deprecated properties appear for older stream jobs
expect("map_input_file", env.getProperty("mapreduce_map_input_file"));
expect("map_input_length", env.getProperty("mapreduce_map_input_length"));
}
// this runs in a subprocess; won't use JUnit's assertTrue()