Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-5113. Streaming input/output types are ignored with java mapper/reducer. (sandyr via tucu) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1466870 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2013-04-11 13:04:29 +00:00
parent cdbfce55df
commit a7d2d7cfc7
4 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -120,6 +120,9 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-5086. MR app master deletes staging dir when sent a reboot MAPREDUCE-5086. MR app master deletes staging dir when sent a reboot
command from the RM. (Jian He via jlowe) command from the RM. (Jian He via jlowe)
MAPREDUCE-5113. Streaming input/output types are ignored with java
mapper/reducer. (sandyr via tucu)
Release 2.0.4-alpha - UNRELEASED Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

4
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
View File

@ -875,7 +875,7 @@ public class StreamJob implements Tool {
IdentifierResolver.TEXT_ID)); IdentifierResolver.TEXT_ID));
jobConf_.setClass("stream.map.output.reader.class", jobConf_.setClass("stream.map.output.reader.class",
idResolver.getOutputReaderClass(), OutputReader.class); idResolver.getOutputReaderClass(), OutputReader.class);
if (isMapperACommand) { if (isMapperACommand || jobConf_.get("stream.map.output") != null) {
// if mapper is a command, then map output key/value classes come from the // if mapper is a command, then map output key/value classes come from the
// idResolver // idResolver
jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass()); jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass());
@ -891,7 +891,7 @@ public class StreamJob implements Tool {
IdentifierResolver.TEXT_ID)); IdentifierResolver.TEXT_ID));
jobConf_.setClass("stream.reduce.output.reader.class", jobConf_.setClass("stream.reduce.output.reader.class",
idResolver.getOutputReaderClass(), OutputReader.class); idResolver.getOutputReaderClass(), OutputReader.class);
if (isReducerACommand) { if (isReducerACommand || jobConf_.get("stream.reduce.output") != null) {
// if reducer is a command, then output key/value classes come from the // if reducer is a command, then output key/value classes come from the
// idResolver // idResolver
jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass()); jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass());

12
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
View File

@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.MRJobConfig;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -177,6 +178,17 @@ public class TestStreamingOutputKeyValueTypes extends TestStreaming {
args.add("0"); args.add("0");
super.testCommandLine(); super.testCommandLine();
} }
@Test
public void testDefaultToIdentityReducer() throws Exception {
args.add("-mapper");args.add(map);
args.add("-jobconf");
args.add("mapreduce.task.files.preserve.failedtasks=true");
args.add("-jobconf");
args.add("stream.tmpdir="+System.getProperty("test.build.data","/tmp"));
args.add("-inputformat");args.add(TextInputFormat.class.getName());
super.testCommandLine();
}
@Override @Override
@Test @Test

1
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
View File

@ -43,7 +43,6 @@ public class TrApp
// Note the dots translated to underscore: // Note the dots translated to underscore:
// property names have been escaped in PipeMapRed.safeEnvVarName() // property names have been escaped in PipeMapRed.safeEnvVarName()
expectDefined("mapreduce_cluster_local_dir"); expectDefined("mapreduce_cluster_local_dir");
expect("mapred_output_format_class", "org.apache.hadoop.mapred.TextOutputFormat");
expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text"); expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text");
expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text"); expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text");