Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3427. Fix streaming unit tests broken after mavenization. Contributed by Hitesh Shah. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1239391 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2012-02-01 23:05:14 +00:00
parent 42d1c9b251
commit ab75b8b420
4 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -645,6 +645,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3696. MR job via oozie does not work on hadoop 23. MAPREDUCE-3696. MR job via oozie does not work on hadoop 23.
(John George via mahadev) (John George via mahadev)
MAPREDUCE-3427. Fix streaming unit tests broken after mavenization.
(Hitesh Shah via acmurthy)
Release 0.23.0 - 2011-11-01 Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

2
hadoop-tools/hadoop-streaming/pom.xml
View File

@ -29,7 +29,7 @@
<properties> <properties>
<hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir> <hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir>
<test.exclude.pattern>%regex[.*(TestStreamingBadRecords|TestStreamingStatus|TestUlimit).*]</test.exclude.pattern> <test.exclude.pattern>%regex[.*(TestStreamingStatus).*]</test.exclude.pattern>
</properties> </properties>
<dependencies> <dependencies>

14
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java
View File

@ -154,6 +154,10 @@ public class TestStreamingBadRecords extends ClusterMapReduceTestCase
} }
} }
/*
* Disable test as skipping bad records not supported in 0.23
*/
/*
public void testSkip() throws Exception { public void testSkip() throws Exception {
JobConf clusterConf = createJobConf(); JobConf clusterConf = createJobConf();
createInput(); createInput();
@ -195,7 +199,12 @@ public class TestStreamingBadRecords extends ClusterMapReduceTestCase
//validate that there is no skip directory as it has been set to "none" //validate that there is no skip directory as it has been set to "none"
assertTrue(SkipBadRecords.getSkipOutputPath(job.jobConf_)==null); assertTrue(SkipBadRecords.getSkipOutputPath(job.jobConf_)==null);
} }
*/
/*
* Disable test as skipping bad records not supported in 0.23
*/
/*
public void testNarrowDown() throws Exception { public void testNarrowDown() throws Exception {
createInput(); createInput();
JobConf clusterConf = createJobConf(); JobConf clusterConf = createJobConf();
@ -231,6 +240,11 @@ public class TestStreamingBadRecords extends ClusterMapReduceTestCase
validateOutput(job.running_, true); validateOutput(job.running_, true);
assertTrue(SkipBadRecords.getSkipOutputPath(job.jobConf_)!=null); assertTrue(SkipBadRecords.getSkipOutputPath(job.jobConf_)!=null);
} }
*/
public void testNoOp() {
// Added to avoid warnings when running this disabled test
}
static class App{ static class App{
boolean isReducer; boolean isReducer;

4
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java
View File

@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.MapReduceTestUtil;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.*; import static org.junit.Assert.*;
@ -52,7 +53,6 @@ public class TestUlimit {
private static String SET_MEMORY_LIMIT = "786432"; // 768MB private static String SET_MEMORY_LIMIT = "786432"; // 768MB
String[] genArgs(String memLimit) { String[] genArgs(String memLimit) {
String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name"); String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name");
return new String[] { return new String[] {
"-input", inputPath.toString(), "-input", inputPath.toString(),
@ -63,7 +63,6 @@ public class TestUlimit {
"-jobconf", MRJobConfig.NUM_MAPS + "=1", "-jobconf", MRJobConfig.NUM_MAPS + "=1",
"-jobconf", JobConf.MAPRED_MAP_TASK_ULIMIT + "=" + memLimit, "-jobconf", JobConf.MAPRED_MAP_TASK_ULIMIT + "=" + memLimit,
"-jobconf", strNamenode, "-jobconf", strNamenode,
"-jobconf", strJobtracker,
"-jobconf", "stream.tmpdir=" + "-jobconf", "stream.tmpdir=" +
System.getProperty("test.build.data","/tmp"), System.getProperty("test.build.data","/tmp"),
"-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR, "-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR,
@ -79,6 +78,7 @@ public class TestUlimit {
* is expected to be a failure. * is expected to be a failure.
*/ */
@Test @Test
@Ignore
public void testCommandLine() { public void testCommandLine() {
if (UtilTest.isCygwin()) { if (UtilTest.isCygwin()) {
return; return;