From 42e93829e5310f3cbd905384cd0529f8fffa887f Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Tue, 11 Oct 2011 00:26:41 +0000 Subject: [PATCH] MAPREDUCE-3158. Fix test failures in MRv1 due to default framework being set to yarn. Contributed by Hitesh Shah. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1181310 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../apache/hadoop/mapred/LocalContainerLauncher.java | 2 ++ .../main/java/org/apache/hadoop/mapred/ReduceTask.java | 6 ++---- .../java/org/apache/hadoop/mapreduce/MRConfig.java | 1 + .../org/apache/hadoop/mapreduce/v2/TestMRJobs.java | 2 +- .../hadoop/mapred/LocalClientProtocolProvider.java | 2 +- .../org/apache/hadoop/mapred/HadoopTestCase.java | 10 +++++++++- .../mapred/org/apache/hadoop/mapred/TestCollect.java | 2 ++ .../org/apache/hadoop/mapred/TestComparators.java | 3 +++ .../org/apache/hadoop/mapred/TestFileOutputFormat.java | 3 +++ .../apache/hadoop/mapred/TestJavaSerialization.java | 5 +++++ .../org/apache/hadoop/mapred/TestMapOutputType.java | 3 +++ .../mapred/org/apache/hadoop/mapred/TestMapRed.java | 5 +++++ .../hadoop/mapred/TestReduceFetchFromPartialMem.java | 1 + .../mapreduce/TestClientProtocolProviderImpls.java | 2 +- .../org/apache/hadoop/mapreduce/TestMapCollection.java | 2 ++ 16 files changed, 44 insertions(+), 8 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ec99a75da07..0124fe22bf8 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1561,6 +1561,9 @@ Release 0.23.0 - Unreleased MAPREDUCE-3154. Fix JobSubmitter to check for output specs before copying job submission files to fail fast. (Abhijit Suresh Shingate via acmurthy) + MAPREDUCE-3158. Fix test failures in MRv1 due to default framework being + set to yarn. (Hitesh Shah) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java index c3508f86944..419b49121b2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java @@ -314,6 +314,8 @@ public class LocalContainerLauncher extends AbstractService implements ReduceTask reduce = (ReduceTask)task; // a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner: + // set framework name to local to make task local + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); conf.set(MRConfig.MASTER_ADDRESS, "local"); // bypass shuffle reduce.run(conf, umbilical); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java index dafa6ce7dcb..4017113bb65 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java @@ -342,11 +342,9 @@ public class ReduceTask extends Task { RawKeyValueIterator rIter = null; boolean isLocal = false; - // local iff framework == classic && master address == local + // local iff framework == local String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { - isLocal = "local".equals(job.get(MRConfig.MASTER_ADDRESS, "local")); - } + isLocal = framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME); if (!isLocal) { Class combinerClass = conf.getCombinerClass(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java index 2a9823c3854..4516cb9eda4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java @@ -67,6 +67,7 @@ public interface MRConfig { public static final String FRAMEWORK_NAME = "mapreduce.framework.name"; public static final String CLASSIC_FRAMEWORK_NAME = "classic"; public static final String YARN_FRAMEWORK_NAME = "yarn"; + public static final String LOCAL_FRAMEWORK_NAME = "local"; public static final String TASK_LOCAL_OUTPUT_CLASS = "mapreduce.task.local.output.class"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index 437ae135713..817a7611753 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -135,7 +135,7 @@ public class TestMRJobs { } Configuration sleepConf = new Configuration(mrCluster.getConfig()); - // set master address to local to test that local mode applied iff framework == classic and master_address == local + // set master address to local to test that local mode applied iff framework == local sleepConf.set(MRConfig.MASTER_ADDRESS, "local"); SleepJob sleepJob = new SleepJob(); diff --git a/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java b/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java index d09b222ee9b..2b6ad992ceb 100644 --- a/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java +++ b/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java @@ -34,7 +34,7 @@ public class LocalClientProtocolProvider extends ClientProtocolProvider { @Override public ClientProtocol create(Configuration conf) throws IOException { String framework = conf.get(MRConfig.FRAMEWORK_NAME); - if (framework != null && !framework.equals("local")) { + if (framework != null && !framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME)) { return null; } String tracker = conf.get(JTConfig.JT_IPC_ADDRESS, "local"); diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java index b1623d74253..c102e8f8626 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java @@ -22,6 +22,7 @@ import junit.framework.TestCase; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.MRConfig; import java.io.File; import java.io.IOException; @@ -200,7 +201,14 @@ public abstract class HadoopTestCase extends TestCase { * @return configuration that works on the testcase Hadoop instance */ protected JobConf createJobConf() { - return (localMR) ? new JobConf() : mrCluster.createJobConf(); + if (localMR) { + JobConf conf = new JobConf(); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); + return conf; + } + else { + return mrCluster.createJobConf(); + } } } diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java index f50c7d887d1..4bd20d54ad5 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCollect.java @@ -20,6 +20,7 @@ package org.apache.hadoop.mapred; import org.apache.hadoop.fs.*; import org.apache.hadoop.io.*; import org.apache.hadoop.mapred.UtilsForTests.RandomInputFormat; +import org.apache.hadoop.mapreduce.MRConfig; import junit.framework.TestCase; import java.io.*; @@ -120,6 +121,7 @@ public class TestCollect extends TestCase conf.setOutputValueClass(IntWritable.class); FileOutputFormat.setOutputPath(conf, OUTPUT_DIR); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); conf.setMapperClass(Map.class); conf.setReducerClass(Reduce.class); conf.setNumMapTasks(1); diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java index cc2c1db7e81..1cef5cb42f2 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestComparators.java @@ -19,6 +19,8 @@ package org.apache.hadoop.mapred; import org.apache.hadoop.fs.*; import org.apache.hadoop.io.*; +import org.apache.hadoop.mapreduce.MRConfig; + import junit.framework.TestCase; import java.io.*; import java.util.*; @@ -305,6 +307,7 @@ public class TestComparators extends TestCase conf.setMapOutputValueClass(IntWritable.class); // set up two map jobs, so we can test merge phase in Reduce also conf.setNumMapTasks(2); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); conf.setOutputFormat(SequenceFileOutputFormat.class); if (!fs.mkdirs(testdir)) { diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java index eeaf48bee64..81b53cc3b32 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.MRConfig; import java.io.DataOutputStream; import java.io.IOException; @@ -77,6 +78,8 @@ public class TestFileOutputFormat extends HadoopTestCase { conf.setMapperClass(TestMap.class); conf.setReducerClass(TestReduce.class); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); + FileInputFormat.setInputPaths(conf, inDir); FileOutputFormat.setOutputPath(conf, outDir); diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java index 1ad797bca0e..3ab6f561bd8 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJavaSerialization.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.serializer.JavaSerializationComparator; +import org.apache.hadoop.mapreduce.MRConfig; public class TestJavaSerialization extends TestCase { @@ -109,6 +110,8 @@ public class TestJavaSerialization extends TestCase { conf.setMapperClass(WordCountMapper.class); conf.setReducerClass(SumReducer.class); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); + FileInputFormat.setInputPaths(conf, INPUT_DIR); FileOutputFormat.setOutputPath(conf, OUTPUT_DIR); @@ -155,6 +158,8 @@ public class TestJavaSerialization extends TestCase { conf.setMapperClass(WordCountMapper.class); conf.setReducerClass(SumReducer.class); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); + FileInputFormat.setInputPaths(conf, INPUT_DIR); FileOutputFormat.setOutputPath(conf, OUTPUT_DIR); diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java index 9e74ab1336c..d11d7bc30b4 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapOutputType.java @@ -20,6 +20,7 @@ package org.apache.hadoop.mapred; import org.apache.hadoop.fs.*; import org.apache.hadoop.io.*; import org.apache.hadoop.mapred.lib.*; +import org.apache.hadoop.mapreduce.MRConfig; import junit.framework.TestCase; import java.io.*; import java.util.*; @@ -90,6 +91,8 @@ public class TestMapOutputType extends TestCase conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(Text.class); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); + conf.setOutputFormat(SequenceFileOutputFormat.class); if (!fs.mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.toString()); diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java index afc0cdc0fa9..3f7a6f7e3b4 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java @@ -43,6 +43,7 @@ import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.hadoop.mapred.lib.IdentityReducer; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.junit.Test; @@ -351,6 +352,7 @@ public class TestMapRed extends Configured implements Tool { conf.setInputFormat(SequenceFileInputFormat.class); conf.setOutputFormat(SequenceFileOutputFormat.class); conf.setNumReduceTasks(1); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); JobClient.runJob(conf); @@ -382,6 +384,7 @@ public class TestMapRed extends Configured implements Tool { conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(Text.class); conf.setOutputFormat(SequenceFileOutputFormat.class); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); if (includeCombine) { conf.setCombinerClass(IdentityReducer.class); } @@ -445,6 +448,7 @@ public class TestMapRed extends Configured implements Tool { } else { conf = new JobConf(getConf()); } + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); conf.setJarByClass(TestMapRed.class); int countsToGo = counts; int dist[] = new int[range]; @@ -737,6 +741,7 @@ public class TestMapRed extends Configured implements Tool { conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(Text.class); conf.setOutputFormat(SequenceFileOutputFormat.class); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); if (!fs.mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.toString()); } diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java index 656c6c8c66f..37c78054f5f 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java @@ -30,6 +30,7 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.mapreduce.TaskCounter; +import org.apache.hadoop.mapreduce.MRConfig; import java.io.DataInput; import java.io.DataOutput; diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java index a9044e24308..89eecb67b8f 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java @@ -43,7 +43,7 @@ public class TestClientProtocolProviderImpls extends TestCase { } try { - conf.set(MRConfig.FRAMEWORK_NAME, "local"); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); conf.set(JTConfig.JT_IPC_ADDRESS, "127.0.0.1:0"); new Cluster(conf); diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java index 05b434808e6..813bf9f1cc8 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java +++ b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapCollection.java @@ -40,6 +40,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.*; import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.util.ReflectionUtils; public class TestMapCollection { @@ -311,6 +312,7 @@ public class TestMapCollection { private static void runTest(String name, Job job) throws Exception { job.setNumReduceTasks(1); + job.getConfiguration().set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); job.getConfiguration().setInt(MRJobConfig.IO_SORT_FACTOR, 1000); job.getConfiguration().set("fs.default.name", "file:///"); job.getConfiguration().setInt("test.mapcollection.num.maps", 1);