From 812bd0c0e583fce925e3151510860ca9781b3e40 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 27 Aug 2014 15:14:54 +0000 Subject: [PATCH] MAPREDUCE-5885. build/test/test.mapred.spill causes release audit warnings. Contributed by Chen He --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../apache/hadoop/mapred/TestComparators.java | 61 ++++++++++++------ .../hadoop/mapred/TestMapOutputType.java | 62 ++++++++++++------- .../org/apache/hadoop/mapred/TestMapRed.java | 28 ++++++--- .../lib/TestKeyFieldBasedComparator.java | 25 +++++++- .../hadoop/mapreduce/TestMapReduce.java | 30 ++++++--- 6 files changed, 150 insertions(+), 59 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a6d2981e6e8..de0767d2a45 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -258,6 +258,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6044. Fully qualified intermediate done dir path breaks per-user dir creation on Windows. (zjshen) + MAPREDUCE-5885. build/test/test.mapred.spill causes release audit warnings + (Chen He via jlowe) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java index 1cef5cb42f2..f83dbe28578 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java @@ -17,13 +17,30 @@ */ package org.apache.hadoop.mapred; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.Random; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.mapreduce.MRConfig; -import junit.framework.TestCase; -import java.io.*; -import java.util.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Two different types of comparators can be used in MapReduce. One is used @@ -37,8 +54,11 @@ import java.util.*; * 2. Test the common use case where values are grouped by keys but values * within each key are grouped by a secondary key (a timestamp, for example). */ -public class TestComparators extends TestCase -{ +public class TestComparators { + private static final File TEST_DIR = new File( + System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), "TestComparators-mapred"); + JobConf conf = new JobConf(TestMapOutputType.class); JobClient jc; static Random rng = new Random(); @@ -292,9 +312,9 @@ public class TestComparators extends TestCase } } - + @Before public void configure() throws Exception { - Path testdir = new Path("build/test/test.mapred.spill"); + Path testdir = new Path(TEST_DIR.getAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.get(conf); @@ -334,14 +354,18 @@ public class TestComparators extends TestCase jc = new JobClient(conf); } - + + @After + public void cleanup() { + FileUtil.fullyDelete(TEST_DIR); + } /** * Test the default comparator for Map/Reduce. * Use the identity mapper and see if the keys are sorted at the end * @throws Exception */ - public void testDefaultMRComparator() throws Exception { - configure(); + @Test + public void testDefaultMRComparator() throws Exception { conf.setMapperClass(IdentityMapper.class); conf.setReducerClass(AscendingKeysReducer.class); @@ -361,8 +385,8 @@ public class TestComparators extends TestCase * comparator. Keys should be sorted in reverse order in the reducer. * @throws Exception */ - public void testUserMRComparator() throws Exception { - configure(); + @Test + public void testUserMRComparator() throws Exception { conf.setMapperClass(IdentityMapper.class); conf.setReducerClass(DescendingKeysReducer.class); conf.setOutputKeyComparatorClass(DecreasingIntComparator.class); @@ -384,8 +408,8 @@ public class TestComparators extends TestCase * values for a key should be sorted by the 'timestamp'. * @throws Exception */ - public void testUserValueGroupingComparator() throws Exception { - configure(); + @Test + public void testUserValueGroupingComparator() throws Exception { conf.setMapperClass(RandomGenMapper.class); conf.setReducerClass(AscendingGroupReducer.class); conf.setOutputValueGroupingComparator(CompositeIntGroupFn.class); @@ -409,8 +433,8 @@ public class TestComparators extends TestCase * order. This lets us make sure that the right comparators are used. * @throws Exception */ - public void testAllUserComparators() throws Exception { - configure(); + @Test + public void testAllUserComparators() throws Exception { conf.setMapperClass(RandomGenMapper.class); // use a decreasing comparator so keys are sorted in reverse order conf.setOutputKeyComparatorClass(DecreasingIntComparator.class); @@ -430,6 +454,7 @@ public class TestComparators extends TestCase * Test a user comparator that relies on deserializing both arguments * for each compare. */ + @Test public void testBakedUserComparator() throws Exception { MyWritable a = new MyWritable(8, 8); MyWritable b = new MyWritable(7, 9); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java index d11d7bc30b4..e3860fd2e25 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java @@ -17,21 +17,36 @@ */ package org.apache.hadoop.mapred; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.mapred.lib.*; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.MRConfig; -import junit.framework.TestCase; -import java.io.*; -import java.util.*; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.fail; + /** * TestMapOutputType checks whether the Map task handles type mismatch * between mapper output and the type specified in * JobConf.MapOutputKeyType and JobConf.MapOutputValueType. */ -public class TestMapOutputType extends TestCase -{ +public class TestMapOutputType { + private static final File TEST_DIR = new File( + System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), "TestMapOutputType-mapred"); JobConf conf = new JobConf(TestMapOutputType.class); JobClient jc; /** @@ -75,9 +90,9 @@ public class TestMapOutputType extends TestCase } } - + @Before public void configure() throws Exception { - Path testdir = new Path("build/test/test.mapred.spill"); + Path testdir = new Path(TEST_DIR.getAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.get(conf); @@ -101,17 +116,21 @@ public class TestMapOutputType extends TestCase throw new IOException("Mkdirs failed to create " + inDir.toString()); } Path inFile = new Path(inDir, "part0"); - SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inFile, + SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inFile, Text.class, Text.class); writer.append(new Text("rec: 1"), new Text("Hello")); writer.close(); jc = new JobClient(conf); } - + + @After + public void cleanup() { + FileUtil.fullyDelete(TEST_DIR); + } + + @Test public void testKeyMismatch() throws Exception { - configure(); - // Set bad MapOutputKeyClass and MapOutputValueClass conf.setMapOutputKeyClass(IntWritable.class); conf.setMapOutputValueClass(IntWritable.class); @@ -125,11 +144,9 @@ public class TestMapOutputType extends TestCase fail("Oops! The job was supposed to break due to an exception"); } } - + + @Test public void testValueMismatch() throws Exception { - configure(); - - // Set good MapOutputKeyClass, bad MapOutputValueClass conf.setMapOutputKeyClass(Text.class); conf.setMapOutputValueClass(IntWritable.class); @@ -142,11 +159,10 @@ public class TestMapOutputType extends TestCase fail("Oops! The job was supposed to break due to an exception"); } } - - public void testNoMismatch() throws Exception{ - configure(); - - // Set good MapOutputKeyClass and MapOutputValueClass + + @Test + public void testNoMismatch() throws Exception{ + // Set good MapOutputKeyClass and MapOutputValueClass conf.setMapOutputKeyClass(Text.class); conf.setMapOutputValueClass(Text.class); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java index 3f7a6f7e3b4..02a083b4f0a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java @@ -24,7 +24,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; -import java.util.Collections; +import java.io.File; import java.util.EnumSet; import java.util.HashSet; import java.util.Iterator; @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; @@ -46,11 +47,11 @@ import org.apache.hadoop.mapred.lib.IdentityReducer; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.junit.After; import org.junit.Test; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; /********************************************************** * MapredLoadTest generates a bunch of work that exercises @@ -110,6 +111,10 @@ public class TestMapRed extends Configured implements Tool { * of numbers in random order, but where each number appears * as many times as we were instructed. */ + private static final File TEST_DIR = new File( + System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), "TestMapRed-mapred"); + static class RandomGenMapper implements Mapper { @@ -248,6 +253,11 @@ public class TestMapRed extends Configured implements Tool { private static int counts = 100; private static Random r = new Random(); + @After + public void cleanup() { + FileUtil.fullyDelete(TEST_DIR); + } + /** public TestMapRed(int range, int counts, Configuration conf) throws IOException { this.range = range; @@ -372,7 +382,7 @@ public class TestMapRed extends Configured implements Tool { boolean includeCombine ) throws Exception { JobConf conf = new JobConf(TestMapRed.class); - Path testdir = new Path("build/test/test.mapred.compress"); + Path testdir = new Path(TEST_DIR.getAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.get(conf); @@ -440,7 +450,7 @@ public class TestMapRed extends Configured implements Tool { // // Generate distribution of ints. This is the answer key. // - JobConf conf = null; + JobConf conf; //Check to get configuration and check if it is configured thro' Configured //interface. This would happen when running testcase thro' command line. if(getConf() == null) { @@ -465,7 +475,7 @@ public class TestMapRed extends Configured implements Tool { // Write the answer key to a file. // FileSystem fs = FileSystem.get(conf); - Path testdir = new Path("mapred.loadtest"); + Path testdir = new Path(TEST_DIR.getAbsolutePath(), "mapred.loadtest"); if (!fs.mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.toString()); } @@ -635,8 +645,8 @@ public class TestMapRed extends Configured implements Tool { in.close(); } int originalTotal = 0; - for (int i = 0; i < dist.length; i++) { - originalTotal += dist[i]; + for (int aDist : dist) { + originalTotal += aDist; } System.out.println("Original sum: " + originalTotal); System.out.println("Recomputed sum: " + totalseen); @@ -727,7 +737,7 @@ public class TestMapRed extends Configured implements Tool { public void runJob(int items) { try { JobConf conf = new JobConf(TestMapRed.class); - Path testdir = new Path("build/test/test.mapred.spill"); + Path testdir = new Path(TEST_DIR.getAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.get(conf); @@ -777,7 +787,7 @@ public class TestMapRed extends Configured implements Tool { System.err.println("Usage: TestMapRed "); System.err.println(); System.err.println("Note: a good test will have a " + - " value that is substantially larger than the "); + " value that is substantially larger than the "); return -1; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java index 0bee2b564b9..34a4d2c6c92 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java @@ -18,7 +18,6 @@ package org.apache.hadoop.mapred.lib; -import java.io.*; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -35,9 +34,23 @@ import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.mapred.TextOutputFormat; import org.apache.hadoop.mapred.Utils; +import org.junit.After; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; public class TestKeyFieldBasedComparator extends HadoopTestCase { + + private static final File TEST_DIR = new File( + System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), + "TestKeyFieldBasedComparator-lib"); JobConf conf; JobConf localConf; @@ -50,8 +63,9 @@ public class TestKeyFieldBasedComparator extends HadoopTestCase { localConf = createJobConf(); localConf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " "); } + public void configure(String keySpec, int expect) throws Exception { - Path testdir = new Path("build/test/test.mapred.spill"); + Path testdir = new Path(TEST_DIR.getAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = getFileSystem(); @@ -116,6 +130,13 @@ public class TestKeyFieldBasedComparator extends HadoopTestCase { reader.close(); } } + + @After + public void cleanup() { + FileUtil.fullyDelete(TEST_DIR); + } + + @Test public void testBasicUnixComparator() throws Exception { configure("-k1,1n", 1); configure("-k2,2n", 1); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java index 01e1283dd23..48ad47af220 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java @@ -23,14 +23,14 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; +import java.io.File; import java.util.Iterator; import java.util.Random; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.SequenceFile; @@ -41,6 +41,10 @@ import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.junit.After; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; /********************************************************** * MapredLoadTest generates a bunch of work that exercises @@ -75,8 +79,10 @@ import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; * 7) A mapred job integrates all the count files into a single one. * **********************************************************/ -public class TestMapReduce extends TestCase { - +public class TestMapReduce { + private static final File TEST_DIR = new File( + System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), "TestMapReduce-mapreduce"); private static FileSystem fs; static { @@ -215,6 +221,12 @@ public class TestMapReduce extends TestCase { private static int counts = 100; private static Random r = new Random(); + @After + public void cleanup() { + FileUtil.fullyDelete(TEST_DIR); + } + + @Test public void testMapred() throws Exception { launch(); } @@ -239,7 +251,7 @@ public class TestMapReduce extends TestCase { // // Write the answer key to a file. // - Path testdir = new Path("mapred.loadtest"); + Path testdir = new Path(TEST_DIR.getAbsolutePath()); if (!fs.mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.toString()); } @@ -488,13 +500,17 @@ public class TestMapReduce extends TestCase { System.err.println("Usage: TestMapReduce "); System.err.println(); System.err.println("Note: a good test will have a value" + - " that is substantially larger than the "); + " that is substantially larger than the "); return; } int i = 0; range = Integer.parseInt(argv[i++]); counts = Integer.parseInt(argv[i++]); - launch(); + try { + launch(); + } finally { + FileUtil.fullyDelete(TEST_DIR); + } } }