From 26910fd0582b19f58aaf684f76c222ce18fc48ab Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 12 Jul 2012 15:52:12 +0000 Subject: [PATCH] MAPREDUCE-4416. Some tests fail if Clover is enabled (Kihwal Lee via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1360735 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + .../apache/hadoop/mapred/TestIndexCache.java | 87 +++++++++++++++---- .../hadoop-mapreduce-client-jobclient/pom.xml | 17 ++++ .../hadoop-yarn-applications/pom.xml | 17 ++++ hadoop-project/pom.xml | 7 +- 5 files changed, 113 insertions(+), 17 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 867aefb854a..ed9c7e46c1c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -221,6 +221,8 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-3993. Graceful handling of codec errors during decompression (kkambatl via tucu) + MAPREDUCE-4416. Some tests fail if Clover is enabled (Kihwal Lee via bobby) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java index e0037722641..b6a2df08833 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java @@ -35,16 +35,23 @@ import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; import junit.framework.TestCase; public class TestIndexCache extends TestCase { + private JobConf conf; + private FileSystem fs; + private Path p; + + @Override + public void setUp() throws IOException { + conf = new JobConf(); + fs = FileSystem.getLocal(conf).getRaw(); + p = new Path(System.getProperty("test.build.data", "/tmp"), + "cache").makeQualified(fs.getUri(), fs.getWorkingDirectory()); + } public void testLRCPolicy() throws Exception { Random r = new Random(); long seed = r.nextLong(); r.setSeed(seed); System.out.println("seed: " + seed); - JobConf conf = new JobConf(); - FileSystem fs = FileSystem.getLocal(conf).getRaw(); - Path p = new Path(System.getProperty("test.build.data", "/tmp"), - "cache").makeQualified(fs); fs.delete(p, true); conf.setInt(TTConfig.TT_INDEX_CACHE, 1); final int partsPerMap = 1000; @@ -115,10 +122,6 @@ public class TestIndexCache extends TestCase { public void testBadIndex() throws Exception { final int parts = 30; - JobConf conf = new JobConf(); - FileSystem fs = FileSystem.getLocal(conf).getRaw(); - Path p = new Path(System.getProperty("test.build.data", "/tmp"), - "cache").makeQualified(fs); fs.delete(p, true); conf.setInt(TTConfig.TT_INDEX_CACHE, 1); IndexCache cache = new IndexCache(conf); @@ -150,10 +153,6 @@ public class TestIndexCache extends TestCase { } public void testInvalidReduceNumberOrLength() throws Exception { - JobConf conf = new JobConf(); - FileSystem fs = FileSystem.getLocal(conf).getRaw(); - Path p = new Path(System.getProperty("test.build.data", "/tmp"), - "cache").makeQualified(fs); fs.delete(p, true); conf.setInt(TTConfig.TT_INDEX_CACHE, 1); final int partsPerMap = 1000; @@ -199,10 +198,6 @@ public class TestIndexCache extends TestCase { // This test case may not repeatable. But on my macbook this test // fails with probability of 100% on code before MAPREDUCE-2541, // so it is repeatable in practice. - JobConf conf = new JobConf(); - FileSystem fs = FileSystem.getLocal(conf).getRaw(); - Path p = new Path(System.getProperty("test.build.data", "/tmp"), - "cache").makeQualified(fs); fs.delete(p, true); conf.setInt(TTConfig.TT_INDEX_CACHE, 10); // Make a big file so removeMapThread almost surely runs faster than @@ -247,6 +242,66 @@ public class TestIndexCache extends TestCase { } } + public void testCreateRace() throws Exception { + fs.delete(p, true); + conf.setInt(TTConfig.TT_INDEX_CACHE, 1); + final int partsPerMap = 1000; + final int bytesPerFile = partsPerMap * 24; + final IndexCache cache = new IndexCache(conf); + + final Path racy = new Path(p, "racyIndex"); + final String user = + UserGroupInformation.getCurrentUser().getShortUserName(); + writeFile(fs, racy, bytesPerFile, partsPerMap); + + // run multiple instances + Thread[] getInfoThreads = new Thread[50]; + for (int i = 0; i < 50; i++) { + getInfoThreads[i] = new Thread() { + @Override + public void run() { + try { + cache.getIndexInformation("racyIndex", partsPerMap, racy, user); + cache.removeMap("racyIndex"); + } catch (Exception e) { + // should not be here + } + } + }; + } + + for (int i = 0; i < 50; i++) { + getInfoThreads[i].start(); + } + + final Thread mainTestThread = Thread.currentThread(); + + Thread timeoutThread = new Thread() { + @Override + public void run() { + try { + Thread.sleep(15000); + mainTestThread.interrupt(); + } catch (InterruptedException ie) { + // we are done; + } + } + }; + + for (int i = 0; i < 50; i++) { + try { + getInfoThreads[i].join(); + } catch (InterruptedException ie) { + // we haven't finished in time. Potential deadlock/race. + fail("Unexpectedly long delay during concurrent cache entry creations"); + } + } + // stop the timeoutThread. If we get interrupted before stopping, there + // must be something wrong, although it wasn't a deadlock. No need to + // catch and swallow. + timeoutThread.interrupt(); + } + private static void checkRecord(IndexRecord rec, long fill) { assertEquals(fill, rec.startOffset); assertEquals(fill, rec.rawLength); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml index 2c2370bfc74..b14ab2686c2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml @@ -99,6 +99,23 @@ + + + clover + + false + + clover + + + + + com.cenqua.clover + clover + + + + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml index b396bc3cfd5..881dbcbc057 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml @@ -31,4 +31,21 @@ hadoop-yarn-applications-distributedshell + + + clover + + false + + clover + + + + + com.cenqua.clover + clover + + + + diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index a44923b870a..99083001893 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -648,7 +648,12 @@ 4.0.0 compile - + + com.cenqua.clover + clover + + 3.0.2 +