From f22ce96b50f8148e95a453ff41f84b99a9197bf6 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Tue, 29 Mar 2016 10:32:21 +0800 Subject: [PATCH] MAPREDUCE-6662. Clear ASF Warnings on test data files. Contributed by Vinayakumar B (cherry picked from commit 0ef8bbfd8791899c7cfed3dd9c1670182fd87575) --- .../v2/hs/TestHistoryFileManager.java | 5 +++-- .../hadoop/mapred/TestFieldSelection.java | 5 +++-- .../mapred/TestMRTimelineEventHandling.java | 19 ++++++++++++------- .../hadoop/mapred/TestMiniMRChildTask.java | 16 ++++++++++------ .../mapred/TestOldCombinerGrouping.java | 5 +++-- .../mapred/lib/aggregate/TestAggregates.java | 7 ++++--- .../mapreduce/TestNewCombinerGrouping.java | 5 +++-- .../aggregate/TestMapReduceAggregates.java | 5 +++-- .../lib/input/TestCombineFileInputFormat.java | 4 +++- 9 files changed, 44 insertions(+), 27 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index 18e3e6a38ac..aa2e9796511 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -71,8 +71,9 @@ public class TestHistoryFileManager { Configuration conf = new HdfsConfiguration(); Configuration conf2 = new HdfsConfiguration(); dfsCluster = new MiniDFSCluster.Builder(conf).build(); - conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, - conf.get(MiniDFSCluster.HDFS_MINIDFS_BASEDIR) + "_2"); + conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, conf.get( + MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MiniDFSCluster.getBaseDirectory()) + + "_2"); dfsCluster2 = new MiniDFSCluster.Builder(conf2).build(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java index 239c239230e..29333b7bfdb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java @@ -44,8 +44,9 @@ private static NumberFormat idFormat = NumberFormat.getInstance(); FileSystem fs = FileSystem.get(conf); int numOfInputLines = 10; - Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test"); - Path INPUT_DIR = new Path("build/test/input_for_field_selection_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_field_selection_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_field_selection_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index a82007f97c9..40ed9ad1160 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -97,9 +97,10 @@ public class TestMRTimelineEventHandling { + cluster.getApplicationHistoryServer().getPort()); TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); RunningJob job = UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); Assert.assertEquals(JobStatus.SUCCEEDED, @@ -155,8 +156,10 @@ public class TestMRTimelineEventHandling { TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); RunningJob job = UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); Assert.assertEquals(JobStatus.SUCCEEDED, @@ -195,8 +198,10 @@ public class TestMRTimelineEventHandling { TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false); RunningJob job = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java index 74be8a758db..cbeeccffc7a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java @@ -436,8 +436,10 @@ public class TestMiniMRChildTask { JobConf conf = new JobConf(mr.getConfig()); // initialize input, output directories - Path inDir = new Path("input"); - Path outDir = new Path("output"); + Path rootDir = new Path(System.getProperty("test.build.data", + "build/test/data")); + Path inDir = new Path(rootDir, "input"); + Path outDir = new Path(rootDir, "output"); String input = "The input"; // set config to use the ExecutionEnvCheckMapClass map class @@ -462,9 +464,10 @@ public class TestMiniMRChildTask { public void testTaskEnv(){ try { JobConf conf = new JobConf(mr.getConfig()); + String baseDir = System.getProperty("test.build.data", "build/test/data"); // initialize input, output directories - Path inDir = new Path("testing/wc/input1"); - Path outDir = new Path("testing/wc/output1"); + Path inDir = new Path(baseDir + "/testing/wc/input1"); + Path outDir = new Path(baseDir + "/testing/wc/output1"); FileSystem outFs = outDir.getFileSystem(conf); runTestTaskEnv(conf, inDir, outDir, false); outFs.delete(outDir, true); @@ -485,9 +488,10 @@ public class TestMiniMRChildTask { public void testTaskOldEnv(){ try { JobConf conf = new JobConf(mr.getConfig()); + String baseDir = System.getProperty("test.build.data", "build/test/data"); // initialize input, output directories - Path inDir = new Path("testing/wc/input1"); - Path outDir = new Path("testing/wc/output1"); + Path inDir = new Path(baseDir + "/testing/wc/input1"); + Path outDir = new Path(baseDir + "/testing/wc/output1"); FileSystem outFs = outDir.getFileSystem(conf); runTestTaskEnv(conf, inDir, outDir, true); outFs.delete(outDir, true); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java index 80a670209d2..046c2d37eed 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java @@ -37,8 +37,9 @@ import java.util.Set; import java.util.UUID; public class TestOldCombinerGrouping { - private static String TEST_ROOT_DIR = - new File("build", UUID.randomUUID().toString()).getAbsolutePath(); + private static String TEST_ROOT_DIR = new File(System.getProperty( + "test.build.data", "build/test/data"), UUID.randomUUID().toString()) + .getAbsolutePath(); public static class Map implements Mapper { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java index 6da96ce22bd..e2fdd429c56 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java @@ -46,8 +46,9 @@ public class TestAggregates extends TestCase { FileSystem fs = FileSystem.get(conf); int numOfInputLines = 20; - Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test"); - Path INPUT_DIR = new Path("build/test/input_for_aggregates_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); @@ -115,7 +116,7 @@ public class TestAggregates extends TestCase { outdata = outdata.substring(0, expectedOutput.toString().length()); assertEquals(expectedOutput.toString(),outdata); - //fs.delete(OUTPUT_DIR); + fs.delete(OUTPUT_DIR, true); fs.delete(INPUT_DIR, true); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java index ab9ddbb4aaa..c2054f1d4c1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java @@ -39,8 +39,9 @@ import java.util.Set; import java.util.UUID; public class TestNewCombinerGrouping { - private static String TEST_ROOT_DIR = - new File("build", UUID.randomUUID().toString()).getAbsolutePath(); + private static String TEST_ROOT_DIR = new File(System.getProperty( + "test.build.data", "build/test/data"), UUID.randomUUID().toString()) + .getAbsolutePath(); public static class Map extends Mapper { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java index f24dffe2655..788ad41ff96 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java @@ -51,8 +51,9 @@ public class TestMapReduceAggregates extends TestCase { FileSystem fs = FileSystem.get(conf); int numOfInputLines = 20; - Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test"); - Path INPUT_DIR = new Path("build/test/input_for_aggregates_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java index b49f2d831ab..1fca5c982bf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java @@ -1772,7 +1772,9 @@ public class TestCombineFileInputFormat { // default fs path assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString()); // add a local file - Path localPath = new Path("testFile1"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path localPath = new Path(localPathRoot, "testFile1"); FileSystem lfs = FileSystem.getLocal(conf); FSDataOutputStream dos = lfs.create(localPath); dos.writeChars("Local file for CFIF");