MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)

hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt

@@ -18,3 +18,4 @@ MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang v
 MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
 MAPREDUCE-6054. native-task: Speed up tests (todd)
 MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang)
+MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java

@@ -38,11 +38,14 @@
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.AfterClass;
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
+
 public class CombinerTest {
   private FileSystem fs;
   private String inputpath;
@@ -50,16 +53,13 @@ public class CombinerTest {
   private String hadoopoutputpath;
 
   @Test
-  public void testWordCountCombiner() {
+  public void testWordCountCombiner() throws Exception {
-    try {
-
     final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
     nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
     final Job nativejob = getJob("nativewordcount", nativeConf, inputpath, nativeoutputpath);
 
     final Configuration commonConf = ScenarioConfiguration.getNormalConfiguration();
     commonConf.addResource(TestConstants.COMBINER_CONF_PATH);
-
     final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
 
     assertTrue(nativejob.waitForCompletion(true));
@@ -72,11 +72,6 @@ public void testWordCountCombiner() {
     assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
     assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
       nativeReduceGroups.getValue(), normalReduceGroups.getValue());
-      
-    } catch (final Exception e) {
-      e.printStackTrace();
-      assertEquals("run exception", true, false);
-    }
   }
 
   @Before
@@ -88,8 +83,7 @@ public void startUp() throws Exception {
 
     this.fs = FileSystem.get(conf);
 
-    this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
+    this.inputpath = TestConstants.NATIVETASK_COMBINER_TEST_INPUTDIR + "/wordcount";
-        TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
 
     if (!fs.exists(new Path(inputpath))) {
       new TestInputFile(
@@ -98,10 +92,15 @@ public void startUp() throws Exception {
           Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
     }
 
-    this.nativeoutputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
+    this.nativeoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR + "/nativewordcount";
-        TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativewordcount";
+    this.hadoopoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR + "/normalwordcount";
-    this.hadoopoutputpath = conf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
+  }
-        TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalwordcount";
+
+  @AfterClass
+  public static void cleanUp() throws IOException {
+    final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+    fs.delete(new Path(TestConstants.NATIVETASK_COMBINER_TEST_DIR), true);
+    fs.close();
   }
 
   protected static Job getJob(String jobname, Configuration inputConf, String inputpath, String outputpath)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java

@@ -35,11 +35,14 @@
 import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.Job;
+import org.junit.AfterClass;
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
+
 public class LargeKVCombinerTest {
   private static final Log LOG = LogFactory.getLog(LargeKVCombinerTest.class);
 
@@ -50,23 +53,21 @@ public void startUp() throws Exception {
   }
 
   @Test
-  public void testLargeValueCombiner(){
+  public void testLargeValueCombiner() throws Exception {
     final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
     final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
     normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
     nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
     final int deafult_KVSize_Maximum = 1 << 22; // 4M
-    final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
+    final int KVSize_Maximum = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
         deafult_KVSize_Maximum);
-    final String inputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
+    final String inputPath = TestConstants.NATIVETASK_COMBINER_TEST_INPUTDIR + "/largeKV";
-        TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/largeKV";
+    final String nativeOutputPath = TestConstants.NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR
-    final String nativeOutputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
+      + "/nativeLargeKV";
-        TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativeLargeKV";
+    final String hadoopOutputPath = TestConstants.NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR
-    final String hadoopOutputPath = normalConf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
+      + "/normalLargeKV";
-        TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalLargeKV";
-    try {
     final FileSystem fs = FileSystem.get(normalConf);
-      for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
+    for (int i = 65536; i <= KVSize_Maximum; i *= 4) {
 
       int max = i;
       int min = Math.max(i / 4, max - 10);
@@ -101,10 +102,13 @@ public void testLargeValueCombiner(){
 //            nativeReduceGroups.getValue(), normalReduceGroups.getValue());
     }
     fs.close();
-    } catch (final Exception e) {
-      e.printStackTrace();
-      assertEquals("run exception", true, false);
   }
+
+  @AfterClass
+  public static void cleanUp() throws IOException {
+    final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+    fs.delete(new Path(TestConstants.NATIVETASK_COMBINER_TEST_DIR), true);
+    fs.close();
   }
   
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java

@@ -38,11 +38,14 @@
 import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
 import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
 import org.apache.hadoop.mapreduce.Counter;
+import org.junit.AfterClass;
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
+
 public class OldAPICombinerTest {
   private FileSystem fs;
   private String inputpath;
@@ -51,7 +54,7 @@ public class OldAPICombinerTest {
   public void testWordCountCombinerWithOldAPI() throws Exception {
     final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
     nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
-    final String nativeoutput = nativeConf.get(TestConstants.OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH);
+    final String nativeoutput = TestConstants.NATIVETASK_OLDAPI_COMBINER_TEST_NATIVE_OUTPUTPATH;
     final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", inputpath, nativeoutput);
     RunningJob nativeRunning = JobClient.runJob(nativeJob);
 
@@ -59,7 +62,7 @@ public void testWordCountCombinerWithOldAPI() throws Exception {
 
     final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
     normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
-    final String normaloutput = normalConf.get(TestConstants.OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH);
+    final String normaloutput = TestConstants.NATIVETASK_OLDAPI_COMBINER_TEST_NORMAL_OUTPUTPATH;
     final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", inputpath, normaloutput);
 
     RunningJob normalRunning = JobClient.runJob(normalJob);
@@ -78,15 +81,21 @@ public void startUp() throws Exception {
     final ScenarioConfiguration conf = new ScenarioConfiguration();
     conf.addcombinerConf();
     this.fs = FileSystem.get(conf);
-    this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
+    this.inputpath = TestConstants.NATIVETASK_COMBINER_TEST_INPUTDIR + "/wordcount";
-        TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
 
     if (!fs.exists(new Path(inputpath))) {
-      new TestInputFile(conf.getInt("nativetask.combiner.wordcount.filesize", 1000000), Text.class.getName(),
+      new TestInputFile(conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000), Text.class.getName(),
           Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
     }
   }
 
+  @AfterClass
+  public static void cleanUp() throws IOException {
+    final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+    fs.delete(new Path(TestConstants.NATIVETASK_COMBINER_TEST_DIR), true);
+    fs.close();
+  }
+
   private static JobConf getOldAPIJobconf(Configuration configuration, String name, String input, String output)
       throws Exception {
     final JobConf jobConf = new JobConf(configuration);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java

@@ -24,7 +24,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -32,8 +31,6 @@
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 
 public class CompressMapper {
-  public static final String inputFile = "./compress/input.txt";
-  public static final String outputFileDir = "./compress/output/";
 
   public static class TextCompressMapper extends Mapper<Text, Text, Text, Text> {
 
@@ -43,27 +40,22 @@ protected void map(Text key, Text value, Context context) throws IOException, In
     }
   }
 
-  public static Job getCompressJob(String jobname, Configuration conf) {
+  public static Job getCompressJob(String jobname, Configuration conf, String inputpath, String outputpath)
-    Job job = null;
+    throws Exception {
-    try {
+    Job job = new Job(conf, jobname + "-CompressMapperJob");
-      job = new Job(conf, jobname + "-CompressMapperJob");
     job.setJarByClass(CompressMapper.class);
     job.setMapperClass(TextCompressMapper.class);
     job.setOutputKeyClass(Text.class);
     job.setMapOutputValueClass(Text.class);
-      final Path outputpath = new Path(outputFileDir + jobname);
     // if output file exists ,delete it
     final FileSystem hdfs = FileSystem.get(new ScenarioConfiguration());
-      if (hdfs.exists(outputpath)) {
+    if (hdfs.exists(new Path(outputpath))) {
-        hdfs.delete(outputpath);
+      hdfs.delete(new Path(outputpath));
     }
     hdfs.close();
     job.setInputFormatClass(SequenceFileInputFormat.class);
-      FileInputFormat.addInputPath(job, new Path(inputFile));
+    FileInputFormat.addInputPath(job, new Path(inputpath));
-      FileOutputFormat.setOutputPath(job, outputpath);
+    FileOutputFormat.setOutputPath(job, new Path(outputpath));
-    } catch (final Exception e) {
-      e.printStackTrace();
-    }
     return job;
   }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java

@@ -30,60 +30,87 @@
 import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
 import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.AfterClass;
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
+
 public class CompressTest {
 
+  private static final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+  private static final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
+
+  static {
+    nativeConf.addResource(TestConstants.COMPRESS_TEST_CONF_PATH);
+    hadoopConf.addResource(TestConstants.COMPRESS_TEST_CONF_PATH);
+  }
+
   @Test
   public void testSnappyCompress() throws Exception {
-    final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
+    final String snappyCodec = "org.apache.hadoop.io.compress.SnappyCodec";
-    conf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
+
-    final Job job = CompressMapper.getCompressJob("nativesnappy", conf);
+    nativeConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, snappyCodec);
+    final String nativeOutputPath =
+      TestConstants.NATIVETASK_COMPRESS_TEST_NATIVE_OUTPUTDIR + "/snappy";
+    final Job job = CompressMapper.getCompressJob("nativesnappy", nativeConf,
+      TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, nativeOutputPath);
     assertTrue(job.waitForCompletion(true));
 
-    final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
+    hadoopConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, snappyCodec);
-    hadoopconf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
+    final String hadoopOutputPath =
-    final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopconf);
+      TestConstants.NATIVETASK_COMPRESS_TEST_NORMAL_OUTPUTDIR + "/snappy";
+    final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopConf,
+      TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, hadoopOutputPath);
     assertTrue(hadoopjob.waitForCompletion(true));
 
-    final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativesnappy",
+    final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
-        CompressMapper.outputFileDir + "hadoopsnappy");
     assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
   }
 
   @Test
   public void testGzipCompress() throws Exception {
-    final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
+    final String gzipCodec = "org.apache.hadoop.io.compress.GzipCodec";
-    conf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
+
-    final Job job = CompressMapper.getCompressJob("nativegzip", conf);
+    nativeConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, gzipCodec);
+    final String nativeOutputPath =
+      TestConstants.NATIVETASK_COMPRESS_TEST_NATIVE_OUTPUTDIR + "/gzip";
+    final Job job = CompressMapper.getCompressJob("nativegzip", nativeConf,
+      TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, nativeOutputPath);
     assertTrue(job.waitForCompletion(true));
 
-    final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
+    hadoopConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, gzipCodec);
-    hadoopconf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
+    final String hadoopOutputPath =
-    final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopconf);
+      TestConstants.NATIVETASK_COMPRESS_TEST_NORMAL_OUTPUTDIR + "/gzip";
+    final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopConf,
+      TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, hadoopOutputPath);
     assertTrue(hadoopjob.waitForCompletion(true));
 
-    final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativegzip",
+    final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
-        CompressMapper.outputFileDir + "hadoopgzip");
     assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
   }
 
   @Test
   public void testLz4Compress() throws Exception {
-    final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
+    final String lz4Codec = "org.apache.hadoop.io.compress.Lz4Codec";
-    nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
+
-    final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf);
+    nativeConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, lz4Codec);
+    final String nativeOutputPath =
+      TestConstants.NATIVETASK_COMPRESS_TEST_NATIVE_OUTPUTDIR + "/lz4";
+    final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf,
+      TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, nativeOutputPath);
     assertTrue(nativeJob.waitForCompletion(true));
 
-    final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
+    hadoopConf.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, lz4Codec);
-    hadoopConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
+    final String hadoopOutputPath =
-    final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf);
+      TestConstants.NATIVETASK_COMPRESS_TEST_NORMAL_OUTPUTDIR + "/lz4";
+    final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf,
+      TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR, hadoopOutputPath);
     assertTrue(hadoopJob.waitForCompletion(true));
-    final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativelz4",
+    final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
-        CompressMapper.outputFileDir + "hadooplz4");
     assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
   }
 
@@ -93,15 +120,22 @@ public void startUp() throws Exception {
     Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
     final ScenarioConfiguration conf = new ScenarioConfiguration();
     final FileSystem fs = FileSystem.get(conf);
-    final Path path = new Path(CompressMapper.inputFile);
+    final Path path = new Path(TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR);
     fs.delete(path);
     if (!fs.exists(path)) {
-      new TestInputFile(ScenarioConfiguration.getNormalConfiguration().getInt(
+      new TestInputFile(hadoopConf.getInt(
           TestConstants.NATIVETASK_COMPRESS_FILESIZE, 100000),
           Text.class.getName(), Text.class.getName(), conf)
-      .createSequenceTestFile(CompressMapper.inputFile);
+      .createSequenceTestFile(TestConstants.NATIVETASK_COMPRESS_TEST_INPUTDIR);
-
     }
     fs.close();
   }
+
+  @AfterClass
+  public static void cleanUp() throws IOException {
+    final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+    fs.delete(new Path(TestConstants.NATIVETASK_COMPRESS_TEST_DIR), true);
+    fs.close();
+  }
+
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java

@@ -22,6 +22,9 @@
 import junit.framework.Assert;
 import junit.framework.TestCase;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.nativetask.Command;
@@ -30,6 +33,7 @@
 import org.apache.hadoop.mapred.nativetask.TaskContext;
 import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
 import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
+import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
 import org.apache.hadoop.mapred.nativetask.util.OutputUtil;
 import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
 import org.mockito.Matchers;
@@ -43,7 +47,7 @@ public class TestNativeCollectorOnlyHandler extends TestCase {
   private BufferPusher pusher;
   private ICombineHandler combiner;
   private TaskContext taskContext;
-  private String localDir = "build/test/mapred/local";
+  private static final String LOCAL_DIR = TestConstants.NATIVETASK_TEST_DIR + "/local";
 
   @Override
   public void setUp() throws IOException {
@@ -53,7 +57,7 @@ public void setUp() throws IOException {
     JobConf jobConf = new JobConf();
     jobConf.set(OutputUtil.NATIVE_TASK_OUTPUT_MANAGER,
         "org.apache.hadoop.mapred.nativetask.util.LocalJobOutputFiles");
-    jobConf.set("mapred.local.dir", localDir);
+    jobConf.set("mapred.local.dir", LOCAL_DIR);
     this.taskContext = new TaskContext(jobConf,
         BytesWritable.class, BytesWritable.class,
         BytesWritable.class,
@@ -64,6 +68,12 @@ public void setUp() throws IOException {
     Mockito.when(nativeHandler.getInputBuffer()).thenReturn(new InputBuffer(BufferType.HEAP_BUFFER, 100));
   }
 
+  @Override
+  public void tearDown() throws IOException {
+    FileSystem.getLocal(new Configuration()).delete(new Path(LOCAL_DIR));
+  }
+
+
   public void testCollect() throws IOException {
     this.handler = new NativeCollectorOnlyHandler(taskContext, nativeHandler, pusher, combiner);
     handler.collect(new BytesWritable(), new BytesWritable(), 100);
@@ -95,9 +105,9 @@ public void testOnCall() throws IOException {
     }
     Assert.assertTrue("exception thrown", thrown);
 
-    final String expectedOutputPath = localDir + "/output/file.out";
+    final String expectedOutputPath = LOCAL_DIR + "/output/file.out";
-    final String expectedOutputIndexPath = localDir + "/output/file.out.index";
+    final String expectedOutputIndexPath = LOCAL_DIR + "/output/file.out.index";
-    final String expectedSpillPath = localDir + "/output/spill0.out";
+    final String expectedSpillPath = LOCAL_DIR + "/output/spill0.out";
 
     final String outputPath = handler.onCall(NativeCollectorOnlyHandler.GET_OUTPUT_PATH, null).readString();
     Assert.assertEquals(expectedOutputPath, outputPath);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
 import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
 import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.junit.AfterClass;
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
@@ -112,16 +113,14 @@ public void startUp() throws Exception {
   }
 
   @Test
-  public void testKVCompability() {
+  public void testKVCompability() throws Exception {
-    try {
     final String nativeoutput = this.runNativeTest(
       "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
     final String normaloutput = this.runNormalTest(
       "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
     final boolean compareRet = ResultVerifier.verify(normaloutput, nativeoutput);
-      final String input = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
+    final String input = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
-          + keyclass.getName()
+      + keyclass.getName() + "/" + valueclass.getName();
-          + "/" + valueclass.getName();
     if(compareRet){
       final FileSystem fs = FileSystem.get(hadoopkvtestconf);
       fs.delete(new Path(nativeoutput), true);
@@ -130,52 +129,42 @@ public void testKVCompability() {
       fs.close();
     }
     assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-    } catch (final IOException e) {
-      assertEquals("test run exception:", null, e);
-    } catch (final Exception e) {
-      assertEquals("test run exception:", null, e);
-    }
   }
 
-  private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
+  @AfterClass
-    final String inputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
+  public static void cleanUp() throws IOException {
-        + keyclass.getName()
+    final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
-        + "/" + valueclass.getName();
+    fs.delete(new Path(TestConstants.NATIVETASK_KVTEST_DIR), true);
-    final String outputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/"
+    fs.close();
+  }
+
+  private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws Exception {
+    final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
+      + keyclass.getName() + "/" + valueclass.getName();
+    final String outputpath = TestConstants.NATIVETASK_KVTEST_NATIVE_OUTPUTDIR + "/"
       + keyclass.getName() + "/" + valueclass.getName();
     // if output file exists ,then delete it
     final FileSystem fs = FileSystem.get(nativekvtestconf);
     fs.delete(new Path(outputpath));
     fs.close();
     nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
-    try {
     final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
     assertTrue("job should complete successfully", keyJob.runJob());
-    } catch (final Exception e) {
-      return "native testcase run time error.";
-    }
     return outputpath;
   }
 
-  private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
+  private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws Exception {
-    final String inputpath = hadoopkvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
+    final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/"
-        + keyclass.getName()
+        + keyclass.getName() + "/" + valueclass.getName();
-        + "/" + valueclass.getName();
+    final String outputpath = TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR + "/"
-    final String outputpath = hadoopkvtestconf
-        .get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR)
-        + "/"
         + keyclass.getName() + "/" + valueclass.getName();
     // if output file exists ,then delete it
     final FileSystem fs = FileSystem.get(hadoopkvtestconf);
     fs.delete(new Path(outputpath));
     fs.close();
     hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
-    try {
     final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
     assertTrue("job should complete successfully", keyJob.runJob());
-    } catch (final Exception e) {
-      return "normal testcase run time error.";
-    }
     return outputpath;
   }
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
 import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
 import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.junit.AfterClass;
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
@@ -47,16 +48,6 @@ public void startUp() throws Exception {
     Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
   }
 
-  @Test
-  public void testKeySize() {
-    runKVSizeTests(Text.class, IntWritable.class);
-  }
-
-  @Test
-  public void testValueSize() {
-    runKVSizeTests(IntWritable.class, Text.class);
-  }
-
   private static Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
   private static Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
   static {
@@ -66,14 +57,30 @@ public void testValueSize() {
     normalConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
   }
 
-  public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) {
+  @Test
+  public void testKeySize() throws Exception {
+    runKVSizeTests(Text.class, IntWritable.class);
+  }
+
+  @Test
+  public void testValueSize() throws Exception {
+    runKVSizeTests(IntWritable.class, Text.class);
+  }
+
+  @AfterClass
+  public static void cleanUp() throws IOException {
+    final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+    fs.delete(new Path(TestConstants.NATIVETASK_KVTEST_DIR), true);
+    fs.close();
+  }
+
+  public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) throws Exception {
     if (!keyClass.equals(Text.class) && !valueClass.equals(Text.class)) {
       return;
     }
     final int deafult_KVSize_Maximum = 1 << 22; // 4M
     final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
         deafult_KVSize_Maximum);
-    try {
 
     for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
       int min = i / 4;
@@ -96,48 +103,35 @@ public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) {
         + ", max size: " + max + ", normal out: " + normalOutPut + ", native Out: " + nativeOutPut;
       assertEquals(reason, true, compareRet);
     }
-    } catch (final Exception e) {
-      // TODO: handle exception
-      // assertEquals("test run exception:", null, e);
-      e.printStackTrace();
-    }
   }
 
   private String runNativeLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
       throws Exception {
-    final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
+    final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/LargeKV/"
-        + "/" + valueclass.getName();
+      + keyclass.getName() + "/" + valueclass.getName();
-    final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/LargeKV/" + keyclass.getName()
+    final String outputpath = TestConstants.NATIVETASK_KVTEST_NATIVE_OUTPUTDIR + "/LargeKV/"
-        + "/" + valueclass.getName();
-    // if output file exists ,then delete it
-    final FileSystem fs = FileSystem.get(conf);
-    fs.delete(new Path(outputpath), true);
-    fs.close();
-    try {
-      final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
-      assertTrue("job should complete successfully", keyJob.runJob());
-    } catch (final Exception e) {
-      return "normal testcase run time error.";
-    }
-    return outputpath;
-  }
-
-  private String runNormalLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
-      throws IOException {
-    final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
-        + "/" + valueclass.getName();
-    final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR) + "/LargeKV/"
       + keyclass.getName() + "/" + valueclass.getName();
     // if output file exists ,then delete it
     final FileSystem fs = FileSystem.get(conf);
     fs.delete(new Path(outputpath), true);
     fs.close();
-    try {
     final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
     assertTrue("job should complete successfully", keyJob.runJob());
-    } catch (final Exception e) {
+    return outputpath;
-      return "normal testcase run time error.";
   }
+
+  private String runNormalLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
+      throws Exception {
+    final String inputpath = TestConstants.NATIVETASK_KVTEST_INPUTDIR + "/LargeKV/"
+      + keyclass.getName() + "/" + valueclass.getName();
+    final String outputpath = TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR + "/LargeKV/"
+      + keyclass.getName() + "/" + valueclass.getName();
+    // if output file exists ,then delete it
+    final FileSystem fs = FileSystem.get(conf);
+    fs.delete(new Path(outputpath), true);
+    fs.close();
+    final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
+    assertTrue("job should complete successfully", keyJob.runJob());
     return outputpath;
   }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java

@@ -37,6 +37,7 @@
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.junit.AfterClass;
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
@@ -49,20 +50,21 @@ public void nonSortTest() throws Exception {
     Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
     nativeConf.addResource(TestConstants.NONSORT_TEST_CONF);
     nativeConf.set(TestConstants.NATIVETASK_MAP_OUTPUT_SORT, "false");
-    String inputpath = nativeConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
+    final Job nativeNonSort = getJob(nativeConf, "NativeNonSort",
-    String outputpath = nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT);
+      TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR,
-    final Job nativeNonSort = getJob(nativeConf, "NativeNonSort", inputpath, outputpath);
+      TestConstants.NATIVETASK_NONSORT_TEST_NATIVE_OUTPUT);
     nativeNonSort.waitForCompletion(true);
 
     Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
     normalConf.addResource(TestConstants.NONSORT_TEST_CONF);
-    inputpath = normalConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
+    final Job hadoopWithSort = getJob(normalConf, "NormalJob",
-    outputpath = normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT);
+      TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR,
-    final Job hadoopWithSort = getJob(normalConf, "NormalJob", inputpath, outputpath);
+      TestConstants.NATIVETASK_NONSORT_TEST_NORMAL_OUTPUT);
     hadoopWithSort.waitForCompletion(true);
 
-    final boolean compareRet = ResultVerifier.verify(nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT),
+    final boolean compareRet = ResultVerifier.verify(
-        normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT));
+      TestConstants.NATIVETASK_NONSORT_TEST_NATIVE_OUTPUT,
+      TestConstants.NATIVETASK_NONSORT_TEST_NORMAL_OUTPUT);
     assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
   }
 
@@ -73,14 +75,22 @@ public void startUp() throws Exception {
     final ScenarioConfiguration configuration = new ScenarioConfiguration();
     configuration.addNonSortTestConf();
     final FileSystem fs = FileSystem.get(configuration);
-    final Path path = new Path(configuration.get(TestConstants.NONSORT_TEST_INPUTDIR));
+    final Path path = new Path(TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR);
     if (!fs.exists(path)) {
-      new TestInputFile(configuration.getInt("nativetask.nonsorttest.filesize", 10000000), Text.class.getName(),
+      new TestInputFile(configuration.getInt(TestConstants.NATIVETASK_NONSORTTEST_FILESIZE, 10000000), Text.class.getName(),
           Text.class.getName(), configuration).createSequenceTestFile(path.toString());
     }
     fs.close();
   }
 
+  @AfterClass
+  public static void cleanUp() throws IOException {
+    final FileSystem fs = FileSystem.get(new ScenarioConfiguration());
+    fs.delete(new Path(TestConstants.NATIVETASK_NONSORT_TEST_DIR), true);
+    fs.close();
+  }
+
+
   private Job getJob(Configuration conf, String jobName, String inputpath, String outputpath) throws IOException {
     final FileSystem fs = FileSystem.get(conf);
     if (fs.exists(new Path(outputpath))) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java

@@ -19,49 +19,53 @@
 
 public class TestConstants {
   // conf path
+  public static final String COMMON_CONF_PATH = "common_conf.xml";
   public static final String COMBINER_CONF_PATH = "test-combiner-conf.xml";
   public static final String KVTEST_CONF_PATH = "kvtest-conf.xml";
   public static final String NONSORT_TEST_CONF = "test-nonsort-conf.xml";
+  public static final String COMPRESS_TEST_CONF_PATH = "test-compress-conf.xml";
 
+  // common constants
+  public static final String NATIVETASK_TEST_DIR = System.getProperty("test.build.data", "target/test/data");
+  public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class";
+  public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator";
   public static final String NATIVETASK_KVSIZE_MIN = "nativetask.kvsize.min";
   public static final String NATIVETASK_KVSIZE_MAX = "nativetask.kvsize.max";
+  public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv";
 
-  public static final String NATIVETASK_KVTEST_INPUTDIR = "nativetask.kvtest.inputdir";
+  // kv test
-  public static final String NATIVETASK_KVTEST_OUTPUTDIR = "nativetask.kvtest.outputdir";
+  public static final String FILESIZE_KEY = "kvtest.file.size";
-  public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = "normal.kvtest.outputdir";
+  public static final String NATIVETASK_KVTEST_DIR = NATIVETASK_TEST_DIR + "/kvtest";
+  public static final String NATIVETASK_KVTEST_INPUTDIR = NATIVETASK_KVTEST_DIR + "/input";
+  public static final String NATIVETASK_KVTEST_NATIVE_OUTPUTDIR = NATIVETASK_KVTEST_DIR + "/output/native";
+  public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = NATIVETASK_KVTEST_DIR + "/output/normal";
   public static final String NATIVETASK_KVTEST_CREATEFILE = "nativetask.kvtest.createfile";
   public static final String NATIVETASK_KVTEST_FILE_RECORDNUM = "nativetask.kvtest.file.recordnum";
   public static final String NATIVETASK_KVTEST_KEYCLASSES = "nativetask.kvtest.keyclasses";
   public static final String NATIVETASK_KVTEST_VALUECLASSES = "nativetask.kvtest.valueclasses";
-  public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class";
-  public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator";
 
-  public static final String SNAPPY_COMPRESS_CONF_PATH = "test-snappy-compress-conf.xml";
+  // compress test
-  public static final String GZIP_COMPRESS_CONF_PATH = "test-gzip-compress-conf.xml";
-  public static final String BZIP2_COMPRESS_CONF_PATH = "test-bzip2-compress-conf.xml";
-  public static final String DEFAULT_COMPRESS_CONF_PATH = "test-default-compress-conf.xml";
-  public static final String LZ4_COMPRESS_CONF_PATH = "test-lz4-compress-conf.xml";
   public static final String NATIVETASK_COMPRESS_FILESIZE = "nativetask.compress.filesize";
+  public static final String NATIVETASK_COMPRESS_TEST_DIR = NATIVETASK_TEST_DIR + "/compresstest";
+  public static final String NATIVETASK_COMPRESS_TEST_INPUTDIR = NATIVETASK_COMPRESS_TEST_DIR + "/input";
+  public static final String NATIVETASK_COMPRESS_TEST_NATIVE_OUTPUTDIR = NATIVETASK_COMPRESS_TEST_DIR + "/output/native";
+  public static final String NATIVETASK_COMPRESS_TEST_NORMAL_OUTPUTDIR = NATIVETASK_COMPRESS_TEST_DIR + "/output/normal";
 
-  public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_KEY = "nativetask.combinertest.inputpath";
+  // combiner test
-  public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV = "./combinertest/input";
+  public static final String NATIVETASK_COMBINER_TEST_DIR = NATIVETASK_TEST_DIR + "/combinertest";
-  public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH = "nativetask.combinertest.outputdir";
+  public static final String NATIVETASK_COMBINER_TEST_INPUTDIR = NATIVETASK_COMBINER_TEST_DIR + "/input";
-  public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/native";
+  public static final String NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR = NATIVETASK_COMBINER_TEST_DIR + "/output/native";
-  public static final String NORMAL_TEST_COMBINER_OUTPUTPATH = "normal.combinertest.outputdir";
+  public static final String NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR = NATIVETASK_COMBINER_TEST_DIR + "/output/normal";
-  public static final String NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/normal";
+  public static final String NATIVETASK_OLDAPI_COMBINER_TEST_NATIVE_OUTPUTPATH = NATIVETASK_COMBINER_TEST_DIR + "/oldapi/output/native";
-  public static final String OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH = "oldAPI.nativetask.combinertest.outputdir";
+  public static final String NATIVETASK_OLDAPI_COMBINER_TEST_NORMAL_OUTPUTPATH = NATIVETASK_COMBINER_TEST_DIR + "/oldapi/output/normal";
-  public static final String OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH = "oldAPI.normal.combinertest.outputdir";
   public static final String NATIVETASK_COMBINER_WORDCOUNT_FILESIZE = "nativetask.combiner.wordcount.filesize";
   public static final String NATIVETASK_NONSORTTEST_FILESIZE = "nativetask.nonsorttest.filesize";
 
-  public static final String COMMON_CONF_PATH = "common_conf.xml";
+  // nonsort test
-
-  public static final String FILESIZE_KEY = "kvtest.file.size";
-  public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv";
-
   public static final String NATIVETASK_MAP_OUTPUT_SORT = "mapreduce.sort.avoidance";
-  public static final String NONSORT_TEST_INPUTDIR = "nativetask.nonsorttest.inputpath";
+  public static final String NATIVETASK_NONSORT_TEST_DIR = NATIVETASK_TEST_DIR + "/nonsorttest";
-  public static final String NONSORT_TEST_NATIVE_OUTPUT = "nonsorttest.native.outputdir";
+  public static final String NATIVETASK_NONSORT_TEST_INPUTDIR = NATIVETASK_NONSORT_TEST_DIR + "/input";
-  public static final String NONSORT_TEST_NORMAL_OUTPUT = "nonsorttest.normal.outputdir";
+  public static final String NATIVETASK_NONSORT_TEST_NATIVE_OUTPUT = NATIVETASK_NONSORT_TEST_DIR + "/output/native";
+  public static final String NATIVETASK_NONSORT_TEST_NORMAL_OUTPUT = NATIVETASK_NONSORT_TEST_DIR + "/output/normal";
 
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml

@@ -1,39 +1,48 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
-<!-- Put site-specific property overrides in this file. -->
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
 
 <configuration>
-<property>
+  <property>
-<name>mapred.job.tracker</name>
+    <name>mapred.job.tracker</name>
-<value>local</value>
+    <value>local</value>
-</property>
+  </property>
-
+  <property>
-<property>
+    <name>keep.failed.task.files</name>
-<name>keep.failed.task.files</name>
+    <value>true</value>
-<value>true</value>
+  </property>
-</property>
-
   <property>
     <name>keep.task.files.pattern</name>
     <value>.*_m_</value>
     <description>Keep all files from tasks whose task names match the given
       regular expression. Defaults to none.</description>
   </property>
-
+  <property>
-<property>
+    <name>nativetask.kvsize.max.largekv</name>
-<name>nativetask.kvsize.max.largekv</name>
+    <value>1048576</value>
-<value>1048576</value>
+  </property>
-</property>
+  <property>
-
+    <name>native.processor.buffer.kb</name>
-<property>
+    <value>128</value>
-<name>native.processor.buffer.kb</name>
+  </property>
-<value>128</value>
+  <property>
-</property>
+    <name>nativetask.output.manager</name>
-
+    <value>org.apache.hadoop.mapred.nativetask.util.LocalJobOutputFiles</value>
-<property>
+  </property>
-<name>nativetask.output.manager</name>
-<value>org.apache.hadoop.mapred.nativetask.util.LocalJobOutputFiles</value>
-</property>
-
 </configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml

@@ -1,87 +1,85 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
-<!-- Put site-specific property overrides in this file. -->
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
 
 <configuration>
-<property>
+  <property>
-<name>nativetask.kvtest.inputdir</name>
+    <name>kvtest.file.size</name>
-<value>./kvtest/input</value>
+    <value>10485760</value>
-</property>
+  </property>
-<property>
+  <property>
-<name>nativetask.kvtest.outputdir</name>
+    <name>mapreduce.reduce.class</name>
-<value>./kvtest/output/native</value>
+    <value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
+  </property>
-<property>
+  <property>
-<name>normal.kvtest.outputdir</name>
+    <name>mapred.output.value.class</name>
-<value>./kvtest/output/normal</value>
+    <value>org.apache.hadoop.io.IntWritable</value>
-</property>
+  </property>
-<property>
+  <property>
-<name>kvtest.file.size</name>
+    <name>nativetask.kvtest.keyclasses</name>
-<value>10485760</value>
+    <value>
-</property>
+      org.apache.hadoop.io.BytesWritable;
-<property>
+      org.apache.hadoop.io.BooleanWritable;
-<name>mapreduce.reduce.class</name>
+      org.apache.hadoop.io.ByteWritable;
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
+      org.apache.hadoop.io.DoubleWritable;
-</property>
+      org.apache.hadoop.io.FloatWritable;
-<property>
+      org.apache.hadoop.io.IntWritable;
-<name>mapred.output.value.class</name>
+      org.apache.hadoop.io.LongWritable;
-<value>org.apache.hadoop.io.IntWritable</value>
+      org.apache.hadoop.io.Text;
-</property>
+      org.apache.hadoop.io.VIntWritable;
-<property>
+      org.apache.hadoop.io.VLongWritable;
-<name>nativetask.kvtest.keyclasses</name>
+    </value>
-<value>
+  </property>
-org.apache.hadoop.io.BytesWritable;
+  <property>
-org.apache.hadoop.io.BooleanWritable;
+    <name>nativetask.kvtest.valueclasses</name>
-org.apache.hadoop.io.ByteWritable;
+    <value>
-org.apache.hadoop.io.DoubleWritable;
+      org.apache.hadoop.io.BytesWritable;
-org.apache.hadoop.io.FloatWritable;
+      org.apache.hadoop.io.BooleanWritable;
-org.apache.hadoop.io.IntWritable;
+      org.apache.hadoop.io.ByteWritable;
-org.apache.hadoop.io.LongWritable;
+      org.apache.hadoop.io.DoubleWritable;
-org.apache.hadoop.io.Text;
+      org.apache.hadoop.io.FloatWritable;
-org.apache.hadoop.io.VIntWritable;
+      org.apache.hadoop.io.IntWritable;
-org.apache.hadoop.io.VLongWritable;
+      org.apache.hadoop.io.LongWritable;
-</value>
+      org.apache.hadoop.io.Text;
-</property>
+      org.apache.hadoop.io.VIntWritable;
-<property>
+      org.apache.hadoop.io.VLongWritable;
-<name>nativetask.kvtest.valueclasses</name>
+      org.apache.hadoop.mapred.nativetask.testutil.MockValueClass;
-<value>
+    </value>
-org.apache.hadoop.io.BytesWritable;
+  </property>
-org.apache.hadoop.io.BooleanWritable;
+  <property>
-org.apache.hadoop.io.ByteWritable;
+    <name>mapred.output.compress</name>
-org.apache.hadoop.io.DoubleWritable;
+    <value>false</value>
-org.apache.hadoop.io.FloatWritable;
+  </property>
-org.apache.hadoop.io.IntWritable;
+  <property>
-org.apache.hadoop.io.LongWritable;
+    <name>mapred.output.compression.type</name>
-org.apache.hadoop.io.Text;
+    <value>BLOCK</value>
-org.apache.hadoop.io.VIntWritable;
+  </property>
-org.apache.hadoop.io.VLongWritable;
+  <property>
-org.apache.hadoop.mapred.nativetask.testutil.MockValueClass;
+    <name>mapred.compress.map.output</name>
-</value>
+    <value>true</value>
-</property>
+  </property>
-
+  <property>
-<property>
+    <name>mapred.map.output.compression.codec</name>
-<name>mapred.output.compress</name>
+    <value>org.apache.hadoop.io.compress.SnappyCodec</value>
-<value>false</value>
+  </property>
-</property>
+  <property>
-
+    <name>hadoop.native.lib</name>
-
+    <value>true</value>
-<property>
+  </property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.SnappyCodec</value>
-</property>
-
-
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
 </configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml

@@ -1,15 +1,30 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
 <configuration>
-
+  <property>
-<property>
+    <name>mapreduce.job.map.output.collector.class</name>
-<name>mapreduce.job.map.output.collector.class</name>
+    <value>org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator</value>
-<value>org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator</value>
+  </property>
-</property>
+  <property>
-
+    <name>io.sort.mb</name>
-<property>
+    <value>5</value>
-<name>io.sort.mb</name>
+  </property>
-<value>5</value>
-</property>
-
 </configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml

@@ -1,10 +1,26 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
 <configuration>
-
+  <property>
-<property>
+    <name>io.sort.mb</name>
-<name>io.sort.mb</name>
+    <value>30</value>
-<value>30</value>
+  </property>
-</property>
-
 </configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml

@@ -1,36 +1,30 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
-<!-- Put site-specific property overrides in this file. -->
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
 
 <configuration>
-<property>
+  <property>
-<name>nativetask.combinertest.inputpath</name>
+    <name>nativetask.combiner.wordcount.filesize</name>
-<value>./combinertest/input</value>
+    <value>20971520</value>
-</property>
+  </property>
-<property>
+  <property>
-<name>nativetask.combiner.wordcount.filesize</name>
+    <name>mapred.job.tracker</name>
-<value>20971520</value>
+    <value>local</value>
-</property>
+  </property>
-
-<property>
-<name>nativetask.combinertest.outputdir</name>
-<value>./combinertest/output/native</value>
-</property>
-<property>
-<name>normal.combinertest.outputdir</name>
-<value>./combinertest/output/normal</value>
-</property>
-<property>
-<name>oldAPI.nativetask.combinertest.outputdir</name>
-<value>./combinertest/oldapi/output/native</value>
-</property>
-<property>
-<name>oldAPI.normal.combinertest.outputdir</name>
-<value>./combinertest/oldapi/output/normal</value>
-</property>
-<property>
-<name>mapred.job.tracker</name>
-<value>local</value>
-</property>
 </configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-compress-conf.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+  <property>
+    <name>mapred.output.compress</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>mapred.output.value.class</name>
+    <value>org.apache.hadoop.io.IntWritable</value>
+  </property>
+  <property>
+    <name>nativetask.compress.filesize</name>
+    <value>100000</value>
+  </property>
+  <property>
+    <name>mapreduce.reduce.class</name>
+    <value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
+  </property>
+  <property>
+    <name>mapred.compress.map.output</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>mapred.output.compression.type</name>
+    <value>BLOCK</value>
+  </property>
+  <property>
+    <name>hadoop.native.lib</name>
+    <value>true</value>
+  </property>
+</configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml

@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>nativetask.compress.filesize</name>
-<value>100000</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.GzipCodec</value>
-</property>
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
-</configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml

@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>nativetask.compress.filesize</name>
-<value>100000</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.Lz4Codec</value>
-</property>
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
-</configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml

@@ -1,31 +1,30 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
-<!-- Put site-specific property overrides in this file. -->
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
 
 <configuration>
-<property>
+  <property>
-<name>nativetask.nonsorttest.inputpath</name>
+    <name>nativetask.nonsorttest.filesize</name>
-<value>./sorttest/input</value>
+    <value>4194304</value>
-</property>
+  </property>
-<property>
+  <property>
-<name>nativetask.nonsorttest.filesize</name>
+    <name>mapred.job.tracker</name>
-<value>4194304</value>
+    <value>local</value>
-</property>
+  </property>
-<property>
-<name>nonsorttest.native.outputdir.tmp</name>
-<value>./sorttest/output/tmp</value>
-</property>
-<property>
-<name>nonsorttest.native.outputdir</name>
-<value>./sorttest/output/native</value>
-</property>
-<property>
-<name>nonsorttest.normal.outputdir</name>
-<value>./sort/output/normal</value>
-</property>
-<property>
-<name>mapred.job.tracker</name>
-<value>local</value>
-</property>
 </configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml

@@ -1,41 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-<property>
-<name>mapred.output.compress</name>
-<value>false</value>
-</property>
-<property>
-<name>mapreduce.reduce.class</name>
-<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value>
-</property>
-
-<property>
-<name>mapred.output.value.class</name>
-<value>org.apache.hadoop.io.IntWritable</value>
-</property>
-<property>
-<name>nativetask.compress.filesize</name>
-<value>100000</value>
-</property>
-
-<property>
-<name>mapred.compress.map.output</name>
-<value>true</value>
-</property>
-<property>
-<name>mapred.output.compression.type</name>
-<value>BLOCK</value>
-</property>
-<property>
-<name>mapred.map.output.compression.codec</name>
-<value>org.apache.hadoop.io.compress.SnappyCodec</value>
-</property>
-<property>
-<name>hadoop.native.lib</name>
-<value>true</value>
-</property>
-</configuration>