diff --git a/mapreduce/CHANGES.txt b/mapreduce/CHANGES.txt index 988c40eda0b..75c454e9144 100644 --- a/mapreduce/CHANGES.txt +++ b/mapreduce/CHANGES.txt @@ -186,6 +186,9 @@ Trunk (unreleased changes) BUG FIXES + MAPREDUCE-2603. Disable High-Ram emulation in system tests. + (Vinay Kumar Thota via amarrk) + MAPREDUCE-2539. Fixed NPE in getMapTaskReports in JobClient. (Robert Evans via acmurthy) diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java index 4144bae842e..3ade9e34e68 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationEnableForAllTypesOfJobs.java @@ -56,6 +56,7 @@ public void testInputCompressionEmualtionEnableForAllJobsWithDefaultRatios() final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true", "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.46", "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.35", @@ -84,6 +85,7 @@ public void testInputCompressionEmulationEnableForAllJobsWithCustomRatios() final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java index 6f0dcbff0f0..4b7fc3a15aa 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForCompressInAndUncompressOut.java @@ -56,6 +56,7 @@ public void testCompressionEmulationOfCompressedInputWithDefaultRatios() final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true" }; @@ -85,6 +86,7 @@ public void testCompressionEmulationOfCompressedInputWithCustomRatios() "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true", "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.58", "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.42" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java index 70dc0d12764..383fc83de4b 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationForUncompressInAndCompressOut.java @@ -54,6 +54,7 @@ public void testCompressionEmulationOfCompressedOuputWithDefaultRatios() final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true" }; @@ -82,6 +83,7 @@ public void testCompressionEmulationOfCompressedOutputWithCustomRatios() final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.38" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java index d98b259177a..a1ae1e9dfaf 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java @@ -59,6 +59,7 @@ public void testGenerateDataEmulateHDFSAndLocalFSDCFiles() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -85,6 +86,7 @@ public void testEmulationOfHDFSAndLocalFSDCFiles() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java index 00d2e4825a2..7f8938f88a7 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java @@ -58,6 +58,7 @@ public void testGenerateAndEmulationOfHDFSDCFile() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -81,6 +82,7 @@ public void testGridmixEmulationOfHDFSPublicDCFile() tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java index 3840f1bbeaf..453e5b99081 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java @@ -60,6 +60,7 @@ public void testGenerateAndEmulateOfHDFSDCFilesWithDiffVisibilities() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -81,6 +82,7 @@ public void testHDFSDCFilesWithoutEnableDCEmulation() "REPLAY", tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java index e50eb6e2e81..eff47f2d641 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java @@ -57,6 +57,7 @@ public void testGenerateInputAndEmulateLocalFSDCFile() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -83,6 +84,7 @@ public void testEmulationOfLocalFSDCFile() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java index f1501bf850b..ef273b5fd25 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java @@ -93,6 +93,7 @@ public void testGenerateDataWithSTRESSSubmission() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; int exitCode = @@ -123,6 +124,7 @@ public void testGenerateDataWithREPLAYSubmission() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; @@ -154,6 +156,7 @@ public void testGenerateDataWithSERIALSubmission() throws Exception { long bytesPerFile = 200 * 1024 * 1024; // 200 mb per file of data String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile, + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java index 1ad10d8af50..883feec88fc 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java @@ -80,6 +80,7 @@ public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java index adaa0d2363b..3fdd16d7f6f 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java @@ -59,6 +59,7 @@ public void testGridmixCompressionRatiosAgainstDefaultCompressionRatio() final String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -89,6 +90,7 @@ public void testGridmixOuputCompressionRatiosAgainstCustomRatios() "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true", "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.68", "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.35", "-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.40" diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java index 5289bf3c8a1..e6c7e6af46b 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java @@ -56,6 +56,7 @@ public void testGenerateAndEmulateOfHDFSPrivateDCFile() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -78,6 +79,7 @@ public void testGridmixEmulationOfHDFSPrivateDCFile() "REPLAY", tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java index e12180c72e4..0bf07fdf4d2 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java @@ -55,6 +55,7 @@ public void testGenerateAndEmulationOfSingleHDFSDCFile() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -80,6 +81,7 @@ public void testGridmixEmulationOfSingleHDFSPublicDCFile() tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java index 4dca1a214ce..5f464ce39be 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java @@ -58,6 +58,7 @@ public void testGenerateAndEmulationOfMultipleHDFSPrivateDCFiles() tracePath}; final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -81,6 +82,7 @@ public void testGridmixEmulationOfMultipleHDFSPrivateDCFiles() "STRESS", tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java index 09bbf181226..cca5da83ecb 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java @@ -59,6 +59,7 @@ public void testGenerateAndEmulationOfMultipleHDFSDCFiles() final String [] otherArgs = { "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, @@ -81,6 +82,7 @@ public void testGridmixEmulationOfMulitpleHDFSPublicDCFile() tracePath}; final String [] otherArgs = { + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath, diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java index c48a7461d3f..ec11a2b36e6 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java @@ -55,6 +55,7 @@ public void testGridmixWith10minTrace() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize, "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=false", "-D", GridMixConfig.GRIDMIX_SLEEPJOB_MAPTASK_ONLY + "=true", diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java index ec2a1377bd6..9bcb45a3fbb 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java @@ -51,6 +51,7 @@ public void testGridmixWith12minTrace() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_SLEEP_MAP_MAX_TIME + "=10", "-D", GridMixConfig.GRIDMIX_SLEEP_REDUCE_MAX_TIME + "=5" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java index ed2648448fa..c583e6d3a29 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java @@ -49,6 +49,7 @@ public void testGridmixWith1minTrace() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java index 9628dd2db8d..d9fb7c70f7f 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java @@ -56,6 +56,7 @@ public void testGridmixWith2minStreamJobTrace() throws Exception { "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true", "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize, "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; runGridmixAndVerify(runtimeValues, otherArgs, tracePath); diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java index 926f795b747..85dedf6675f 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java @@ -59,6 +59,7 @@ public void testGridmixWith3minStreamJobTrace() throws Exception { "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true", "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile, "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java index bed33d0dd3e..5f2171fb401 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java @@ -52,6 +52,7 @@ public void testGridmixWith3minTrace() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java index 370f120aa96..ef1878c0855 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java @@ -56,6 +56,7 @@ public void testGridmixWith5minStreamJobTrace() throws Exception { "-D", GridMixConfig.GRIDMIX_KEY_FRC + "=0.5f", "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile, "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java index 5a141d4c815..c55167e3b4f 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minTrace.java @@ -51,6 +51,7 @@ public void testGridmixWith5minTrace() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize }; diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java index 0791d68aee2..55be37b17dd 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith7minTrace.java @@ -51,6 +51,7 @@ public void testGridmixWith7minTrace() throws Exception { String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize, "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=false" diff --git a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java index 46872b2418c..ae71ec5764b 100644 --- a/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java +++ b/mapreduce/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java @@ -316,7 +316,9 @@ public JobConf getSimulatedJobConf(JobID simulatedJobID, File tmpJHFolder) Path jhpath = new Path(historyFilePath); fs = jhpath.getFileSystem(conf); fs.copyToLocalFile(jhpath,new Path(tmpJHFolder.toString())); - fs.copyToLocalFile(new Path(historyFilePath + "_conf.xml"), + String historyPath = + historyFilePath.substring(0,historyFilePath.lastIndexOf("_")); + fs.copyToLocalFile(new Path(historyPath + "_conf.xml"), new Path(tmpJHFolder.toString())); JobConf jobConf = new JobConf(); jobConf.addResource(new Path(tmpJHFolder.toString() diff --git a/mapreduce/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz b/mapreduce/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz index c4d4657c3cb..229d8d321bc 100644 Binary files a/mapreduce/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz and b/mapreduce/src/contrib/gridmix/src/test/system/resources/highram_mr_jobs_case4.json.gz differ