MAPREDUCE-2603. Disable High-Ram emulation in system tests. (Vinay Kumar Thota via amarrk)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1138301 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Amar Kamat 2011-06-22 04:48:53 +00:00
parent a851ce5dcb
commit c7c8f2fe48
26 changed files with 43 additions and 1 deletions

View File

@ -186,6 +186,9 @@ Trunk (unreleased changes)
BUG FIXES
MAPREDUCE-2603. Disable High-Ram emulation in system tests.
(Vinay Kumar Thota via amarrk)
MAPREDUCE-2539. Fixed NPE in getMapTaskReports in JobClient. (Robert Evans via
acmurthy)

View File

@ -56,6 +56,7 @@ public class TestCompressionEmulationEnableForAllTypesOfJobs
final String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true",
"-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.46",
"-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.35",
@ -84,6 +85,7 @@ public class TestCompressionEmulationEnableForAllTypesOfJobs
final String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};

View File

@ -56,6 +56,7 @@ public class TestCompressionEmulationForCompressInAndUncompressOut
final String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true"
};
@ -85,6 +86,7 @@ public class TestCompressionEmulationForCompressInAndUncompressOut
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true",
"-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.58",
"-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.42"
};

View File

@ -54,6 +54,7 @@ public class TestCompressionEmulationForUncompressInAndCompressOut
final String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true"
};
@ -82,6 +83,7 @@ public class TestCompressionEmulationForUncompressInAndCompressOut
final String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.38"
};

View File

@ -59,6 +59,7 @@ public class TestEmulationOfHDFSAndLocalFSDCFiles extends
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -85,6 +86,7 @@ public class TestEmulationOfHDFSAndLocalFSDCFiles extends
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,

View File

@ -58,6 +58,7 @@ public class TestEmulationOfHDFSDCFileUsesMultipleJobs extends
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -81,6 +82,7 @@ public class TestEmulationOfHDFSDCFileUsesMultipleJobs extends
tracePath};
final String [] otherArgs = {
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,

View File

@ -60,6 +60,7 @@ public class TestEmulationOfHDFSDCFilesWithDifferentVisibilities
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -81,6 +82,7 @@ public class TestEmulationOfHDFSDCFilesWithDifferentVisibilities
"REPLAY",
tracePath};
final String [] otherArgs = {
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,

View File

@ -57,6 +57,7 @@ public class TestEmulationOfLocalFSDCFiles extends GridmixSystemTestCase {
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -83,6 +84,7 @@ public class TestEmulationOfLocalFSDCFiles extends GridmixSystemTestCase {
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,

View File

@ -93,6 +93,7 @@ public class TestGridMixDataGeneration {
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
int exitCode =
@ -123,6 +124,7 @@ public class TestGridMixDataGeneration {
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
@ -154,6 +156,7 @@ public class TestGridMixDataGeneration {
long bytesPerFile = 200 * 1024 * 1024; // 200 mb per file of data
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};

View File

@ -80,6 +80,7 @@ public class TestGridMixFilePool {
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};

View File

@ -59,6 +59,7 @@ public class TestGridmixCompressionEmulationWithCompressInput
final String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -89,6 +90,7 @@ public class TestGridmixCompressionEmulationWithCompressInput
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true",
"-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.68",
"-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.35",
"-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.40"

View File

@ -56,6 +56,7 @@ public class TestGridmixEmulationOfHDFSPrivateDCFile
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -78,6 +79,7 @@ public class TestGridmixEmulationOfHDFSPrivateDCFile
"REPLAY",
tracePath};
final String [] otherArgs = {
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,

View File

@ -55,6 +55,7 @@ public class TestGridmixEmulationOfHDFSPublicDCFile
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -80,6 +81,7 @@ public class TestGridmixEmulationOfHDFSPublicDCFile
tracePath};
final String [] otherArgs = {
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,

View File

@ -58,6 +58,7 @@ public class TestGridmixEmulationOfMultipleHDFSPrivateDCFiles
tracePath};
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -81,6 +82,7 @@ public class TestGridmixEmulationOfMultipleHDFSPrivateDCFiles
"STRESS",
tracePath};
final String [] otherArgs = {
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,

View File

@ -59,6 +59,7 @@ public class TestGridmixEmulationOfMultipleHDFSPublicDCFiles
final String [] otherArgs = {
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
@ -81,6 +82,7 @@ public class TestGridmixEmulationOfMultipleHDFSPublicDCFiles
tracePath};
final String [] otherArgs = {
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,

View File

@ -55,6 +55,7 @@ public class TestGridmixWith10minTrace extends GridmixSystemTestCase {
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize,
"-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=false",
"-D", GridMixConfig.GRIDMIX_SLEEPJOB_MAPTASK_ONLY + "=true",

View File

@ -51,6 +51,7 @@ public class TestGridmixWith12minTrace extends GridmixSystemTestCase {
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_SLEEP_MAP_MAX_TIME + "=10",
"-D", GridMixConfig.GRIDMIX_SLEEP_REDUCE_MAX_TIME + "=5"
};

View File

@ -49,6 +49,7 @@ public class TestGridmixWith1minTrace extends GridmixSystemTestCase{
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};

View File

@ -56,6 +56,7 @@ public class TestGridmixWith2minStreamingJobTrace
"-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true",
"-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize,
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
runGridmixAndVerify(runtimeValues, otherArgs, tracePath);

View File

@ -59,6 +59,7 @@ public class TestGridmixWith3minStreamingJobTrace
"-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true",
"-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};

View File

@ -52,6 +52,7 @@ public class TestGridmixWith3minTrace extends GridmixSystemTestCase {
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};

View File

@ -56,6 +56,7 @@ public class TestGridmixWith5minStreamingJobTrace
"-D", GridMixConfig.GRIDMIX_KEY_FRC + "=0.5f",
"-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};

View File

@ -51,6 +51,7 @@ public class TestGridmixWith5minTrace extends GridmixSystemTestCase {
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize
};

View File

@ -51,6 +51,7 @@ public class TestGridmixWith7minTrace extends GridmixSystemTestCase {
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize,
"-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=false"

View File

@ -316,7 +316,9 @@ public class GridmixJobVerification {
Path jhpath = new Path(historyFilePath);
fs = jhpath.getFileSystem(conf);
fs.copyToLocalFile(jhpath,new Path(tmpJHFolder.toString()));
fs.copyToLocalFile(new Path(historyFilePath + "_conf.xml"),
String historyPath =
historyFilePath.substring(0,historyFilePath.lastIndexOf("_"));
fs.copyToLocalFile(new Path(historyPath + "_conf.xml"),
new Path(tmpJHFolder.toString()));
JobConf jobConf = new JobConf();
jobConf.addResource(new Path(tmpJHFolder.toString()