MAPREDUCE-3375. [Gridmix] Memory Emulation system tests. (Vinay Thota via amarrk)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1201999 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -7,6 +7,8 @@ Release 0.23.1 - Unreleased
   NEW FEATURES                                                                    
 
   IMPROVEMENTS
+    MAPREDUCE-3375. [Gridmix] Memory Emulation system tests.
+                    (Vinay Thota via amarrk)
 
     MAPREDUCE-2733. [Gridmix] Gridmix3 cpu emulation system tests. 
                     (Vinay Thota via amarrk)

hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java

@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for the jobs with 
+ * custom progress interval, different input data, submission policies
+ * and user resolver modes. Verify the total heap usage of map and reduce
+ * tasks of the jobs with corresponding original job in the trace. 
+ */
+public class TestMemEmulForMapsAndReducesWithCustomIntrvl 
+    extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+      LogFactory.getLog("TestMemEmulForMapsAndReducesWithCustomIntrvl.class");
+  /**
+   * Generate compressed input and run {@link Gridmix} by turning on the
+   * memory emulation with custom progress interval. The {@link Gridmix} 
+   * should use the following runtime parameters while running the jobs.
+   * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+   * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs 
+   * with corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForReducesWithCompressedInputCase7() 
+      throws Exception { 
+    final long inputSizeInMB = 1024 * 7;
+    String tracePath = getTraceFile("mem_emul_case2");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+            { "LOADJOB",
+              RoundRobinUserResolver.class.getName(),
+              "STRESS",
+              inputSizeInMB + "m",
+              "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+              tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + 
+                GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.3F",
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+           GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on the
+   * memory emulation with custom progress interval. The {@link Gridmix}
+   * should use the following runtime parameters while running the jobs.
+   * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+   * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs 
+   * with corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForReducesWithUncompressedInputCase8()
+      throws Exception {
+    final long inputSizeInMB = cSize * 300;
+    String tracePath = getTraceFile("mem_emul_case2");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+              { "LOADJOB", 
+                SubmitterUserResolver.class.getName(), 
+                "REPLAY", 
+                inputSizeInMB + "m",
+                tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + 
+                  GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.2F",
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+            GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+}

hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java

@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for gridmix jobs
+ * with default progress interval, different input data, submission 
+ * policies and user resolver modes. Verify the total heap usage of
+ * map and reduce tasks of the jobs with corresponding original
+ * job in the trace. 
+ */
+public class TestMemEmulForMapsAndReducesWithDefaultIntrvl 
+    extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+      LogFactory.getLog("TestMemEmulForMapsAndReducesWithDefaultIntrvl.class");
+
+  /**
+   * Generate compressed input and run {@link Gridmix} by turning on the
+   * memory emulation with default progress interval. The {@link Gridmix} 
+   * should use the following runtime parameters while running the jobs.
+   * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+   * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs 
+   * with corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForReducesWithCompressedInputCase5() 
+      throws Exception { 
+    final long inputSizeInMB = 1024 * 7;
+    String tracePath = getTraceFile("mem_emul_case2");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+            { "LOADJOB",
+              RoundRobinUserResolver.class.getName(),
+              "STRESS",
+              inputSizeInMB + "m",
+              "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+              tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + 
+                GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+           GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on the
+   * memory emulation with default progress interval. The {@link Gridmix} 
+   * should use the following runtime parameters while running the jobs. 
+   * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+   * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs 
+   * with corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForReducesWithUncompressedInputCase6()
+      throws Exception {
+    final long inputSizeInMB = cSize * 300;
+    String tracePath = getTraceFile("mem_emul_case2");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+              { "LOADJOB", 
+                SubmitterUserResolver.class.getName(), 
+                "REPLAY", 
+                inputSizeInMB + "m",
+                tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + 
+                  GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+            GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+}

hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java

@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs 
+ * with default progress interval, custom heap memory ratio, different input 
+ * data, submission policies and user resolver modes. Verify the total heap 
+ * usage of map and reduce tasks of the jobs with corresponding the original job 
+ * in the trace. 
+ */
+public class TestMemEmulForMapsWithCustomHeapMemoryRatio 
+    extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+      LogFactory.getLog("TestMemEmulForMapsWithCustomHeapMemoryRatio.class");
+
+  /**
+   * Generate compressed input and run {@link Gridmix} by turning on the
+   * memory emulation. The {@link Gridmix} should use the following runtime 
+   * parameters while running the jobs.
+   * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+   * Verify total heap memory usage of the tasks of {@link Gridmix} jobs with 
+   * corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForMapsWithCompressedInputCase1() 
+     throws Exception {
+    final long inputSizeInMB = 1024 * 7;
+    String tracePath = getTraceFile("mem_emul_case2");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    String [] runtimeValues = 
+            { "LOADJOB",
+              SubmitterUserResolver.class.getName(),
+              "STRESS",
+              inputSizeInMB + "m",
+              tracePath};
+
+    String [] otherArgs = { 
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "="  +
+                  GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+            "-D", GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO + "=0.5F"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+           GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on the
+   * memory emulation. The {@link Gridmix} should use the following runtime 
+   * parameters while running the jobs.
+   *  Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
+   * Verify total heap memory usage of tasks of {@link Gridmix} jobs with 
+   * corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForMapsWithUncompressedInputCase2() 
+      throws Exception {
+    final long inputSizeInMB = cSize * 300;
+    String tracePath = getTraceFile("mem_emul_case2");
+    Assert.assertNotNull("Trace file has not found.", tracePath);
+    String [] runtimeValues = 
+              { "LOADJOB", 
+                RoundRobinUserResolver.class.getName(), 
+                "STRESS",
+                inputSizeInMB + "m",
+                "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+                tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +  
+                  GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+            "-D", GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO + "=0.4F"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+            GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+}

hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java

@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs 
+ * with custom progress interval, different input data, submission policies 
+ * and user resolver modes. Verify the total heap usage of map tasks of
+ * the jobs with corresponding the original job in the trace. 
+ */
+public class TestMemEmulForMapsWithCustomIntrvl extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+      LogFactory.getLog("TestMemEmulForMapsWithCustomIntrvl.class");
+
+  /**
+   * Generate compressed input and run {@link Gridmix} by turning on the
+   * memory emulation with custom progress interval. The {@link Gridmix}
+   * should use the following runtime parameters while running the jobs.
+   * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+   * Verify maps total heap memory usage of {@link Gridmix} jobs with 
+   * corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForMapsWithCompressedInputCase3() 
+     throws Exception {
+    final long inputSizeInMB = 1024 * 7;
+    String tracePath = getTraceFile("mem_emul_case1");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+            { "LOADJOB",
+              SubmitterUserResolver.class.getName(),
+              "STRESS",
+              inputSizeInMB + "m",
+              tracePath};
+
+    String [] otherArgs = { 
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "="  +
+                  GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.2F",
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+           GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on the
+   * memory emulation with custom progress interval. The {@link Gridmix} 
+   * should use the following runtime parameters while running the jobs.
+   *  Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
+   * Verify maps total heap memory usage of {@link Gridmix} jobs with 
+   * corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForMapsWithUncompressedInputCase4() 
+      throws Exception {
+    final long inputSizeInMB = cSize * 300;
+    String tracePath = getTraceFile("mem_emul_case1");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+              { "LOADJOB", 
+                RoundRobinUserResolver.class.getName(), 
+                "STRESS",
+                inputSizeInMB + "m",
+                "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+                tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +  
+                  GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.3F",
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+            GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+}

hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java

@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs 
+ * with default progress interval, different input data, submission policies 
+ * and user resolver modes. Verify the total heap usage of map tasks of the 
+ * jobs with corresponding original job in the trace. 
+ */
+public class TestMemEmulForMapsWithDefaultIntrvl extends GridmixSystemTestCase { 
+  private static final Log LOG = 
+          LogFactory.getLog("TestMemEmulForMapsWithDefaultIntrvl.class");
+
+  /**
+   * Generate compressed input and run {@link Gridmix} by turning on the
+   * memory emulation with default progress interval. The {@link Gridmix} 
+   * should use the following runtime parameters while running the jobs.
+   * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+   * Verify maps total heap memory usage of {@link Gridmix} jobs with 
+   * corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForMapsWithCompressedInputCase1() 
+     throws Exception {
+    final long inputSizeInMB = 1024 * 7;
+    String tracePath = getTraceFile("mem_emul_case1");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+            { "LOADJOB",
+              SubmitterUserResolver.class.getName(),
+              "STRESS",
+              inputSizeInMB + "m",
+              tracePath};
+
+    String [] otherArgs = { 
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "="  +
+                  GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+           GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+  
+  /**
+   * Generate uncompressed input and run {@link Gridmix} by turning on the
+   * memory emulation with default progress interval. The {@link Gridmix} 
+   * should use the following runtime parameters while running the jobs.
+   *  Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
+   * Verify maps total heap memory usage of {@link Gridmix} jobs with 
+   * corresponding original job in the trace. 
+   * @throws Exception - if an error occurs.
+   */
+  @Test
+  public void testMemoryEmulationForMapsWithUncompressedInputCase2() 
+      throws Exception {
+    final long inputSizeInMB = cSize * 300;
+    String tracePath = getTraceFile("mem_emul_case1");
+    Assert.assertNotNull("Trace file not found!", tracePath);
+    String [] runtimeValues = 
+              { "LOADJOB", 
+                RoundRobinUserResolver.class.getName(), 
+                "STRESS",
+                inputSizeInMB + "m",
+                "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+                tracePath};
+
+    String [] otherArgs = {
+            "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +  
+                  GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+            "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+            "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+            "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+    runGridmixAndVerify(runtimeValues, otherArgs, tracePath, 
+            GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+  }
+}

hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java

@@ -248,4 +248,38 @@ public class GridMixConfig {
   */
  public static int GRIDMIX_CPU_EMULATION_UPPER_LIMIT = 130;
 
+ /**
+  * Gridmix heap memory custom interval
+  */
+ public static final String GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL = 
+     TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL;
+  
+ /**
+  *  Gridmix heap free memory ratio
+  */
+ public static final String GRIDMIX_HEAP_FREE_MEMORY_RATIO =
+     TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO;
+  
+ /**
+  *  Gridmix memory emulation plugin
+  */
+ public static final String GRIDMIX_MEMORY_EMULATION_PLUGIN = 
+     TotalHeapUsageEmulatorPlugin.class.getName();
+  
+ /**
+  *  Gridmix memory emulation
+  */
+ public static final String GRIDMIX_MEMORY_EMULATON = 
+     ResourceUsageMatcher.RESOURCE_USAGE_EMULATION_PLUGINS;
+  
+ /**
+  *  Gridmix memory emulation lower limit.
+  */
+ public static int GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT = 55;
+  
+ /**
+  * Gridmix memory emulation upper limit. 
+  */
+ public static int GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT = 130;
+
 }

hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java

@@ -149,6 +149,7 @@ public class GridmixJobVerification {
          zombieJob.getJobConf());
       verifyHighRamMemoryJobs(zombieJob, simuJobConf);
       verifyCPUEmulationOfJobs(zombieJob, jhInfo, simuJobConf);
+      verifyMemoryEmulationOfJobs(zombieJob, jhInfo, simuJobConf);
       LOG.info("Done.");
     }
     verifyDistributedCacheBetweenJobs(simuAndOrigJobsInfo);
@@ -356,6 +357,109 @@ public class GridmixJobVerification {
     }
   }
 
+  /**
+   * It verifies the heap memory resource usage of gridmix jobs with
+   * corresponding original job in the trace.
+   * @param zombieJob - Original job history.
+   * @param jhInfo - Simulated job history.
+   * @param simuJobConf - simulated job configuration.
+   */
+  public void verifyMemoryEmulationOfJobs(ZombieJob zombieJob,
+                 JobHistoryParser.JobInfo jhInfo,
+                                 JobConf simuJobConf) throws Exception {
+    long origJobMapsTHU = 0;
+    long origJobReducesTHU = 0;
+    long simuJobMapsTHU = 0;
+    long simuJobReducesTHU = 0;
+    boolean isMemEmulOn = false;
+    if (simuJobConf.get(GridMixConfig.GRIDMIX_MEMORY_EMULATON) != null) {
+      isMemEmulOn = 
+          simuJobConf.get(GridMixConfig.GRIDMIX_MEMORY_EMULATON).
+              contains(GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN);
+    }
+
+    if (isMemEmulOn) {
+      for (int index = 0; index < zombieJob.getNumberMaps(); index ++) {
+        TaskInfo mapTask = zombieJob.getTaskInfo(TaskType.MAP, index);
+        if (mapTask.getResourceUsageMetrics().getHeapUsage() > 0) {
+          origJobMapsTHU += 
+                  mapTask.getResourceUsageMetrics().getHeapUsage();
+        }
+      }
+      LOG.info("Original Job Maps Total Heap Usage: " + origJobMapsTHU);
+
+      for (int index = 0; index < zombieJob.getNumberReduces(); index ++) {
+        TaskInfo reduceTask = zombieJob.getTaskInfo(TaskType.REDUCE, index);
+        if (reduceTask.getResourceUsageMetrics().getHeapUsage() > 0) {
+          origJobReducesTHU += 
+                  reduceTask.getResourceUsageMetrics().getHeapUsage();
+        }
+      }
+      LOG.info("Original Job Reduces Total Heap Usage: " + origJobReducesTHU);
+
+      simuJobMapsTHU = 
+          getCounterValue(jhInfo.getMapCounters(), 
+                          TaskCounter.COMMITTED_HEAP_BYTES.toString());
+      LOG.info("Simulated Job Maps Total Heap Usage: " + simuJobMapsTHU);
+
+      simuJobReducesTHU = 
+          getCounterValue(jhInfo.getReduceCounters(), 
+                          TaskCounter.COMMITTED_HEAP_BYTES.toString());
+      LOG.info("Simulated Jobs Reduces Total Heap Usage: " + simuJobReducesTHU);
+
+      long mapCount = jhInfo.getTotalMaps();
+      long reduceCount = jhInfo.getTotalReduces();
+
+      String strHeapRatio =
+          simuJobConf.get(GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO);
+      if (strHeapRatio == null) {
+        strHeapRatio = "0.3F";
+      }
+
+      if (mapCount > 0) {
+        double mapEmulFactor = (simuJobMapsTHU * 100) / origJobMapsTHU;
+        long mapEmulAccuracy = Math.round(mapEmulFactor);
+        LOG.info("Maps memory emulation accuracy of a job:" 
+                + mapEmulAccuracy + "%");
+        Assert.assertTrue("Map phase total memory emulation had crossed the "
+                         + "configured max limit.", mapEmulAccuracy 
+                         <= GridMixConfig.GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT);
+        Assert.assertTrue("Map phase total memory emulation had not crossed " 
+                         + "the configured min limit.", mapEmulAccuracy 
+                         >= GridMixConfig.GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT);
+        double expHeapRatio = Double.parseDouble(strHeapRatio);
+        LOG.info("expHeapRatio for maps:" + expHeapRatio);
+        double actHeapRatio = 
+                ((double)Math.abs(origJobMapsTHU - simuJobMapsTHU)) ;
+        actHeapRatio /= origJobMapsTHU;
+          LOG.info("actHeapRatio for maps:" + actHeapRatio);
+          Assert.assertTrue("Simulate job maps heap ratio not matched.",
+                            actHeapRatio <= expHeapRatio); 
+      }
+
+      if (reduceCount >0) {
+        double reduceEmulFactor = (simuJobReducesTHU * 100) / origJobReducesTHU;
+        long reduceEmulAccuracy = Math.round(reduceEmulFactor);
+        LOG.info("Reduces memory emulation accuracy of a job:" 
+                + reduceEmulAccuracy + "%");
+        Assert.assertTrue("Reduce phase total memory emulation had crossed "
+                         + "configured max limit.", reduceEmulAccuracy 
+                         <= GridMixConfig.GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT); 
+        Assert.assertTrue("Reduce phase total memory emulation had not " 
+                         + "crosssed configured min limit.", reduceEmulAccuracy 
+                         >= GridMixConfig.GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT);
+        double expHeapRatio = Double.parseDouble(strHeapRatio);
+        LOG.info("expHeapRatio for reduces:" + expHeapRatio);
+        double actHeapRatio = 
+                ((double)Math.abs(origJobReducesTHU - simuJobReducesTHU));
+        actHeapRatio /= origJobReducesTHU;
+          LOG.info("actHeapRatio for reduces:" + actHeapRatio);
+          Assert.assertTrue("Simulate job reduces heap ratio not matched.",
+                            actHeapRatio <= expHeapRatio); 
+      }
+    }
+  }
+
   /**
    * It verifies the cpu resource usage of  a gridmix job against
    * their original job.
@@ -367,7 +471,14 @@ public class GridmixJobVerification {
        JobHistoryParser.JobInfo simuJobHistoryInfo,
        JobConf simuJobConf) throws Exception {
 
+    boolean isCpuEmulOn = false;
     if (simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON) != null) {
+      isCpuEmulOn = 
+          simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON).
+              contains(GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN);
+    }
+
+    if (isCpuEmulOn) {
       Map<String,Long> origJobMetrics =
                        getOriginalJobCPUMetrics(origJobHistory);
       Map<String,Long> simuJobMetrics =