diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index de77619d06a..06942df163b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -10,6 +10,9 @@ Trunk (unreleased changes) (Plamen Jeliazkov via shv) IMPROVEMENTS + MAPREDUCE-3375. [Gridmix] Memory Emulation system tests. + (Vinay Thota via amarrk) + MAPREDUCE-2733. [Gridmix] Gridmix3 cpu emulation system tests. (Vinay Thota via amarrk) diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java new file mode 100644 index 00000000000..a82e806059b --- /dev/null +++ b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.gridmix; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode; +import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.junit.Test; +import org.junit.Assert; + +/** + * Test the {@link Gridmix} memory emulation feature for the jobs with + * custom progress interval, different input data, submission policies + * and user resolver modes. Verify the total heap usage of map and reduce + * tasks of the jobs with corresponding original job in the trace. + */ +public class TestMemEmulForMapsAndReducesWithCustomIntrvl + extends GridmixSystemTestCase { + private static final Log LOG = + LogFactory.getLog("TestMemEmulForMapsAndReducesWithCustomIntrvl.class"); + /** + * Generate compressed input and run {@link Gridmix} by turning on the + * memory emulation with custom progress interval. The {@link Gridmix} + * should use the following runtime parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver + * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs + * with corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForReducesWithCompressedInputCase7() + throws Exception { + final long inputSizeInMB = 1024 * 7; + String tracePath = getTraceFile("mem_emul_case2"); + Assert.assertNotNull("Trace file not found!", tracePath); + String [] runtimeValues = + { "LOADJOB", + RoundRobinUserResolver.class.getName(), + "STRESS", + inputSizeInMB + "m", + "file://" + UtilsForGridmix.getProxyUsersFile(conf), + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.3F", + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } + + /** + * Generate uncompressed input and run {@link Gridmix} by turning on the + * memory emulation with custom progress interval. The {@link Gridmix} + * should use the following runtime parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver + * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs + * with corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForReducesWithUncompressedInputCase8() + throws Exception { + final long inputSizeInMB = cSize * 300; + String tracePath = getTraceFile("mem_emul_case2"); + Assert.assertNotNull("Trace file not found!", tracePath); + String [] runtimeValues = + { "LOADJOB", + SubmitterUserResolver.class.getName(), + "REPLAY", + inputSizeInMB + "m", + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.2F", + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } +} diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java new file mode 100644 index 00000000000..e1f211a11f3 --- /dev/null +++ b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.gridmix; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode; +import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.junit.Test; +import org.junit.Assert; + +/** + * Test the {@link Gridmix} memory emulation feature for gridmix jobs + * with default progress interval, different input data, submission + * policies and user resolver modes. Verify the total heap usage of + * map and reduce tasks of the jobs with corresponding original + * job in the trace. + */ +public class TestMemEmulForMapsAndReducesWithDefaultIntrvl + extends GridmixSystemTestCase { + private static final Log LOG = + LogFactory.getLog("TestMemEmulForMapsAndReducesWithDefaultIntrvl.class"); + + /** + * Generate compressed input and run {@link Gridmix} by turning on the + * memory emulation with default progress interval. The {@link Gridmix} + * should use the following runtime parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver + * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs + * with corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForReducesWithCompressedInputCase5() + throws Exception { + final long inputSizeInMB = 1024 * 7; + String tracePath = getTraceFile("mem_emul_case2"); + Assert.assertNotNull("Trace file not found!", tracePath); + String [] runtimeValues = + { "LOADJOB", + RoundRobinUserResolver.class.getName(), + "STRESS", + inputSizeInMB + "m", + "file://" + UtilsForGridmix.getProxyUsersFile(conf), + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } + + /** + * Generate uncompressed input and run {@link Gridmix} by turning on the + * memory emulation with default progress interval. The {@link Gridmix} + * should use the following runtime parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver + * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs + * with corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForReducesWithUncompressedInputCase6() + throws Exception { + final long inputSizeInMB = cSize * 300; + String tracePath = getTraceFile("mem_emul_case2"); + Assert.assertNotNull("Trace file not found!", tracePath); + String [] runtimeValues = + { "LOADJOB", + SubmitterUserResolver.class.getName(), + "REPLAY", + inputSizeInMB + "m", + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } +} diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java new file mode 100644 index 00000000000..da48ad4538e --- /dev/null +++ b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.gridmix; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode; +import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.junit.Test; +import org.junit.Assert; + +/** + * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs + * with default progress interval, custom heap memory ratio, different input + * data, submission policies and user resolver modes. Verify the total heap + * usage of map and reduce tasks of the jobs with corresponding the original job + * in the trace. + */ +public class TestMemEmulForMapsWithCustomHeapMemoryRatio + extends GridmixSystemTestCase { + private static final Log LOG = + LogFactory.getLog("TestMemEmulForMapsWithCustomHeapMemoryRatio.class"); + + /** + * Generate compressed input and run {@link Gridmix} by turning on the + * memory emulation. The {@link Gridmix} should use the following runtime + * parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver + * Verify total heap memory usage of the tasks of {@link Gridmix} jobs with + * corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForMapsWithCompressedInputCase1() + throws Exception { + final long inputSizeInMB = 1024 * 7; + String tracePath = getTraceFile("mem_emul_case2"); + Assert.assertNotNull("Trace file has not found.", tracePath); + String [] runtimeValues = + { "LOADJOB", + SubmitterUserResolver.class.getName(), + "STRESS", + inputSizeInMB + "m", + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO + "=0.5F"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } + + /** + * Generate uncompressed input and run {@link Gridmix} by turning on the + * memory emulation. The {@link Gridmix} should use the following runtime + * parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver + * Verify total heap memory usage of tasks of {@link Gridmix} jobs with + * corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForMapsWithUncompressedInputCase2() + throws Exception { + final long inputSizeInMB = cSize * 300; + String tracePath = getTraceFile("mem_emul_case2"); + Assert.assertNotNull("Trace file has not found.", tracePath); + String [] runtimeValues = + { "LOADJOB", + RoundRobinUserResolver.class.getName(), + "STRESS", + inputSizeInMB + "m", + "file://" + UtilsForGridmix.getProxyUsersFile(conf), + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false", + "-D", GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO + "=0.4F"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } +} diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java new file mode 100644 index 00000000000..5d1d452b8d6 --- /dev/null +++ b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.gridmix; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode; +import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.junit.Test; +import org.junit.Assert; + +/** + * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs + * with custom progress interval, different input data, submission policies + * and user resolver modes. Verify the total heap usage of map tasks of + * the jobs with corresponding the original job in the trace. + */ +public class TestMemEmulForMapsWithCustomIntrvl extends GridmixSystemTestCase { + private static final Log LOG = + LogFactory.getLog("TestMemEmulForMapsWithCustomIntrvl.class"); + + /** + * Generate compressed input and run {@link Gridmix} by turning on the + * memory emulation with custom progress interval. The {@link Gridmix} + * should use the following runtime parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver + * Verify maps total heap memory usage of {@link Gridmix} jobs with + * corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForMapsWithCompressedInputCase3() + throws Exception { + final long inputSizeInMB = 1024 * 7; + String tracePath = getTraceFile("mem_emul_case1"); + Assert.assertNotNull("Trace file not found!", tracePath); + String [] runtimeValues = + { "LOADJOB", + SubmitterUserResolver.class.getName(), + "STRESS", + inputSizeInMB + "m", + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.2F", + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } + + /** + * Generate uncompressed input and run {@link Gridmix} by turning on the + * memory emulation with custom progress interval. The {@link Gridmix} + * should use the following runtime parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver + * Verify maps total heap memory usage of {@link Gridmix} jobs with + * corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForMapsWithUncompressedInputCase4() + throws Exception { + final long inputSizeInMB = cSize * 300; + String tracePath = getTraceFile("mem_emul_case1"); + Assert.assertNotNull("Trace file not found!", tracePath); + String [] runtimeValues = + { "LOADJOB", + RoundRobinUserResolver.class.getName(), + "STRESS", + inputSizeInMB + "m", + "file://" + UtilsForGridmix.getProxyUsersFile(conf), + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.3F", + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } +} diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java new file mode 100644 index 00000000000..ff136b89c7d --- /dev/null +++ b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.gridmix; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig; +import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode; +import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.junit.Test; +import org.junit.Assert; + +/** + * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs + * with default progress interval, different input data, submission policies + * and user resolver modes. Verify the total heap usage of map tasks of the + * jobs with corresponding original job in the trace. + */ +public class TestMemEmulForMapsWithDefaultIntrvl extends GridmixSystemTestCase { + private static final Log LOG = + LogFactory.getLog("TestMemEmulForMapsWithDefaultIntrvl.class"); + + /** + * Generate compressed input and run {@link Gridmix} by turning on the + * memory emulation with default progress interval. The {@link Gridmix} + * should use the following runtime parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver + * Verify maps total heap memory usage of {@link Gridmix} jobs with + * corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForMapsWithCompressedInputCase1() + throws Exception { + final long inputSizeInMB = 1024 * 7; + String tracePath = getTraceFile("mem_emul_case1"); + Assert.assertNotNull("Trace file not found!", tracePath); + String [] runtimeValues = + { "LOADJOB", + SubmitterUserResolver.class.getName(), + "STRESS", + inputSizeInMB + "m", + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } + + /** + * Generate uncompressed input and run {@link Gridmix} by turning on the + * memory emulation with default progress interval. The {@link Gridmix} + * should use the following runtime parameters while running the jobs. + * Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver + * Verify maps total heap memory usage of {@link Gridmix} jobs with + * corresponding original job in the trace. + * @throws Exception - if an error occurs. + */ + @Test + public void testMemoryEmulationForMapsWithUncompressedInputCase2() + throws Exception { + final long inputSizeInMB = cSize * 300; + String tracePath = getTraceFile("mem_emul_case1"); + Assert.assertNotNull("Trace file not found!", tracePath); + String [] runtimeValues = + { "LOADJOB", + RoundRobinUserResolver.class.getName(), + "STRESS", + inputSizeInMB + "m", + "file://" + UtilsForGridmix.getProxyUsersFile(conf), + tracePath}; + + String [] otherArgs = { + "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" + + GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN, + "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", + "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false", + "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"}; + + runGridmixAndVerify(runtimeValues, otherArgs, tracePath, + GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue()); + } +} diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java index ea0acc53d54..fc99162bd86 100644 --- a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java +++ b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java @@ -248,4 +248,38 @@ public class GridMixConfig { */ public static int GRIDMIX_CPU_EMULATION_UPPER_LIMIT = 130; + /** + * Gridmix heap memory custom interval + */ + public static final String GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL = + TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL; + + /** + * Gridmix heap free memory ratio + */ + public static final String GRIDMIX_HEAP_FREE_MEMORY_RATIO = + TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO; + + /** + * Gridmix memory emulation plugin + */ + public static final String GRIDMIX_MEMORY_EMULATION_PLUGIN = + TotalHeapUsageEmulatorPlugin.class.getName(); + + /** + * Gridmix memory emulation + */ + public static final String GRIDMIX_MEMORY_EMULATON = + ResourceUsageMatcher.RESOURCE_USAGE_EMULATION_PLUGINS; + + /** + * Gridmix memory emulation lower limit. + */ + public static int GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT = 55; + + /** + * Gridmix memory emulation upper limit. + */ + public static int GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT = 130; + } diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java index bef82338749..e448412a39f 100644 --- a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java +++ b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java @@ -149,6 +149,7 @@ public class GridmixJobVerification { zombieJob.getJobConf()); verifyHighRamMemoryJobs(zombieJob, simuJobConf); verifyCPUEmulationOfJobs(zombieJob, jhInfo, simuJobConf); + verifyMemoryEmulationOfJobs(zombieJob, jhInfo, simuJobConf); LOG.info("Done."); } verifyDistributedCacheBetweenJobs(simuAndOrigJobsInfo); @@ -356,6 +357,109 @@ public class GridmixJobVerification { } } + /** + * It verifies the heap memory resource usage of gridmix jobs with + * corresponding original job in the trace. + * @param zombieJob - Original job history. + * @param jhInfo - Simulated job history. + * @param simuJobConf - simulated job configuration. + */ + public void verifyMemoryEmulationOfJobs(ZombieJob zombieJob, + JobHistoryParser.JobInfo jhInfo, + JobConf simuJobConf) throws Exception { + long origJobMapsTHU = 0; + long origJobReducesTHU = 0; + long simuJobMapsTHU = 0; + long simuJobReducesTHU = 0; + boolean isMemEmulOn = false; + if (simuJobConf.get(GridMixConfig.GRIDMIX_MEMORY_EMULATON) != null) { + isMemEmulOn = + simuJobConf.get(GridMixConfig.GRIDMIX_MEMORY_EMULATON). + contains(GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN); + } + + if (isMemEmulOn) { + for (int index = 0; index < zombieJob.getNumberMaps(); index ++) { + TaskInfo mapTask = zombieJob.getTaskInfo(TaskType.MAP, index); + if (mapTask.getResourceUsageMetrics().getHeapUsage() > 0) { + origJobMapsTHU += + mapTask.getResourceUsageMetrics().getHeapUsage(); + } + } + LOG.info("Original Job Maps Total Heap Usage: " + origJobMapsTHU); + + for (int index = 0; index < zombieJob.getNumberReduces(); index ++) { + TaskInfo reduceTask = zombieJob.getTaskInfo(TaskType.REDUCE, index); + if (reduceTask.getResourceUsageMetrics().getHeapUsage() > 0) { + origJobReducesTHU += + reduceTask.getResourceUsageMetrics().getHeapUsage(); + } + } + LOG.info("Original Job Reduces Total Heap Usage: " + origJobReducesTHU); + + simuJobMapsTHU = + getCounterValue(jhInfo.getMapCounters(), + TaskCounter.COMMITTED_HEAP_BYTES.toString()); + LOG.info("Simulated Job Maps Total Heap Usage: " + simuJobMapsTHU); + + simuJobReducesTHU = + getCounterValue(jhInfo.getReduceCounters(), + TaskCounter.COMMITTED_HEAP_BYTES.toString()); + LOG.info("Simulated Jobs Reduces Total Heap Usage: " + simuJobReducesTHU); + + long mapCount = jhInfo.getTotalMaps(); + long reduceCount = jhInfo.getTotalReduces(); + + String strHeapRatio = + simuJobConf.get(GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO); + if (strHeapRatio == null) { + strHeapRatio = "0.3F"; + } + + if (mapCount > 0) { + double mapEmulFactor = (simuJobMapsTHU * 100) / origJobMapsTHU; + long mapEmulAccuracy = Math.round(mapEmulFactor); + LOG.info("Maps memory emulation accuracy of a job:" + + mapEmulAccuracy + "%"); + Assert.assertTrue("Map phase total memory emulation had crossed the " + + "configured max limit.", mapEmulAccuracy + <= GridMixConfig.GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT); + Assert.assertTrue("Map phase total memory emulation had not crossed " + + "the configured min limit.", mapEmulAccuracy + >= GridMixConfig.GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT); + double expHeapRatio = Double.parseDouble(strHeapRatio); + LOG.info("expHeapRatio for maps:" + expHeapRatio); + double actHeapRatio = + ((double)Math.abs(origJobMapsTHU - simuJobMapsTHU)) ; + actHeapRatio /= origJobMapsTHU; + LOG.info("actHeapRatio for maps:" + actHeapRatio); + Assert.assertTrue("Simulate job maps heap ratio not matched.", + actHeapRatio <= expHeapRatio); + } + + if (reduceCount >0) { + double reduceEmulFactor = (simuJobReducesTHU * 100) / origJobReducesTHU; + long reduceEmulAccuracy = Math.round(reduceEmulFactor); + LOG.info("Reduces memory emulation accuracy of a job:" + + reduceEmulAccuracy + "%"); + Assert.assertTrue("Reduce phase total memory emulation had crossed " + + "configured max limit.", reduceEmulAccuracy + <= GridMixConfig.GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT); + Assert.assertTrue("Reduce phase total memory emulation had not " + + "crosssed configured min limit.", reduceEmulAccuracy + >= GridMixConfig.GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT); + double expHeapRatio = Double.parseDouble(strHeapRatio); + LOG.info("expHeapRatio for reduces:" + expHeapRatio); + double actHeapRatio = + ((double)Math.abs(origJobReducesTHU - simuJobReducesTHU)); + actHeapRatio /= origJobReducesTHU; + LOG.info("actHeapRatio for reduces:" + actHeapRatio); + Assert.assertTrue("Simulate job reduces heap ratio not matched.", + actHeapRatio <= expHeapRatio); + } + } + } + /** * It verifies the cpu resource usage of a gridmix job against * their original job. @@ -367,7 +471,14 @@ public class GridmixJobVerification { JobHistoryParser.JobInfo simuJobHistoryInfo, JobConf simuJobConf) throws Exception { + boolean isCpuEmulOn = false; if (simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON) != null) { + isCpuEmulOn = + simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON). + contains(GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN); + } + + if (isCpuEmulOn) { Map origJobMetrics = getOriginalJobCPUMetrics(origJobHistory); Map simuJobMetrics = diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case1.json.gz b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case1.json.gz new file mode 100644 index 00000000000..5f7fcab1d55 Binary files /dev/null and b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case1.json.gz differ diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case2.json.gz b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case2.json.gz new file mode 100644 index 00000000000..d0ea21e6b4d Binary files /dev/null and b/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case2.json.gz differ