MAPREDUCE-3375. [Gridmix] Memory Emulation system tests. (Vinay Thota via amarrk)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1201999 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Amar Kamat
parent
29ef7ef0df
commit
c799c3b2e7
|
|
@ -7,6 +7,8 @@ Release 0.23.1 - Unreleased
|
|||
NEW FEATURES
|
||||
|
||||
IMPROVEMENTS
|
||||
MAPREDUCE-3375. [Gridmix] Memory Emulation system tests.
|
||||
(Vinay Thota via amarrk)
|
||||
|
||||
MAPREDUCE-2733. [Gridmix] Gridmix3 cpu emulation system tests.
|
||||
(Vinay Thota via amarrk)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.mapred.gridmix;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
|
||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||
import org.junit.Test;
|
||||
import org.junit.Assert;
|
||||
|
||||
/**
|
||||
* Test the {@link Gridmix} memory emulation feature for the jobs with
|
||||
* custom progress interval, different input data, submission policies
|
||||
* and user resolver modes. Verify the total heap usage of map and reduce
|
||||
* tasks of the jobs with corresponding original job in the trace.
|
||||
*/
|
||||
public class TestMemEmulForMapsAndReducesWithCustomIntrvl
|
||||
extends GridmixSystemTestCase {
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog("TestMemEmulForMapsAndReducesWithCustomIntrvl.class");
|
||||
/**
|
||||
* Generate compressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation with custom progress interval. The {@link Gridmix}
|
||||
* should use the following runtime parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
|
||||
* Verify maps and reduces total heap memory usage of {@link Gridmix} jobs
|
||||
* with corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForReducesWithCompressedInputCase7()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = 1024 * 7;
|
||||
String tracePath = getTraceFile("mem_emul_case2");
|
||||
Assert.assertNotNull("Trace file not found!", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
RoundRobinUserResolver.class.getName(),
|
||||
"STRESS",
|
||||
inputSizeInMB + "m",
|
||||
"file://" + UtilsForGridmix.getProxyUsersFile(conf),
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.3F",
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate uncompressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation with custom progress interval. The {@link Gridmix}
|
||||
* should use the following runtime parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
|
||||
* Verify maps and reduces total heap memory usage of {@link Gridmix} jobs
|
||||
* with corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForReducesWithUncompressedInputCase8()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = cSize * 300;
|
||||
String tracePath = getTraceFile("mem_emul_case2");
|
||||
Assert.assertNotNull("Trace file not found!", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
SubmitterUserResolver.class.getName(),
|
||||
"REPLAY",
|
||||
inputSizeInMB + "m",
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.2F",
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.mapred.gridmix;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
|
||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||
import org.junit.Test;
|
||||
import org.junit.Assert;
|
||||
|
||||
/**
|
||||
* Test the {@link Gridmix} memory emulation feature for gridmix jobs
|
||||
* with default progress interval, different input data, submission
|
||||
* policies and user resolver modes. Verify the total heap usage of
|
||||
* map and reduce tasks of the jobs with corresponding original
|
||||
* job in the trace.
|
||||
*/
|
||||
public class TestMemEmulForMapsAndReducesWithDefaultIntrvl
|
||||
extends GridmixSystemTestCase {
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog("TestMemEmulForMapsAndReducesWithDefaultIntrvl.class");
|
||||
|
||||
/**
|
||||
* Generate compressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation with default progress interval. The {@link Gridmix}
|
||||
* should use the following runtime parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
|
||||
* Verify maps and reduces total heap memory usage of {@link Gridmix} jobs
|
||||
* with corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForReducesWithCompressedInputCase5()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = 1024 * 7;
|
||||
String tracePath = getTraceFile("mem_emul_case2");
|
||||
Assert.assertNotNull("Trace file not found!", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
RoundRobinUserResolver.class.getName(),
|
||||
"STRESS",
|
||||
inputSizeInMB + "m",
|
||||
"file://" + UtilsForGridmix.getProxyUsersFile(conf),
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate uncompressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation with default progress interval. The {@link Gridmix}
|
||||
* should use the following runtime parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
|
||||
* Verify maps and reduces total heap memory usage of {@link Gridmix} jobs
|
||||
* with corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForReducesWithUncompressedInputCase6()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = cSize * 300;
|
||||
String tracePath = getTraceFile("mem_emul_case2");
|
||||
Assert.assertNotNull("Trace file not found!", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
SubmitterUserResolver.class.getName(),
|
||||
"REPLAY",
|
||||
inputSizeInMB + "m",
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.mapred.gridmix;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
|
||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||
import org.junit.Test;
|
||||
import org.junit.Assert;
|
||||
|
||||
/**
|
||||
* Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs
|
||||
* with default progress interval, custom heap memory ratio, different input
|
||||
* data, submission policies and user resolver modes. Verify the total heap
|
||||
* usage of map and reduce tasks of the jobs with corresponding the original job
|
||||
* in the trace.
|
||||
*/
|
||||
public class TestMemEmulForMapsWithCustomHeapMemoryRatio
|
||||
extends GridmixSystemTestCase {
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog("TestMemEmulForMapsWithCustomHeapMemoryRatio.class");
|
||||
|
||||
/**
|
||||
* Generate compressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation. The {@link Gridmix} should use the following runtime
|
||||
* parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
|
||||
* Verify total heap memory usage of the tasks of {@link Gridmix} jobs with
|
||||
* corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForMapsWithCompressedInputCase1()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = 1024 * 7;
|
||||
String tracePath = getTraceFile("mem_emul_case2");
|
||||
Assert.assertNotNull("Trace file has not found.", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
SubmitterUserResolver.class.getName(),
|
||||
"STRESS",
|
||||
inputSizeInMB + "m",
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
|
||||
"-D", GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO + "=0.5F"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate uncompressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation. The {@link Gridmix} should use the following runtime
|
||||
* parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
|
||||
* Verify total heap memory usage of tasks of {@link Gridmix} jobs with
|
||||
* corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForMapsWithUncompressedInputCase2()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = cSize * 300;
|
||||
String tracePath = getTraceFile("mem_emul_case2");
|
||||
Assert.assertNotNull("Trace file has not found.", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
RoundRobinUserResolver.class.getName(),
|
||||
"STRESS",
|
||||
inputSizeInMB + "m",
|
||||
"file://" + UtilsForGridmix.getProxyUsersFile(conf),
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
|
||||
"-D", GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO + "=0.4F"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.mapred.gridmix;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
|
||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||
import org.junit.Test;
|
||||
import org.junit.Assert;
|
||||
|
||||
/**
|
||||
* Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs
|
||||
* with custom progress interval, different input data, submission policies
|
||||
* and user resolver modes. Verify the total heap usage of map tasks of
|
||||
* the jobs with corresponding the original job in the trace.
|
||||
*/
|
||||
public class TestMemEmulForMapsWithCustomIntrvl extends GridmixSystemTestCase {
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog("TestMemEmulForMapsWithCustomIntrvl.class");
|
||||
|
||||
/**
|
||||
* Generate compressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation with custom progress interval. The {@link Gridmix}
|
||||
* should use the following runtime parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
|
||||
* Verify maps total heap memory usage of {@link Gridmix} jobs with
|
||||
* corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForMapsWithCompressedInputCase3()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = 1024 * 7;
|
||||
String tracePath = getTraceFile("mem_emul_case1");
|
||||
Assert.assertNotNull("Trace file not found!", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
SubmitterUserResolver.class.getName(),
|
||||
"STRESS",
|
||||
inputSizeInMB + "m",
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.2F",
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate uncompressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation with custom progress interval. The {@link Gridmix}
|
||||
* should use the following runtime parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
|
||||
* Verify maps total heap memory usage of {@link Gridmix} jobs with
|
||||
* corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForMapsWithUncompressedInputCase4()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = cSize * 300;
|
||||
String tracePath = getTraceFile("mem_emul_case1");
|
||||
Assert.assertNotNull("Trace file not found!", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
RoundRobinUserResolver.class.getName(),
|
||||
"STRESS",
|
||||
inputSizeInMB + "m",
|
||||
"file://" + UtilsForGridmix.getProxyUsersFile(conf),
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.3F",
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.mapred.gridmix;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
|
||||
import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
|
||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||
import org.junit.Test;
|
||||
import org.junit.Assert;
|
||||
|
||||
/**
|
||||
* Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs
|
||||
* with default progress interval, different input data, submission policies
|
||||
* and user resolver modes. Verify the total heap usage of map tasks of the
|
||||
* jobs with corresponding original job in the trace.
|
||||
*/
|
||||
public class TestMemEmulForMapsWithDefaultIntrvl extends GridmixSystemTestCase {
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog("TestMemEmulForMapsWithDefaultIntrvl.class");
|
||||
|
||||
/**
|
||||
* Generate compressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation with default progress interval. The {@link Gridmix}
|
||||
* should use the following runtime parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
|
||||
* Verify maps total heap memory usage of {@link Gridmix} jobs with
|
||||
* corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForMapsWithCompressedInputCase1()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = 1024 * 7;
|
||||
String tracePath = getTraceFile("mem_emul_case1");
|
||||
Assert.assertNotNull("Trace file not found!", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
SubmitterUserResolver.class.getName(),
|
||||
"STRESS",
|
||||
inputSizeInMB + "m",
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate uncompressed input and run {@link Gridmix} by turning on the
|
||||
* memory emulation with default progress interval. The {@link Gridmix}
|
||||
* should use the following runtime parameters while running the jobs.
|
||||
* Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
|
||||
* Verify maps total heap memory usage of {@link Gridmix} jobs with
|
||||
* corresponding original job in the trace.
|
||||
* @throws Exception - if an error occurs.
|
||||
*/
|
||||
@Test
|
||||
public void testMemoryEmulationForMapsWithUncompressedInputCase2()
|
||||
throws Exception {
|
||||
final long inputSizeInMB = cSize * 300;
|
||||
String tracePath = getTraceFile("mem_emul_case1");
|
||||
Assert.assertNotNull("Trace file not found!", tracePath);
|
||||
String [] runtimeValues =
|
||||
{ "LOADJOB",
|
||||
RoundRobinUserResolver.class.getName(),
|
||||
"STRESS",
|
||||
inputSizeInMB + "m",
|
||||
"file://" + UtilsForGridmix.getProxyUsersFile(conf),
|
||||
tracePath};
|
||||
|
||||
String [] otherArgs = {
|
||||
"-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
|
||||
GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
|
||||
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
|
||||
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
|
||||
"-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
|
||||
|
||||
runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
|
||||
GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
|
||||
}
|
||||
}
|
||||
|
|
@ -248,4 +248,38 @@ public class GridMixConfig {
|
|||
*/
|
||||
public static int GRIDMIX_CPU_EMULATION_UPPER_LIMIT = 130;
|
||||
|
||||
/**
|
||||
* Gridmix heap memory custom interval
|
||||
*/
|
||||
public static final String GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL =
|
||||
TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL;
|
||||
|
||||
/**
|
||||
* Gridmix heap free memory ratio
|
||||
*/
|
||||
public static final String GRIDMIX_HEAP_FREE_MEMORY_RATIO =
|
||||
TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO;
|
||||
|
||||
/**
|
||||
* Gridmix memory emulation plugin
|
||||
*/
|
||||
public static final String GRIDMIX_MEMORY_EMULATION_PLUGIN =
|
||||
TotalHeapUsageEmulatorPlugin.class.getName();
|
||||
|
||||
/**
|
||||
* Gridmix memory emulation
|
||||
*/
|
||||
public static final String GRIDMIX_MEMORY_EMULATON =
|
||||
ResourceUsageMatcher.RESOURCE_USAGE_EMULATION_PLUGINS;
|
||||
|
||||
/**
|
||||
* Gridmix memory emulation lower limit.
|
||||
*/
|
||||
public static int GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT = 55;
|
||||
|
||||
/**
|
||||
* Gridmix memory emulation upper limit.
|
||||
*/
|
||||
public static int GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT = 130;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ public class GridmixJobVerification {
|
|||
zombieJob.getJobConf());
|
||||
verifyHighRamMemoryJobs(zombieJob, simuJobConf);
|
||||
verifyCPUEmulationOfJobs(zombieJob, jhInfo, simuJobConf);
|
||||
verifyMemoryEmulationOfJobs(zombieJob, jhInfo, simuJobConf);
|
||||
LOG.info("Done.");
|
||||
}
|
||||
verifyDistributedCacheBetweenJobs(simuAndOrigJobsInfo);
|
||||
|
|
@ -356,6 +357,109 @@ public class GridmixJobVerification {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* It verifies the heap memory resource usage of gridmix jobs with
|
||||
* corresponding original job in the trace.
|
||||
* @param zombieJob - Original job history.
|
||||
* @param jhInfo - Simulated job history.
|
||||
* @param simuJobConf - simulated job configuration.
|
||||
*/
|
||||
public void verifyMemoryEmulationOfJobs(ZombieJob zombieJob,
|
||||
JobHistoryParser.JobInfo jhInfo,
|
||||
JobConf simuJobConf) throws Exception {
|
||||
long origJobMapsTHU = 0;
|
||||
long origJobReducesTHU = 0;
|
||||
long simuJobMapsTHU = 0;
|
||||
long simuJobReducesTHU = 0;
|
||||
boolean isMemEmulOn = false;
|
||||
if (simuJobConf.get(GridMixConfig.GRIDMIX_MEMORY_EMULATON) != null) {
|
||||
isMemEmulOn =
|
||||
simuJobConf.get(GridMixConfig.GRIDMIX_MEMORY_EMULATON).
|
||||
contains(GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN);
|
||||
}
|
||||
|
||||
if (isMemEmulOn) {
|
||||
for (int index = 0; index < zombieJob.getNumberMaps(); index ++) {
|
||||
TaskInfo mapTask = zombieJob.getTaskInfo(TaskType.MAP, index);
|
||||
if (mapTask.getResourceUsageMetrics().getHeapUsage() > 0) {
|
||||
origJobMapsTHU +=
|
||||
mapTask.getResourceUsageMetrics().getHeapUsage();
|
||||
}
|
||||
}
|
||||
LOG.info("Original Job Maps Total Heap Usage: " + origJobMapsTHU);
|
||||
|
||||
for (int index = 0; index < zombieJob.getNumberReduces(); index ++) {
|
||||
TaskInfo reduceTask = zombieJob.getTaskInfo(TaskType.REDUCE, index);
|
||||
if (reduceTask.getResourceUsageMetrics().getHeapUsage() > 0) {
|
||||
origJobReducesTHU +=
|
||||
reduceTask.getResourceUsageMetrics().getHeapUsage();
|
||||
}
|
||||
}
|
||||
LOG.info("Original Job Reduces Total Heap Usage: " + origJobReducesTHU);
|
||||
|
||||
simuJobMapsTHU =
|
||||
getCounterValue(jhInfo.getMapCounters(),
|
||||
TaskCounter.COMMITTED_HEAP_BYTES.toString());
|
||||
LOG.info("Simulated Job Maps Total Heap Usage: " + simuJobMapsTHU);
|
||||
|
||||
simuJobReducesTHU =
|
||||
getCounterValue(jhInfo.getReduceCounters(),
|
||||
TaskCounter.COMMITTED_HEAP_BYTES.toString());
|
||||
LOG.info("Simulated Jobs Reduces Total Heap Usage: " + simuJobReducesTHU);
|
||||
|
||||
long mapCount = jhInfo.getTotalMaps();
|
||||
long reduceCount = jhInfo.getTotalReduces();
|
||||
|
||||
String strHeapRatio =
|
||||
simuJobConf.get(GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO);
|
||||
if (strHeapRatio == null) {
|
||||
strHeapRatio = "0.3F";
|
||||
}
|
||||
|
||||
if (mapCount > 0) {
|
||||
double mapEmulFactor = (simuJobMapsTHU * 100) / origJobMapsTHU;
|
||||
long mapEmulAccuracy = Math.round(mapEmulFactor);
|
||||
LOG.info("Maps memory emulation accuracy of a job:"
|
||||
+ mapEmulAccuracy + "%");
|
||||
Assert.assertTrue("Map phase total memory emulation had crossed the "
|
||||
+ "configured max limit.", mapEmulAccuracy
|
||||
<= GridMixConfig.GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT);
|
||||
Assert.assertTrue("Map phase total memory emulation had not crossed "
|
||||
+ "the configured min limit.", mapEmulAccuracy
|
||||
>= GridMixConfig.GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT);
|
||||
double expHeapRatio = Double.parseDouble(strHeapRatio);
|
||||
LOG.info("expHeapRatio for maps:" + expHeapRatio);
|
||||
double actHeapRatio =
|
||||
((double)Math.abs(origJobMapsTHU - simuJobMapsTHU)) ;
|
||||
actHeapRatio /= origJobMapsTHU;
|
||||
LOG.info("actHeapRatio for maps:" + actHeapRatio);
|
||||
Assert.assertTrue("Simulate job maps heap ratio not matched.",
|
||||
actHeapRatio <= expHeapRatio);
|
||||
}
|
||||
|
||||
if (reduceCount >0) {
|
||||
double reduceEmulFactor = (simuJobReducesTHU * 100) / origJobReducesTHU;
|
||||
long reduceEmulAccuracy = Math.round(reduceEmulFactor);
|
||||
LOG.info("Reduces memory emulation accuracy of a job:"
|
||||
+ reduceEmulAccuracy + "%");
|
||||
Assert.assertTrue("Reduce phase total memory emulation had crossed "
|
||||
+ "configured max limit.", reduceEmulAccuracy
|
||||
<= GridMixConfig.GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT);
|
||||
Assert.assertTrue("Reduce phase total memory emulation had not "
|
||||
+ "crosssed configured min limit.", reduceEmulAccuracy
|
||||
>= GridMixConfig.GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT);
|
||||
double expHeapRatio = Double.parseDouble(strHeapRatio);
|
||||
LOG.info("expHeapRatio for reduces:" + expHeapRatio);
|
||||
double actHeapRatio =
|
||||
((double)Math.abs(origJobReducesTHU - simuJobReducesTHU));
|
||||
actHeapRatio /= origJobReducesTHU;
|
||||
LOG.info("actHeapRatio for reduces:" + actHeapRatio);
|
||||
Assert.assertTrue("Simulate job reduces heap ratio not matched.",
|
||||
actHeapRatio <= expHeapRatio);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* It verifies the cpu resource usage of a gridmix job against
|
||||
* their original job.
|
||||
|
|
@ -367,7 +471,14 @@ public class GridmixJobVerification {
|
|||
JobHistoryParser.JobInfo simuJobHistoryInfo,
|
||||
JobConf simuJobConf) throws Exception {
|
||||
|
||||
boolean isCpuEmulOn = false;
|
||||
if (simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON) != null) {
|
||||
isCpuEmulOn =
|
||||
simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON).
|
||||
contains(GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN);
|
||||
}
|
||||
|
||||
if (isCpuEmulOn) {
|
||||
Map<String,Long> origJobMetrics =
|
||||
getOriginalJobCPUMetrics(origJobHistory);
|
||||
Map<String,Long> simuJobMetrics =
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue