Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-2890. MiniYARNCluster should start the timeline server based on the configuration. Contributed by Mit Desai.

Browse Source 
(cherry picked from commit 51af8d367de94689770f57c64bea3b244d7755f6)
(cherry picked from commit d21ef79707a0f32939d9a5af4fed2d9f5fe6f2ec)
(cherry picked from commit db325c053c6ef3ee8731d7273d1d92da7e5deee7)
This commit is contained in:
Zhijie Shen 2014-12-01 19:04:04 -08:00 committed by Vinod Kumar Vavilapalli
parent 2f0a34b5ba
commit 8a47d1aa55
7 changed files with 67 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
View File

@ -453,7 +453,7 @@ public void testTimelineEventHandling() throws Exception {
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
jheh.start();

58
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
View File

@ -34,6 +34,58 @@
public class TestMRTimelineEventHandling {
@Test
public void testTimelineServiceStartInMiniCluster() throws Exception {
Configuration conf = new YarnConfiguration();
/*
* Timeline service should not start if the config is set to false
* Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
*/
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
//verify that the timeline service is not started.
Assert.assertNull("Timeline Service should not have been started",
cluster.getApplicationHistoryServer());
//Run a MR job and verify it succeeds
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job =
UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
Assert.assertEquals(JobStatus.SUCCEEDED,
job.getJobStatus().getState().getValue());
}
finally {
if(cluster != null) {
cluster.stop();
}
}
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
cluster = null;
try {
cluster = new MiniMRYarnCluster(
TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
Assert.assertNull("Timeline Service should not have been started",
cluster.getApplicationHistoryServer());
}
finally {
if(cluster != null) {
cluster.stop();
}
}
}
@Test
public void testMRTimelineEventHandling() throws Exception {
Configuration conf = new YarnConfiguration();
@ -42,7 +94,7 @@ public void testMRTimelineEventHandling() throws Exception {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@ -96,7 +148,7 @@ public void testMapreduceJobTimelineServiceEnabled()
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@ -133,7 +185,7 @@ public void testMapreduceJobTimelineServiceEnabled()
cluster = null;
try {
cluster = new MiniMRYarnCluster(
TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
View File

@ -72,11 +72,7 @@ public MiniMRYarnCluster(String testName) {
}
public MiniMRYarnCluster(String testName, int noOfNMs) {
this(testName, noOfNMs, false);
}
public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) {
super(testName, 1, noOfNMs, 4, 4, enableAHS);
super(testName, 1, noOfNMs, 4, 4);
historyServerWrapper = new JobHistoryServerWrapper();
addService(historyServerWrapper);
}

3
hadoop-yarn-project/CHANGES.txt
View File

@ -30,6 +30,9 @@ Release 2.6.1 - UNRELEASED
YARN-2905. AggregatedLogsBlock page can infinitely loop if the aggregated
log file is corrupted (Varun Saxena via jlowe)
YARN-2890. MiniYARNCluster should start the timeline server based on the
configuration. (Mit Desai via zjshen)
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
View File

@ -84,7 +84,7 @@ protected void setupInternal(int numNodeManager) throws Exception {
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
numNodeManager, 1, 1, true);
numNodeManager, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
View File

@ -295,7 +295,7 @@ protected void startHACluster(int numOfNMs, boolean overrideClientRMService,
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster =
new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS,
numOfNMs, 1, 1, overrideClientRMService, overrideRTS,
overrideApplicationMasterService);
cluster.resetStartFailoverFlag(false);
cluster.init(conf);
@ -326,10 +326,10 @@ public class MiniYARNClusterForHATesting extends MiniYARNCluster {
public MiniYARNClusterForHATesting(String testName,
int numResourceManagers, int numNodeManagers, int numLocalDirs,
int numLogDirs, boolean enableAHS, boolean overrideClientRMService,
int numLogDirs, boolean overrideClientRMService,
boolean overrideRTS, boolean overrideApplicationMasterService) {
super(testName, numResourceManagers, numNodeManagers, numLocalDirs,
numLogDirs, enableAHS);
numLogDirs);
this.overrideClientRMService = overrideClientRMService;
this.overrideRTS = overrideRTS;
this.overrideApplicationMasterService = overrideApplicationMasterService;

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
View File

@ -57,7 +57,6 @@
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@ -120,7 +119,6 @@ public class MiniYARNCluster extends CompositeService {
private int numLocalDirs;
// Number of nm-log-dirs per nodemanager
private int numLogDirs;
private boolean enableAHS;
/**
* @param testName name of the test
@ -128,15 +126,13 @@ public class MiniYARNCluster extends CompositeService {
* @param numNodeManagers the number of node managers in the cluster
* @param numLocalDirs the number of nm-local-dirs per nodemanager
* @param numLogDirs the number of nm-log-dirs per nodemanager
* @param enableAHS enable ApplicationHistoryServer or not
*/
public MiniYARNCluster(
String testName, int numResourceManagers, int numNodeManagers,
int numLocalDirs, int numLogDirs, boolean enableAHS) {
int numLocalDirs, int numLogDirs) {
super(testName.replace("$", ""));
this.numLocalDirs = numLocalDirs;
this.numLogDirs = numLogDirs;
this.enableAHS = enableAHS;
String testSubDir = testName.replace("$", "");
File targetWorkDir = new File("target", testSubDir);
try {
@ -186,20 +182,6 @@ public MiniYARNCluster(
nodeManagers = new NodeManager[numNodeManagers];
}
/**
* @param testName name of the test
* @param numResourceManagers the number of resource managers in the cluster
* @param numNodeManagers the number of node managers in the cluster
* @param numLocalDirs the number of nm-local-dirs per nodemanager
* @param numLogDirs the number of nm-log-dirs per nodemanager
*/
public MiniYARNCluster(
String testName, int numResourceManagers, int numNodeManagers,
int numLocalDirs, int numLogDirs) {
this(testName, numResourceManagers, numNodeManagers, numLocalDirs,
numLogDirs, false);
}
/**
* @param testName name of the test
* @param numNodeManagers the number of node managers in the cluster
@ -260,8 +242,8 @@ public void serviceInit(Configuration conf) throws Exception {
addService(new NodeManagerWrapper(index));
}
if (enableAHS) {
addService(new ApplicationHistoryServerWrapper());
if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
addService(new ApplicationHistoryServerWrapper());
}
super.serviceInit(