Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4994. Use MiniYARNCluster with try-with-resources in tests. Contributed by Andras Bokor. 

(cherry picked from commit ae401539ea)
(cherry picked from commit 522bc98026)

 Conflicts:
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
This commit is contained in:
Akira Ajisaka 2016-12-22 14:32:24 +09:00
parent 990e272f96
commit 8e4479dda3
6 changed files with 56 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
View File

@ -559,11 +559,9 @@ public class TestJobHistoryEventHandler {
TestParams t = new TestParams(false);
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
MiniYARNCluster yarnCluster = null;
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
try (MiniYARNCluster yarnCluster = new MiniYARNCluster(
TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1)) {
yarnCluster.init(conf);
yarnCluster.start();
Configuration confJHEH = new YarnConfiguration(conf);
@ -716,10 +714,6 @@ public class TestJobHistoryEventHandler {
tEntity.getEvents().get(0).getEventInfo().get("TASK_TYPE"));
Assert.assertEquals(TaskType.MAP.toString(),
tEntity.getEvents().get(1).getEventInfo().get("TASK_TYPE"));
} finally {
if (yarnCluster != null) {
yarnCluster.stop();
}
}
}

12
hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
View File

@ -165,13 +165,11 @@ public class TestHadoopArchiveLogs {
@Test(timeout = 30000)
public void testFilterAppsByAggregatedStatus() throws Exception {
MiniYARNCluster yarnCluster = null;
try {
try (MiniYARNCluster yarnCluster =
new MiniYARNCluster(TestHadoopArchiveLogs.class.getSimpleName(),
1, 1, 1, 1)) {
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
yarnCluster =
new MiniYARNCluster(TestHadoopArchiveLogs.class.getSimpleName(), 1,
1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
conf = yarnCluster.getConfig();
@ -237,10 +235,6 @@ public class TestHadoopArchiveLogs {
Assert.assertTrue(hal.eligibleApplications.contains(app4));
Assert.assertTrue(hal.eligibleApplications.contains(app7));
Assert.assertTrue(hal.eligibleApplications.contains(app8));
} finally {
if (yarnCluster != null) {
yarnCluster.stop();
}
}
}

11
hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java
View File

@ -52,16 +52,14 @@ public class TestHadoopArchiveLogsRunner {
@Test(timeout = 50000)
public void testHadoopArchiveLogs() throws Exception {
MiniYARNCluster yarnCluster = null;
MiniDFSCluster dfsCluster = null;
FileSystem fs = null;
try {
try (MiniYARNCluster yarnCluster =
new MiniYARNCluster(TestHadoopArchiveLogsRunner.class.getSimpleName(),
1, 2, 1, 1)) {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
yarnCluster =
new MiniYARNCluster(TestHadoopArchiveLogsRunner.class.getSimpleName(),
1, 2, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
conf = yarnCluster.getConfig();
@ -133,9 +131,6 @@ public class TestHadoopArchiveLogsRunner {
harLogs[2].getOwner());
Assert.assertEquals(0, fs.listStatus(workingDir).length);
} finally {
if (yarnCluster != null) {
yarnCluster.stop();
}
if (fs != null) {
fs.close();
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
View File

@ -86,11 +86,11 @@ public class TestAMRMProxy {
*/
@Test(timeout = 60000)
public void testAMRMProxyE2E() throws Exception {
MiniYARNCluster cluster = new MiniYARNCluster("testAMRMProxyE2E", 1, 1, 1);
YarnClient rmClient = null;
ApplicationMasterProtocol client;
try {
try (MiniYARNCluster cluster = new MiniYARNCluster("testAMRMProxyE2E",
1, 1, 1);
YarnClient rmClient = YarnClient.createYarnClient()) {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
cluster.init(conf);
@ -101,7 +101,6 @@ public class TestAMRMProxy {
yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
rmClient = YarnClient.createYarnClient();
rmClient.init(yarnConf);
rmClient.start();
@ -160,11 +159,6 @@ public class TestAMRMProxy {
Thread.sleep(500);
Assert.assertNotEquals(RMAppState.FINISHED, rmApp.getState());
} finally {
if (rmClient != null) {
rmClient.stop();
}
cluster.stop();
}
}
@ -175,12 +169,11 @@ public class TestAMRMProxy {
*/
@Test(timeout = 60000)
public void testE2ETokenRenewal() throws Exception {
MiniYARNCluster cluster =
new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1);
YarnClient rmClient = null;
ApplicationMasterProtocol client;
try {
try (MiniYARNCluster cluster =
new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1);
YarnClient rmClient = YarnClient.createYarnClient()) {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1500);
@ -195,7 +188,6 @@ public class TestAMRMProxy {
final Configuration yarnConf = cluster.getConfig();
yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
rmClient = YarnClient.createYarnClient();
rmClient.init(yarnConf);
rmClient.start();
@ -240,11 +232,6 @@ public class TestAMRMProxy {
client.finishApplicationMaster(FinishApplicationMasterRequest
.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
} finally {
if (rmClient != null) {
rmClient.stop();
}
cluster.stop();
}
}
@ -254,11 +241,11 @@ public class TestAMRMProxy {
*/
@Test(timeout = 60000)
public void testE2ETokenSwap() throws Exception {
MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenSwap", 1, 1, 1);
YarnClient rmClient = null;
ApplicationMasterProtocol client;
try {
try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenSwap",
1, 1, 1);
YarnClient rmClient = YarnClient.createYarnClient()) {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
cluster.init(conf);
@ -266,7 +253,6 @@ public class TestAMRMProxy {
// the client will connect to the RM with the token provided by AMRMProxy
final Configuration yarnConf = cluster.getConfig();
rmClient = YarnClient.createYarnClient();
rmClient.init(yarnConf);
rmClient.start();
@ -283,11 +269,6 @@ public class TestAMRMProxy {
e.getMessage().startsWith("Invalid AMRMToken from appattempt_"));
}
} finally {
if (rmClient != null) {
rmClient.stop();
}
cluster.stop();
}
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -1542,15 +1542,13 @@ public class TestYarnCLI {
conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
conf.setBoolean(
"yarn.scheduler.capacity.root.a.a1.disable_preemption", true);
MiniYARNCluster cluster =
new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
YarnClient yarnClient = null;
try {
try (MiniYARNCluster cluster =
new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
YarnClient yarnClient = YarnClient.createYarnClient()) {
cluster.init(conf);
cluster.start();
final Configuration yarnConf = cluster.getConfig();
yarnClient = YarnClient.createYarnClient();
yarnClient.init(yarnConf);
yarnClient.start();
@ -1563,13 +1561,6 @@ public class TestYarnCLI {
assertEquals(0, result);
Assert.assertTrue(sysOutStream.toString()
.contains("Preemption : disabled"));
} finally {
// clean-up
if (yarnClient != null) {
yarnClient.stop();
}
cluster.stop();
cluster.close();
}
}

80
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
View File

@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
public class TestMiniYarnCluster {
@ -41,10 +42,11 @@ public class TestMiniYarnCluster {
*/
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
enableAHS = false;
MiniYARNCluster cluster = null;
try {
cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
try (MiniYARNCluster cluster =
new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
numNodeManagers, numLocalDirs, numLogDirs, numLogDirs,
enableAHS)) {
cluster.init(conf);
cluster.start();
@ -52,11 +54,6 @@ public class TestMiniYarnCluster {
Assert.assertNull("Timeline Service should not have been started",
cluster.getApplicationHistoryServer());
}
finally {
if(cluster != null) {
cluster.stop();
}
}
/*
* Timeline service should start if TIMELINE_SERVICE_ENABLED == true
@ -64,10 +61,10 @@ public class TestMiniYarnCluster {
*/
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
enableAHS = false;
cluster = null;
try {
cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
try (MiniYARNCluster cluster =
new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
numNodeManagers, numLocalDirs, numLogDirs, numLogDirs,
enableAHS)) {
cluster.init(conf);
// Verify that the timeline-service starts on ephemeral ports by default
@ -89,21 +86,16 @@ public class TestMiniYarnCluster {
Assert.assertNotNull("Timeline Service should have been started",
cluster.getApplicationHistoryServer());
}
finally {
if(cluster != null) {
cluster.stop();
}
}
/*
* Timeline service should start if TIMELINE_SERVICE_ENABLED == false
* and enableAHS == true
*/
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
enableAHS = true;
cluster = null;
try {
cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
try (MiniYARNCluster cluster =
new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
numNodeManagers, numLocalDirs, numLogDirs, numLogDirs,
enableAHS)) {
cluster.init(conf);
cluster.start();
@ -117,15 +109,10 @@ public class TestMiniYarnCluster {
Assert.assertNotNull("Timeline Service should have been started",
cluster.getApplicationHistoryServer());
}
finally {
if(cluster != null) {
cluster.stop();
}
}
}
@Test
public void testMultiRMConf() {
public void testMultiRMConf() throws IOException {
String RM1_NODE_ID = "rm1", RM2_NODE_ID = "rm2";
int RM1_PORT_BASE = 10000, RM2_PORT_BASE = 20000;
Configuration conf = new YarnConfiguration();
@ -139,23 +126,28 @@ public class TestMiniYarnCluster {
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
MiniYARNCluster cluster =
try (MiniYARNCluster cluster =
new MiniYARNCluster(TestMiniYarnCluster.class.getName(),
2, 0, 1, 1);
cluster.init(conf);
Configuration conf1 = cluster.getResourceManager(0).getConfig(),
conf2 = cluster.getResourceManager(1).getConfig();
Assert.assertFalse(conf1 == conf2);
Assert.assertEquals("0.0.0.0:18032",
conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID)));
Assert.assertEquals("0.0.0.0:28032",
conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID)));
Assert.assertEquals("rm1", conf1.get(YarnConfiguration.RM_HA_ID));
2, 0, 1, 1)) {
cluster.init(conf);
Configuration conf1 = cluster.getResourceManager(0).getConfig(),
conf2 = cluster.getResourceManager(1).getConfig();
Assert.assertFalse(conf1 == conf2);
Assert.assertEquals("0.0.0.0:18032",
conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS,
RM1_NODE_ID)));
Assert.assertEquals("0.0.0.0:28032",
conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS,
RM2_NODE_ID)));
Assert.assertEquals("rm1", conf1.get(YarnConfiguration.RM_HA_ID));
Assert.assertEquals("0.0.0.0:18032",
conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID)));
Assert.assertEquals("0.0.0.0:28032",
conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID)));
Assert.assertEquals("rm2", conf2.get(YarnConfiguration.RM_HA_ID));
Assert.assertEquals("0.0.0.0:18032",
conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS,
RM1_NODE_ID)));
Assert.assertEquals("0.0.0.0:28032",
conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS,
RM2_NODE_ID)));
Assert.assertEquals("rm2", conf2.get(YarnConfiguration.RM_HA_ID));
}
}
}