From 522bc98026c6a4a96bbf5246d9d0fbed23401eb6 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Thu, 22 Dec 2016 14:32:24 +0900 Subject: [PATCH] YARN-4994. Use MiniYARNCluster with try-with-resources in tests. Contributed by Andras Bokor. (cherry picked from commit ae401539eaf7745ec8690f9281726fb4cdcdbe94) --- .../TestJobHistoryEventHandler.java | 10 +-- .../hadoop/tools/TestHadoopArchiveLogs.java | 12 +-- .../tools/TestHadoopArchiveLogsRunner.java | 11 +-- ...HedgingRequestRMFailoverProxyProvider.java | 57 ++++++------- .../yarn/client/api/impl/TestAMRMProxy.java | 37 +++------ .../hadoop/yarn/client/cli/TestYarnCLI.java | 15 +--- .../yarn/server/TestMiniYarnCluster.java | 80 +++++++++---------- 7 files changed, 85 insertions(+), 137 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index d1a25b054dd..692bd7095b7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -559,11 +559,9 @@ public void testTimelineEventHandling() throws Exception { TestParams t = new TestParams(false); Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); - MiniYARNCluster yarnCluster = null; long currentTime = System.currentTimeMillis(); - try { - yarnCluster = new MiniYARNCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1); + try (MiniYARNCluster yarnCluster = new MiniYARNCluster( + TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1)) { yarnCluster.init(conf); yarnCluster.start(); Configuration confJHEH = new YarnConfiguration(conf); @@ -716,10 +714,6 @@ public void testTimelineEventHandling() throws Exception { tEntity.getEvents().get(0).getEventInfo().get("TASK_TYPE")); Assert.assertEquals(TaskType.MAP.toString(), tEntity.getEvents().get(1).getEventInfo().get("TASK_TYPE")); - } finally { - if (yarnCluster != null) { - yarnCluster.stop(); - } } } diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java index 7fcb0bfad80..3a605f72395 100644 --- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java +++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java @@ -165,13 +165,11 @@ public void testCheckMaxEligible() throws Exception { @Test(timeout = 30000) public void testFilterAppsByAggregatedStatus() throws Exception { - MiniYARNCluster yarnCluster = null; - try { + try (MiniYARNCluster yarnCluster = + new MiniYARNCluster(TestHadoopArchiveLogs.class.getSimpleName(), + 1, 1, 1, 1)) { Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); - yarnCluster = - new MiniYARNCluster(TestHadoopArchiveLogs.class.getSimpleName(), 1, - 1, 1, 1); yarnCluster.init(conf); yarnCluster.start(); conf = yarnCluster.getConfig(); @@ -237,10 +235,6 @@ public void testFilterAppsByAggregatedStatus() throws Exception { Assert.assertTrue(hal.eligibleApplications.contains(app4)); Assert.assertTrue(hal.eligibleApplications.contains(app7)); Assert.assertTrue(hal.eligibleApplications.contains(app8)); - } finally { - if (yarnCluster != null) { - yarnCluster.stop(); - } } } diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java index 098e2fd367d..fad9b974ac9 100644 --- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java +++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogsRunner.java @@ -52,16 +52,14 @@ public class TestHadoopArchiveLogsRunner { @Test(timeout = 50000) public void testHadoopArchiveLogs() throws Exception { - MiniYARNCluster yarnCluster = null; MiniDFSCluster dfsCluster = null; FileSystem fs = null; - try { + try (MiniYARNCluster yarnCluster = + new MiniYARNCluster(TestHadoopArchiveLogsRunner.class.getSimpleName(), + 1, 2, 1, 1)) { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); - yarnCluster = - new MiniYARNCluster(TestHadoopArchiveLogsRunner.class.getSimpleName(), - 1, 2, 1, 1); yarnCluster.init(conf); yarnCluster.start(); conf = yarnCluster.getConfig(); @@ -133,9 +131,6 @@ public int compare(FileStatus o1, FileStatus o2) { harLogs[2].getOwner()); Assert.assertEquals(0, fs.listStatus(workingDir).length); } finally { - if (yarnCluster != null) { - yarnCluster.stop(); - } if (fs != null) { fs.close(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java index 30b409ee10c..b55cad8e686 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java @@ -35,8 +35,6 @@ public class TestHedgingRequestRMFailoverProxyProvider { @Test public void testHedgingRequestProxyProvider() throws Exception { - final MiniYARNCluster cluster = - new MiniYARNCluster("testHedgingRequestProxyProvider", 5, 0, 1, 1); Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); @@ -49,41 +47,44 @@ public void testHedgingRequestProxyProvider() throws Exception { conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 2000); - HATestUtil.setRpcAddressForRM("rm1", 10000, conf); - HATestUtil.setRpcAddressForRM("rm2", 20000, conf); - HATestUtil.setRpcAddressForRM("rm3", 30000, conf); - HATestUtil.setRpcAddressForRM("rm4", 40000, conf); - HATestUtil.setRpcAddressForRM("rm5", 50000, conf); - conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); + try (MiniYARNCluster cluster = + new MiniYARNCluster("testHedgingRequestProxyProvider", 5, 0, 1, 1)) { - cluster.init(conf); - cluster.start(); + HATestUtil.setRpcAddressForRM("rm1", 10000, conf); + HATestUtil.setRpcAddressForRM("rm2", 20000, conf); + HATestUtil.setRpcAddressForRM("rm3", 30000, conf); + HATestUtil.setRpcAddressForRM("rm4", 40000, conf); + HATestUtil.setRpcAddressForRM("rm5", 50000, conf); + conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); - final YarnClient client = YarnClient.createYarnClient(); - client.init(conf); - client.start(); + cluster.init(conf); + cluster.start(); - // Transition rm5 to active; - long start = System.currentTimeMillis(); - makeRMActive(cluster, 4); + final YarnClient client = YarnClient.createYarnClient(); + client.init(conf); + client.start(); - validateActiveRM(client); + // Transition rm5 to active; + long start = System.currentTimeMillis(); + makeRMActive(cluster, 4); - long end = System.currentTimeMillis(); - System.out.println("Client call succeeded at " + end); - // should return the response fast - Assert.assertTrue(end - start <= 10000); + validateActiveRM(client); - // transition rm5 to standby - cluster.getResourceManager(4).getRMContext().getRMAdminService() - .transitionToStandby(new HAServiceProtocol.StateChangeRequestInfo( - HAServiceProtocol.RequestSource.REQUEST_BY_USER)); + long end = System.currentTimeMillis(); + System.out.println("Client call succeeded at " + end); + // should return the response fast + Assert.assertTrue(end - start <= 10000); - makeRMActive(cluster, 2); + // transition rm5 to standby + cluster.getResourceManager(4).getRMContext().getRMAdminService() + .transitionToStandby(new HAServiceProtocol.StateChangeRequestInfo( + HAServiceProtocol.RequestSource.REQUEST_BY_USER)); - validateActiveRM(client); + makeRMActive(cluster, 2); - cluster.stop(); + validateActiveRM(client); + + } } private void validateActiveRM(YarnClient client) throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java index 33f752796b4..9eef9a076b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java @@ -60,11 +60,11 @@ public class TestAMRMProxy extends BaseAMRMProxyE2ETest { */ @Test(timeout = 120000) public void testAMRMProxyE2E() throws Exception { - MiniYARNCluster cluster = new MiniYARNCluster("testAMRMProxyE2E", 1, 1, 1); - YarnClient rmClient = null; ApplicationMasterProtocol client; - try { + try (MiniYARNCluster cluster = new MiniYARNCluster("testAMRMProxyE2E", + 1, 1, 1); + YarnClient rmClient = YarnClient.createYarnClient()) { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true); cluster.init(conf); @@ -75,7 +75,6 @@ public void testAMRMProxyE2E() throws Exception { yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS); - rmClient = YarnClient.createYarnClient(); rmClient.init(yarnConf); rmClient.start(); @@ -135,11 +134,6 @@ public void testAMRMProxyE2E() throws Exception { Thread.sleep(500); Assert.assertNotEquals(RMAppState.FINISHED, rmApp.getState()); - } finally { - if (rmClient != null) { - rmClient.stop(); - } - cluster.stop(); } } @@ -150,12 +144,11 @@ public void testAMRMProxyE2E() throws Exception { */ @Test(timeout = 120000) public void testE2ETokenRenewal() throws Exception { - MiniYARNCluster cluster = - new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1); - YarnClient rmClient = null; ApplicationMasterProtocol client; - try { + try (MiniYARNCluster cluster = + new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1); + YarnClient rmClient = YarnClient.createYarnClient()) { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true); conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1500); @@ -170,7 +163,6 @@ public void testE2ETokenRenewal() throws Exception { final Configuration yarnConf = cluster.getConfig(); yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS); - rmClient = YarnClient.createYarnClient(); rmClient.init(yarnConf); rmClient.start(); @@ -216,11 +208,6 @@ public void testE2ETokenRenewal() throws Exception { client.finishApplicationMaster(FinishApplicationMasterRequest .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null)); - } finally { - if (rmClient != null) { - rmClient.stop(); - } - cluster.stop(); } } @@ -230,11 +217,11 @@ public void testE2ETokenRenewal() throws Exception { */ @Test(timeout = 120000) public void testE2ETokenSwap() throws Exception { - MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenSwap", 1, 1, 1); - YarnClient rmClient = null; ApplicationMasterProtocol client; - try { + try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenSwap", + 1, 1, 1); + YarnClient rmClient = YarnClient.createYarnClient()) { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true); cluster.init(conf); @@ -242,7 +229,6 @@ public void testE2ETokenSwap() throws Exception { // the client will connect to the RM with the token provided by AMRMProxy final Configuration yarnConf = cluster.getConfig(); - rmClient = YarnClient.createYarnClient(); rmClient.init(yarnConf); rmClient.start(); @@ -260,11 +246,6 @@ public void testE2ETokenSwap() throws Exception { e.getMessage().startsWith("Invalid AMRMToken from appattempt_")); } - } finally { - if (rmClient != null) { - rmClient.stop(); - } - cluster.stop(); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 15ec6b50107..3ec1730f938 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -1721,15 +1721,13 @@ public void testGetQueueInfoPreemptionDisabled() throws Exception { + "ProportionalCapacityPreemptionPolicy"); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); conf.setBoolean(PREFIX + "root.a.a1.disable_preemption", true); - MiniYARNCluster cluster = - new MiniYARNCluster("testReservationAPIs", 2, 1, 1); - YarnClient yarnClient = null; - try { + try (MiniYARNCluster cluster = + new MiniYARNCluster("testReservationAPIs", 2, 1, 1); + YarnClient yarnClient = YarnClient.createYarnClient()) { cluster.init(conf); cluster.start(); final Configuration yarnConf = cluster.getConfig(); - yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConf); yarnClient.start(); @@ -1742,13 +1740,6 @@ public void testGetQueueInfoPreemptionDisabled() throws Exception { assertEquals(0, result); Assert.assertTrue(sysOutStream.toString() .contains("Preemption : disabled")); - } finally { - // clean-up - if (yarnClient != null) { - yarnClient.stop(); - } - cluster.stop(); - cluster.close(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java index e67a236d1a5..81b44427d71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; public class TestMiniYarnCluster { @@ -41,10 +42,11 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { */ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); enableAHS = false; - MiniYARNCluster cluster = null; - try { - cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), - numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + try (MiniYARNCluster cluster = + new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, + enableAHS)) { + cluster.init(conf); cluster.start(); @@ -52,11 +54,6 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { Assert.assertNull("Timeline Service should not have been started", cluster.getApplicationHistoryServer()); } - finally { - if(cluster != null) { - cluster.stop(); - } - } /* * Timeline service should start if TIMELINE_SERVICE_ENABLED == true @@ -64,10 +61,10 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { */ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); enableAHS = false; - cluster = null; - try { - cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), - numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + try (MiniYARNCluster cluster = + new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, + enableAHS)) { cluster.init(conf); // Verify that the timeline-service starts on ephemeral ports by default @@ -89,21 +86,16 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { Assert.assertNotNull("Timeline Service should have been started", cluster.getApplicationHistoryServer()); } - finally { - if(cluster != null) { - cluster.stop(); - } - } /* * Timeline service should start if TIMELINE_SERVICE_ENABLED == false * and enableAHS == true */ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); enableAHS = true; - cluster = null; - try { - cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), - numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + try (MiniYARNCluster cluster = + new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, + enableAHS)) { cluster.init(conf); cluster.start(); @@ -117,15 +109,10 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { Assert.assertNotNull("Timeline Service should have been started", cluster.getApplicationHistoryServer()); } - finally { - if(cluster != null) { - cluster.stop(); - } - } } @Test - public void testMultiRMConf() { + public void testMultiRMConf() throws IOException { String RM1_NODE_ID = "rm1", RM2_NODE_ID = "rm2"; int RM1_PORT_BASE = 10000, RM2_PORT_BASE = 20000; Configuration conf = new YarnConfiguration(); @@ -139,23 +126,28 @@ public void testMultiRMConf() { conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true); - MiniYARNCluster cluster = + try (MiniYARNCluster cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getName(), - 2, 0, 1, 1); - cluster.init(conf); - Configuration conf1 = cluster.getResourceManager(0).getConfig(), - conf2 = cluster.getResourceManager(1).getConfig(); - Assert.assertFalse(conf1 == conf2); - Assert.assertEquals("0.0.0.0:18032", - conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID))); - Assert.assertEquals("0.0.0.0:28032", - conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID))); - Assert.assertEquals("rm1", conf1.get(YarnConfiguration.RM_HA_ID)); + 2, 0, 1, 1)) { + cluster.init(conf); + Configuration conf1 = cluster.getResourceManager(0).getConfig(), + conf2 = cluster.getResourceManager(1).getConfig(); + Assert.assertFalse(conf1 == conf2); + Assert.assertEquals("0.0.0.0:18032", + conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, + RM1_NODE_ID))); + Assert.assertEquals("0.0.0.0:28032", + conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, + RM2_NODE_ID))); + Assert.assertEquals("rm1", conf1.get(YarnConfiguration.RM_HA_ID)); - Assert.assertEquals("0.0.0.0:18032", - conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID))); - Assert.assertEquals("0.0.0.0:28032", - conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID))); - Assert.assertEquals("rm2", conf2.get(YarnConfiguration.RM_HA_ID)); + Assert.assertEquals("0.0.0.0:18032", + conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, + RM1_NODE_ID))); + Assert.assertEquals("0.0.0.0:28032", + conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, + RM2_NODE_ID))); + Assert.assertEquals("rm2", conf2.get(YarnConfiguration.RM_HA_ID)); + } } }