From c09bb46579631a6e3b8dd50e964e77c9357ea14b Mon Sep 17 00:00:00 2001 From: Jian He Date: Wed, 2 Sep 2015 17:45:23 -0700 Subject: [PATCH] YARN-4101. RM should print alert messages if Zookeeper and Resourcemanager gets connection issue. Contributed by Xuan Gong (cherry picked from commit 214fd1408c21f596d1d15217c11b58b34561aab7) --- .../java/org/apache/hadoop/ha/ActiveStandbyElector.java | 4 ++++ hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/server/resourcemanager/AdminService.java | 9 +++++++++ .../server/resourcemanager/EmbeddedElectorService.java | 4 ++++ .../yarn/server/resourcemanager/webapp/AboutBlock.java | 2 ++ .../yarn/server/resourcemanager/webapp/RMWebApp.java | 5 +++++ .../server/resourcemanager/webapp/RMWebAppFilter.java | 5 ++++- .../server/resourcemanager/webapp/dao/ClusterInfo.java | 6 ++++++ .../server/resourcemanager/webapp/TestRMWebServices.java | 8 ++++++-- 9 files changed, 43 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java index 123f119f709..f8f720e6857 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java @@ -1102,4 +1102,8 @@ public class ActiveStandbyElector implements StatCallback, StringCallback { ((appData == null) ? "null" : StringUtils.byteToHexString(appData)) + " cb=" + appClient; } + + public String getHAZookeeperConnectionState() { + return this.zkConnectionState.name(); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a2e0653e80f..6d016b7184c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -11,6 +11,9 @@ Release 2.6.2 - UNRELEASED YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode. (Xuan Gong via jianhe) + YARN-4101. RM should print alert messages if Zookeeper and Resourcemanager + gets connection issue. (Xuan Gong via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 14109f90e07..1660e4aeda7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -722,4 +722,13 @@ public class AdminService extends CompositeService implements throw RPCUtil.getRemoteException(ioe); } } + + public String getHAZookeeperConnectionState() { + if (!rmContext.isHAEnabled()) { + return "ResourceManager HA is not enabled."; + } else if (!autoFailoverEnabled) { + return "Auto Failover is not enabled."; + } + return this.embeddedElector.getHAZookeeperConnectionState(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java index c7b7768d10d..73bdca02067 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java @@ -205,4 +205,8 @@ public class EmbeddedElectorService extends AbstractService elector.quitElection(false); elector.joinElection(localActiveNodeInfo); } + + public String getHAZookeeperConnectionState() { + return elector.getHAZookeeperConnectionState(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java index ea5c48adff6..4225afd31df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java @@ -44,6 +44,8 @@ public class AboutBlock extends HtmlBlock { _("Cluster ID:", cinfo.getClusterId()). _("ResourceManager state:", cinfo.getState()). _("ResourceManager HA state:", cinfo.getHAState()). + _("ResourceManager HA zookeeper connection state:", + cinfo.getHAZookeeperConnectionState()). _("ResourceManager RMStateStore:", cinfo.getRMStateStore()). _("ResourceManager started on:", Times.format(cinfo.getStartedOn())). _("ResourceManager version:", cinfo.getRMBuildVersion() + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 066ba520725..2d198228dab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -113,4 +113,9 @@ public class RMWebApp extends WebApp implements YarnWebParams { } return path; } + + public String getHAZookeeperConnectionState() { + return rm.getRMContext().getRMAdminService() + .getHAZookeeperConnectionState(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index 61600dd4c8d..024a8d46aa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -58,6 +58,7 @@ public class RMWebAppFilter extends GuiceContainer { private String path; private static final int BASIC_SLEEP_TIME = 5; private static final int MAX_SLEEP_TIME = 5 * 60; + private static final Random randnum = new Random(); @Inject public RMWebAppFilter(Injector injector, Configuration conf) { @@ -126,6 +127,8 @@ public class RMWebAppFilter extends GuiceContainer { String redirectMsg = doRetry ? "Can not find any active RM. Will retry in next " + next + " seconds." : "There is no active RM right now."; + redirectMsg += "\nHA Zookeeper Connection State: " + + rmWebApp.getHAZookeeperConnectionState(); PrintWriter out = response.getWriter(); out.println(redirectMsg); if (doRetry) { @@ -172,6 +175,6 @@ public class RMWebAppFilter extends GuiceContainer { private static int calculateExponentialTime(int retries) { long baseTime = BASIC_SLEEP_TIME * (1L << retries); - return (int) (baseTime * ((new Random()).nextDouble() + 0.5)); + return (int) (baseTime * (randnum.nextDouble() + 0.5)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java index b529f211226..512a5c4182a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java @@ -43,6 +43,7 @@ public class ClusterInfo { protected String hadoopVersion; protected String hadoopBuildVersion; protected String hadoopVersionBuiltOn; + protected String haZooKeeperConnectionState; public ClusterInfo() { } // JAXB needs this @@ -62,6 +63,8 @@ public class ClusterInfo { this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); + this.haZooKeeperConnectionState = + rm.getRMContext().getRMAdminService().getHAZookeeperConnectionState(); } public String getState() { @@ -108,4 +111,7 @@ public class ClusterInfo { return this.startedOn; } + public String getHAZookeeperConnectionState() { + return this.haZooKeeperConnectionState; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 9f091d2398f..bd31ff371ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -261,6 +261,8 @@ public class TestRMWebServices extends JerseyTest { WebServicesTestUtils.getXmlLong(element, "startedOn"), WebServicesTestUtils.getXmlString(element, "state"), WebServicesTestUtils.getXmlString(element, "haState"), + WebServicesTestUtils.getXmlString( + element, "haZooKeeperConnectionState"), WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"), WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"), WebServicesTestUtils.getXmlString(element, "hadoopVersion"), @@ -276,9 +278,10 @@ public class TestRMWebServices extends JerseyTest { Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("clusterInfo"); - assertEquals("incorrect number of elements", 11, info.length()); + assertEquals("incorrect number of elements", 12, info.length()); verifyClusterGeneric(info.getLong("id"), info.getLong("startedOn"), info.getString("state"), info.getString("haState"), + info.getString("haZooKeeperConnectionState"), info.getString("hadoopVersionBuiltOn"), info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"), info.getString("resourceManagerVersionBuiltOn"), @@ -288,7 +291,8 @@ public class TestRMWebServices extends JerseyTest { } public void verifyClusterGeneric(long clusterid, long startedon, - String state, String haState, String hadoopVersionBuiltOn, + String state, String haState, String haZooKeeperConnectionState, + String hadoopVersionBuiltOn, String hadoopBuildVersion, String hadoopVersion, String resourceManagerVersionBuiltOn, String resourceManagerBuildVersion, String resourceManagerVersion) {