From 09c64ba1ba8be7a2ac31f4e42efb8c99b682399f Mon Sep 17 00:00:00 2001 From: Jian He Date: Wed, 2 Sep 2015 17:45:23 -0700 Subject: [PATCH] YARN-4101. RM should print alert messages if Zookeeper and Resourcemanager gets connection issue. Contributed by Xuan Gong --- .../java/org/apache/hadoop/ha/ActiveStandbyElector.java | 4 ++++ hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/server/resourcemanager/AdminService.java | 9 +++++++++ .../server/resourcemanager/EmbeddedElectorService.java | 4 ++++ .../yarn/server/resourcemanager/webapp/AboutBlock.java | 2 ++ .../yarn/server/resourcemanager/webapp/RMWebApp.java | 5 +++++ .../server/resourcemanager/webapp/RMWebAppFilter.java | 5 ++++- .../server/resourcemanager/webapp/dao/ClusterInfo.java | 6 ++++++ .../server/resourcemanager/webapp/TestRMWebServices.java | 8 ++++++-- 9 files changed, 43 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java index e458181de60..fcbcfdf1b85 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java @@ -1141,4 +1141,8 @@ public String toString() { ((appData == null) ? "null" : StringUtils.byteToHexString(appData)) + " cb=" + appClient; } + + public String getHAZookeeperConnectionState() { + return this.zkConnectionState.name(); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 13fe9b00b48..5e0c43daf5d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -871,6 +871,9 @@ Release 2.7.2 - UNRELEASED YARN-3893. Both RM in active state when Admin#transitionToActive failure from refeshAll() (Bibin A Chundatt via rohithsharmaks) + YARN-4101. RM should print alert messages if Zookeeper and Resourcemanager + gets connection issue. (Xuan Gong via jianhe) + Release 2.7.1 - 2015-07-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index d96ed8cf9cc..a95e22c0a42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -782,4 +782,13 @@ public RefreshClusterMaxPriorityResponse refreshClusterMaxPriority( throw logAndWrapException(e, user.getShortUserName(), argName, msg); } } + + public String getHAZookeeperConnectionState() { + if (!rmContext.isHAEnabled()) { + return "ResourceManager HA is not enabled."; + } else if (!autoFailoverEnabled) { + return "Auto Failover is not enabled."; + } + return this.embeddedElector.getHAZookeeperConnectionState(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java index c7b7768d10d..73bdca02067 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java @@ -205,4 +205,8 @@ public void resetLeaderElection() { elector.quitElection(false); elector.joinElection(localActiveNodeInfo); } + + public String getHAZookeeperConnectionState() { + return elector.getHAZookeeperConnectionState(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java index ea5c48adff6..4225afd31df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java @@ -44,6 +44,8 @@ protected void render(Block html) { _("Cluster ID:", cinfo.getClusterId()). _("ResourceManager state:", cinfo.getState()). _("ResourceManager HA state:", cinfo.getHAState()). + _("ResourceManager HA zookeeper connection state:", + cinfo.getHAZookeeperConnectionState()). _("ResourceManager RMStateStore:", cinfo.getRMStateStore()). _("ResourceManager started on:", Times.format(cinfo.getStartedOn())). _("ResourceManager version:", cinfo.getRMBuildVersion() + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index a86ed4f11be..91c3d72e06c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -117,4 +117,9 @@ private String buildRedirectPath() { } return path; } + + public String getHAZookeeperConnectionState() { + return rm.getRMContext().getRMAdminService() + .getHAZookeeperConnectionState(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index a8f793a5a8b..9f35b6ce566 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -58,6 +58,7 @@ public class RMWebAppFilter extends GuiceContainer { private String path; private static final int BASIC_SLEEP_TIME = 5; private static final int MAX_SLEEP_TIME = 5 * 60; + private static final Random randnum = new Random(); @Inject public RMWebAppFilter(Injector injector, Configuration conf) { @@ -126,6 +127,8 @@ && shouldRedirect(rmWebApp, uri)) { String redirectMsg = doRetry ? "Can not find any active RM. Will retry in next " + next + " seconds." : "There is no active RM right now."; + redirectMsg += "\nHA Zookeeper Connection State: " + + rmWebApp.getHAZookeeperConnectionState(); PrintWriter out = response.getWriter(); out.println(redirectMsg); if (doRetry) { @@ -172,6 +175,6 @@ private String appendOrReplaceParamter(String uri, String newQuery) { private static int calculateExponentialTime(int retries) { long baseTime = BASIC_SLEEP_TIME * (1L << retries); - return (int) (baseTime * ((new Random()).nextDouble() + 0.5)); + return (int) (baseTime * (randnum.nextDouble() + 0.5)); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java index b529f211226..512a5c4182a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java @@ -43,6 +43,7 @@ public class ClusterInfo { protected String hadoopVersion; protected String hadoopBuildVersion; protected String hadoopVersionBuiltOn; + protected String haZooKeeperConnectionState; public ClusterInfo() { } // JAXB needs this @@ -62,6 +63,8 @@ public ClusterInfo(ResourceManager rm) { this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); + this.haZooKeeperConnectionState = + rm.getRMContext().getRMAdminService().getHAZookeeperConnectionState(); } public String getState() { @@ -108,4 +111,7 @@ public long getStartedOn() { return this.startedOn; } + public String getHAZookeeperConnectionState() { + return this.haZooKeeperConnectionState; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 752e99bb105..530c06fbf6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -285,6 +285,8 @@ public void verifyClusterInfoXML(String xml) throws JSONException, Exception { WebServicesTestUtils.getXmlLong(element, "startedOn"), WebServicesTestUtils.getXmlString(element, "state"), WebServicesTestUtils.getXmlString(element, "haState"), + WebServicesTestUtils.getXmlString( + element, "haZooKeeperConnectionState"), WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"), WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"), WebServicesTestUtils.getXmlString(element, "hadoopVersion"), @@ -300,9 +302,10 @@ public void verifyClusterInfo(JSONObject json) throws JSONException, Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("clusterInfo"); - assertEquals("incorrect number of elements", 11, info.length()); + assertEquals("incorrect number of elements", 12, info.length()); verifyClusterGeneric(info.getLong("id"), info.getLong("startedOn"), info.getString("state"), info.getString("haState"), + info.getString("haZooKeeperConnectionState"), info.getString("hadoopVersionBuiltOn"), info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"), info.getString("resourceManagerVersionBuiltOn"), @@ -312,7 +315,8 @@ public void verifyClusterInfo(JSONObject json) throws JSONException, } public void verifyClusterGeneric(long clusterid, long startedon, - String state, String haState, String hadoopVersionBuiltOn, + String state, String haState, String haZooKeeperConnectionState, + String hadoopVersionBuiltOn, String hadoopBuildVersion, String hadoopVersion, String resourceManagerVersionBuiltOn, String resourceManagerBuildVersion, String resourceManagerVersion) {