From d5145b58f6353de97bee90192c9d51d4046ae35a Mon Sep 17 00:00:00 2001 From: Junping Du Date: Thu, 22 Oct 2015 13:41:09 -0700 Subject: [PATCH] YARN-4243. Add retry on establishing Zookeeper conenction in EmbeddedElectorService#serviceInit. Contributed by Xuan Gong. (cherry picked from commit 0fce5f9a496925f0d53ea6c14318c9b513de9882) --- .../hadoop/ha/ActiveStandbyElector.java | 53 +++++++++++++++++-- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 4 ++ .../src/main/resources/yarn-default.xml | 7 +++ .../EmbeddedElectorService.java | 9 ++-- 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java index fcbcfdf1b85..cb2e08143e3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java @@ -208,8 +208,49 @@ public class ActiveStandbyElector implements StatCallback, StringCallback { */ public ActiveStandbyElector(String zookeeperHostPorts, int zookeeperSessionTimeout, String parentZnodeName, List acl, - List authInfo, - ActiveStandbyElectorCallback app, int maxRetryNum) throws IOException, + List authInfo, ActiveStandbyElectorCallback app, + int maxRetryNum) throws IOException, HadoopIllegalArgumentException, + KeeperException { + this(zookeeperHostPorts, zookeeperSessionTimeout, parentZnodeName, acl, + authInfo, app, maxRetryNum, true); + } + + /** + * Create a new ActiveStandbyElector object
+ * The elector is created by providing to it the Zookeeper configuration, the + * parent znode under which to create the znode and a reference to the + * callback interface.
+ * The parent znode name must be the same for all service instances and + * different across services.
+ * After the leader has been lost, a new leader will be elected after the + * session timeout expires. Hence, the app must set this parameter based on + * its needs for failure response time. The session timeout must be greater + * than the Zookeeper disconnect timeout and is recommended to be 3X that + * value to enable Zookeeper to retry transient disconnections. Setting a very + * short session timeout may result in frequent transitions between active and + * standby states during issues like network outages/GS pauses. + * + * @param zookeeperHostPorts + * ZooKeeper hostPort for all ZooKeeper servers + * @param zookeeperSessionTimeout + * ZooKeeper session timeout + * @param parentZnodeName + * znode under which to create the lock + * @param acl + * ZooKeeper ACL's + * @param authInfo a list of authentication credentials to add to the + * ZK connection + * @param app + * reference to callback interface object + * @param failFast + * whether need to add the retry when establishing ZK connection. + * @throws IOException + * @throws HadoopIllegalArgumentException + */ + public ActiveStandbyElector(String zookeeperHostPorts, + int zookeeperSessionTimeout, String parentZnodeName, List acl, + List authInfo, ActiveStandbyElectorCallback app, + int maxRetryNum, boolean failFast) throws IOException, HadoopIllegalArgumentException, KeeperException { if (app == null || acl == null || parentZnodeName == null || zookeeperHostPorts == null || zookeeperSessionTimeout <= 0) { @@ -225,8 +266,12 @@ public class ActiveStandbyElector implements StatCallback, StringCallback { zkBreadCrumbPath = znodeWorkingDir + "/" + BREADCRUMB_FILENAME; this.maxRetryNum = maxRetryNum; - // createConnection for future API calls - createConnection(); + // establish the ZK Connection for future API calls + if (failFast) { + createConnection(); + } else { + reEstablishSession(); + } } /** diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8f090f58808..eb0e54d1f62 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -476,6 +476,9 @@ Release 2.8.0 - UNRELEASED YARN-3985. Make ReservationSystem persist state using RMStateStore reservation APIs. (adhoot via asuresh) + YARN-4243. Add retry on establishing Zookeeper conenction in + EmbeddedElectorService#serviceInit. (Xuan Gong via junping_du) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0fcdce2cc19..96762fd2bf9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -531,6 +531,10 @@ public class YarnConfiguration extends Configuration { public static final int DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 0; + /** number of zookeeper operation retry times in ActiveStandbyElector */ + public static final String RM_HA_FC_ELECTOR_ZK_RETRIES_KEY = RM_HA_PREFIX + + "failover-controller.active-standby-elector.zk.retries"; + //////////////////////////////// // RM state store configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 5dc45902fe4..c6ffe18be60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -388,6 +388,13 @@ org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore + + When automatic failover is enabled, number of zookeeper + operation retry times in ActiveStandbyElector + yarn.resourcemanager.ha.failover-controller.active-standby-elector.zk.retries + + + The maximum number of completed applications RM state store keeps, less than or equals to ${yarn.resourcemanager.max-completed-applications}. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java index 73bdca02067..72327e82e9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java @@ -86,11 +86,12 @@ public class EmbeddedElectorService extends AbstractService List zkAcls = RMZKUtils.getZKAcls(conf); List zkAuths = RMZKUtils.getZKAuths(conf); - int maxRetryNum = conf.getInt( - CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_KEY, - CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_DEFAULT); + int maxRetryNum = + conf.getInt(YarnConfiguration.RM_HA_FC_ELECTOR_ZK_RETRIES_KEY, conf + .getInt(CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_KEY, + CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_DEFAULT)); elector = new ActiveStandbyElector(zkQuorum, (int) zkSessionTimeout, - electionZNode, zkAcls, zkAuths, this, maxRetryNum); + electionZNode, zkAcls, zkAuths, this, maxRetryNum, false); elector.ensureParentZNode(); if (!isParentZnodeSafe(clusterId)) {