From 690ae753858704966257328ba18db9d6173fda9f Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 13 Dec 2013 23:55:31 +0000 Subject: [PATCH] YARN-1485. Modified RM HA configuration validation to also ensure that service-address configuration are configured for every RM. Contributed by Xuan Gong. svn merge --ignore-ancestry -c 1550854 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1550855 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 +++ .../org/apache/hadoop/yarn/conf/HAUtil.java | 32 ++++++++++++++++--- .../apache/hadoop/yarn/conf/TestHAUtil.java | 11 +++++-- .../recovery/TestZKRMStateStore.java | 4 ++- .../hadoop/yarn/server/MiniYARNCluster.java | 4 ++- 5 files changed, 46 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b7aaf1aadcd..f9054a74d4c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -150,6 +150,10 @@ Release 2.4.0 - UNRELEASED YARN-1311. Fixed app specific scheduler-events' names to be app-attempt based. (vinodkv via jianhe) + YARN-1485. Modified RM HA configuration validation to also ensure that + service-address configuration are configured for every RM. (Xuan Gong via + vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index 1e8a7c4c78b..cbdb8b34c88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -58,13 +58,17 @@ public class HAUtil { */ public static void verifyAndSetConfiguration(Configuration conf) throws YarnRuntimeException { - verifyAndSetRMHAIds(conf); - verifyAndSetRMHAId(conf); + verifyAndSetRMHAIdsList(conf); + verifyAndSetCurrentRMHAId(conf); verifyAndSetAllServiceAddresses(conf); } - - private static void verifyAndSetRMHAIds(Configuration conf) { + /** + * Verify configuration that there are at least two RM-ids + * and RPC addresses are specified for each RM-id. + * Then set the RM-ids. + */ + private static void verifyAndSetRMHAIdsList(Configuration conf) { Collection ids = conf.getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS); if (ids.size() < 2) { @@ -76,6 +80,24 @@ public class HAUtil { StringBuilder setValue = new StringBuilder(); for (String id: ids) { + // verify the RM service addresses configurations for every RMIds + for (String prefix : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + String confKey = null; + try { + confKey = addSuffix(prefix, id); + if (conf.getTrimmed(confKey) == null) { + throwBadConfigurationException(getNeedToSetValueMessage(confKey)); + } + } catch (IllegalArgumentException iae) { + String errmsg = iae.getMessage(); + if (confKey == null) { + // Error at addSuffix + errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, + getRMHAId(conf)); + } + throwBadConfigurationException(errmsg); + } + } setValue.append(id); setValue.append(","); } @@ -83,7 +105,7 @@ public class HAUtil { setValue.substring(0, setValue.length() - 1)); } - private static void verifyAndSetRMHAId(Configuration conf) { + private static void verifyAndSetCurrentRMHAId(Configuration conf) { String rmId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); if (rmId == null) { throwBadConfigurationException( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java index a710c361c9b..1908b6b75ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java @@ -39,6 +39,7 @@ public class TestHAUtil { private static final String RM1_ADDRESS_UNTRIMMED = " \t\t\n 1.2.3.4:8021 \n\t "; private static final String RM1_ADDRESS = RM1_ADDRESS_UNTRIMMED.trim(); private static final String RM2_ADDRESS = "localhost:8022"; + private static final String RM3_ADDRESS = "localhost:8033"; private static final String RM1_NODE_ID_UNTRIMMED = "rm1 "; private static final String RM1_NODE_ID = RM1_NODE_ID_UNTRIMMED.trim(); private static final String RM2_NODE_ID = "rm2"; @@ -113,8 +114,13 @@ public class TestHAUtil { } conf.clear(); - conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + "," - + RM1_NODE_ID); + // simulate the case YarnConfiguration.RM_HA_ID is not set + conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + + RM2_NODE_ID); + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); + conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); + } try { HAUtil.verifyAndSetConfiguration(conf); } catch (YarnRuntimeException e) { @@ -165,6 +171,7 @@ public class TestHAUtil { for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); + conf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); } try { HAUtil.verifyAndSetConfiguration(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 6c7c472cbdf..b09ed0bec23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -143,7 +143,9 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS, hostPort); conf.set(YarnConfiguration.RM_HA_ID, rmId); for (String rpcAddress : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { - conf.set(HAUtil.addSuffix(rpcAddress, rmId), "localhost:0"); + for (String id : HAUtil.getRMHAIds(conf)) { + conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0"); + } } conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId), "localhost:" + adminPort); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 95b1ba5be8b..54de419a63d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -296,7 +296,9 @@ public class MiniYARNCluster extends CompositeService { String hostname = MiniYARNCluster.getHostname(); conf.set(YarnConfiguration.RM_HA_ID, rmId); for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { - conf.set(HAUtil.addSuffix(confKey, rmId), hostname + ":0"); + for (String id : HAUtil.getRMHAIds(conf)) { + conf.set(HAUtil.addSuffix(confKey, id), hostname + ":0"); + } } }