diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 338553ceaf4..83fd9839d3c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -139,6 +139,9 @@ Release 2.9.0 - UNRELEASED
YARN-4669. Fix logging statements in resource manager's Application class.
(Seethana Sidharta via vvasudev)
+ YARN-4629. Distributed shell breaks under strong security.
+ (Daniel Templeton via stevel)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 68d2bde5165..e864ad23005 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.client.util.YarnClientUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -669,7 +670,7 @@ public class Client {
if (UserGroupInformation.isSecurityEnabled()) {
// Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
Credentials credentials = new Credentials();
- String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
+ String tokenRenewer = YarnClientUtils.getRmPrincipal(conf);
if (tokenRenewer == null || tokenRenewer.length() == 0) {
throw new IOException(
"Can't get Master Kerberos principal for the RM to use as renewer");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
new file mode 100644
index 00000000000..1e3112aff70
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * This class is a container for utility methods that are useful when creating
+ * YARN clients.
+ */
+public abstract class YarnClientUtils {
+ /**
+ * Look up and return the resource manager's principal. This method
+ * automatically does the _HOST replacement in the principal and
+ * correctly handles HA resource manager configurations.
+ *
+ * @param conf the {@link Configuration} file from which to read the
+ * principal
+ * @return the resource manager's principal string or null if the
+ * {@link YarnConfiguration#RM_PRINCIPAL} property is not set in the
+ * {@code conf} parameter
+ * @throws IOException thrown if there's an error replacing the host name
+ */
+ public static String getRmPrincipal(Configuration conf) throws IOException {
+ String principal = conf.get(YarnConfiguration.RM_PRINCIPAL);
+ String prepared = null;
+
+ if (principal != null) {
+ prepared = getRmPrincipal(principal, conf);
+ }
+
+ return prepared;
+ }
+
+ /**
+ * Perform the _HOST replacement in the {@code principal},
+ * Returning the result. Correctly handles HA resource manager configurations.
+ *
+ * @param rmPrincipal the principal string to prepare
+ * @param conf the configuration
+ * @return the prepared principal string
+ * @throws IOException thrown if there's an error replacing the host name
+ */
+ public static String getRmPrincipal(String rmPrincipal, Configuration conf)
+ throws IOException {
+ if (rmPrincipal == null) {
+ throw new IllegalArgumentException("RM principal string is null");
+ }
+
+ if (HAUtil.isHAEnabled(conf)) {
+ conf = getYarnConfWithRmHaId(conf);
+ }
+
+ String hostname = conf.getSocketAddr(
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT).getHostName();
+
+ return SecurityUtil.getServerPrincipal(rmPrincipal, hostname);
+ }
+
+ /**
+ * Returns a {@link YarnConfiguration} built from the {@code conf} parameter
+ * that is guaranteed to have the {@link YarnConfiguration#RM_HA_ID}
+ * property set.
+ *
+ * @param conf the base configuration
+ * @return a {@link YarnConfiguration} built from the base
+ * {@link Configuration}
+ * @throws IOException thrown if the {@code conf} parameter contains
+ * inconsistent properties
+ */
+ @VisibleForTesting
+ static YarnConfiguration getYarnConfWithRmHaId(Configuration conf)
+ throws IOException {
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+
+ if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
+ // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
+ // Any valid RM HA ID should work.
+ String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
+
+ if ((rmIds != null) && (rmIds.length > 0)) {
+ yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
+ } else {
+ throw new IOException("RM_HA_IDS property is not set for HA resource "
+ + "manager");
+ }
+ }
+
+ return yarnConf;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/package-info.java
new file mode 100644
index 00000000000..e7eaebe479d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Public
+package org.apache.hadoop.yarn.client.util;
+import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/util/TestYarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/util/TestYarnClientUtils.java
new file mode 100644
index 00000000000..42300a09851
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/util/TestYarnClientUtils.java
@@ -0,0 +1,319 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.util;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * Tests for the YarnClientUtils class
+ */
+public class TestYarnClientUtils {
+ /**
+ * Test of getRMPrincipal(Configuration) method, of class YarnClientUtils
+ * when HA is not enabled.
+ *
+ * @throws java.io.IOException thrown if stuff breaks
+ */
+ @Test
+ public void testGetRMPrincipalStandAlone_Configuration() throws IOException {
+ Configuration conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS, "myhost");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, false);
+
+ String result = YarnClientUtils.getRmPrincipal(conf);
+
+ assertNull("The hostname translation did return null when the principal is "
+ + "missing from the conf: " + result, result);
+
+ conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS, "myhost");
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "test/_HOST@REALM");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, false);
+
+ result = YarnClientUtils.getRmPrincipal(conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/myhost@REALM", result);
+
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "test/yourhost@REALM");
+
+ result = YarnClientUtils.getRmPrincipal(conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/yourhost@REALM", result);
+ }
+
+ /**
+ * Test of getRMPrincipal(Configuration) method, of class YarnClientUtils
+ * when HA is enabled.
+ *
+ * @throws java.io.IOException thrown if stuff breaks
+ */
+ @Test
+ public void testGetRMPrincipalHA_Configuration() throws IOException {
+ Configuration conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS, "myhost");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+
+ String result = YarnClientUtils.getRmPrincipal(conf);
+
+ assertNull("The hostname translation did return null when the principal is "
+ + "missing from the conf: " + result, result);
+
+ conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS + ".rm0", "myhost");
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "test/_HOST@REALM");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm0");
+
+ result = YarnClientUtils.getRmPrincipal(conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/myhost@REALM", result);
+
+ conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS + ".rm0", "myhost");
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "test/_HOST@REALM");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+
+ try {
+ result = YarnClientUtils.getRmPrincipal(conf);
+ fail("The hostname translation succeeded even though no RM ids were "
+ + "set: " + result);
+ } catch (IOException ex) {
+ // Expected
+ }
+
+ conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS + ".rm0", "myhost");
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "test/_HOST@REALM");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_ID, "rm0");
+
+ result = YarnClientUtils.getRmPrincipal(conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/myhost@REALM", result);
+
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "test/yourhost@REALM");
+
+ result = YarnClientUtils.getRmPrincipal(conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/yourhost@REALM", result);
+ }
+
+ /**
+ * Test of getRMPrincipal(Configuration) method, of class YarnClientUtils
+ * when HA is not enabled.
+ *
+ * @throws java.io.IOException thrown if stuff breaks
+ */
+ @Test
+ public void testGetRMPrincipalStandAlone_String() throws IOException {
+ Configuration conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS, "myhost");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, false);
+
+ String result = YarnClientUtils.getRmPrincipal("test/_HOST@REALM", conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/myhost@REALM", result);
+
+ result = YarnClientUtils.getRmPrincipal("test/yourhost@REALM", conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/yourhost@REALM", result);
+
+ try {
+ result = YarnClientUtils.getRmPrincipal(null, conf);
+ fail("The hostname translation succeeded even though the RM principal "
+ + "was null: " + result);
+ } catch (IllegalArgumentException ex) {
+ // Expected
+ }
+ }
+
+ /**
+ * Test of getRMPrincipal(Configuration) method, of class YarnClientUtils
+ * when HA is enabled.
+ *
+ * @throws java.io.IOException thrown if stuff breaks
+ */
+ @Test
+ public void testGetRMPrincipalHA_String() throws IOException {
+ Configuration conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS + ".rm0", "myhost");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm0");
+
+ String result = YarnClientUtils.getRmPrincipal("test/_HOST@REALM", conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/myhost@REALM", result);
+
+ try {
+ result = YarnClientUtils.getRmPrincipal(null, conf);
+ fail("The hostname translation succeeded even though the RM principal "
+ + "was null: " + result);
+ } catch (IllegalArgumentException ex) {
+ // Expected
+ }
+
+ conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS + ".rm0", "myhost");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+
+ try {
+ YarnClientUtils.getRmPrincipal("test/_HOST@REALM", conf);
+ fail("The hostname translation succeeded even though no RM ids were set");
+ } catch (IOException ex) {
+ // Expected
+ }
+
+ conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_ADDRESS + ".rm0", "myhost");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_ID, "rm0");
+
+ result = YarnClientUtils.getRmPrincipal("test/_HOST@REALM", conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/myhost@REALM", result);
+
+ result = YarnClientUtils.getRmPrincipal("test/yourhost@REALM", conf);
+
+ assertEquals("The hostname translation did not produce the expected "
+ + "results: " + result, "test/yourhost@REALM", result);
+ }
+
+ /**
+ * Test of getRMPrincipal method of class YarnClientUtils.
+ *
+ * @throws IOException thrown when something breaks
+ */
+ @Test
+ public void testGetYarnConfWithRmHaId() throws IOException {
+ Configuration conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_HA_ID, "rm0");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, false);
+
+ YarnConfiguration result = YarnClientUtils.getYarnConfWithRmHaId(conf);
+
+ assertSameConf(conf, result);
+ assertEquals("RM_HA_ID was changed when it shouldn't have been: "
+ + result.get(YarnConfiguration.RM_HA_ID), "rm0",
+ result.get(YarnConfiguration.RM_HA_ID));
+
+ conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_HA_ID, "rm0");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+
+ result = YarnClientUtils.getYarnConfWithRmHaId(conf);
+
+ assertSameConf(conf, result);
+ assertEquals("RM_HA_ID was changed when it shouldn't have been: "
+ + result.get(YarnConfiguration.RM_HA_ID), "rm0",
+ result.get(YarnConfiguration.RM_HA_ID));
+
+ conf = new Configuration();
+
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm0,rm1");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+
+ result = YarnClientUtils.getYarnConfWithRmHaId(conf);
+
+ assertSameConf(conf, result);
+ assertEquals("RM_HA_ID was not set correctly: "
+ + result.get(YarnConfiguration.RM_HA_ID), "rm0",
+ result.get(YarnConfiguration.RM_HA_ID));
+
+ conf = new Configuration();
+
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+
+ try {
+ YarnClientUtils.getYarnConfWithRmHaId(conf);
+ fail("Allowed invalid HA configuration: HA is enabled, but no RM ID "
+ + "is set");
+ } catch (IOException ex) {
+ // Expected
+ }
+ }
+
+ /**
+ * Compare two configurations to see that they both have the same values.
+ * The YarnConfiguration.RM_HA_ID property is ignored, as it as expected to
+ * change and be tested external to this method.
+ *
+ * @param master the master Configuration
+ * @param copy the copy Configuration
+ */
+ private void assertSameConf(Configuration master, YarnConfiguration copy) {
+ Set seen = new HashSet<>();
+ Iterator> itr = master.iterator();
+
+ // Always ignore the RM_HA_ID, because we expect it to change
+ seen.add(YarnConfiguration.RM_HA_ID);
+
+ while (itr.hasNext()) {
+ Entry property = itr.next();
+ String key = property.getKey();
+
+ if (!seen.add(key)) {
+ // Here we use master.get() instead of property.getValue() because
+ // they're not the same thing.
+ assertEquals("New configuration changed the value of "
+ + key, master.get(key), copy.get(key));
+ }
+ }
+
+ itr = copy.iterator();
+
+ while (itr.hasNext()) {
+ Entry property = itr.next();
+ String key = property.getKey();
+
+ if (!seen.contains(property.getKey())) {
+ assertEquals("New configuration changed the value of "
+ + key, copy.get(key),
+ master.get(key));
+ }
+ }
+ }
+}