diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 17c2421d5d1..7c98a441add 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -638,6 +638,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12438. Reset RawLocalFileSystem.useDeprecatedFileStatus in TestLocalFileSystem. (Chris Nauroth via wheat9) + HADOOP-12437. Allow SecurityUtil to lookup alternate hostnames. + (Arpit Agarwal) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index ccb49c3c13a..d682f335e85 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -299,6 +299,12 @@ public class CommonConfigurationKeysPublic { /** See core-default.xml */ public static final String HADOOP_SECURITY_AUTH_TO_LOCAL = "hadoop.security.auth_to_local"; + /** See core-default.xml */ + public static final String HADOOP_SECURITY_DNS_INTERFACE_KEY = + "hadoop.security.dns.interface"; + /** See core-default.xml */ + public static final String HADOOP_SECURITY_DNS_NAMESERVER_KEY = + "hadoop.security.dns.nameserver"; @Deprecated /** Only used by HttpServer. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java index f19e80235b3..a6dc8e3d376 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java @@ -18,6 +18,8 @@ package org.apache.hadoop.net; +import com.google.common.net.InetAddresses; +import com.sun.istack.Nullable; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -27,9 +29,11 @@ import java.net.NetworkInterface; import java.net.SocketException; import java.net.UnknownHostException; +import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; import java.util.LinkedHashSet; +import java.util.List; import java.util.Vector; import javax.naming.NamingException; @@ -68,7 +72,7 @@ public class DNS { * @return The host name associated with the provided IP * @throws NamingException If a NamingException is encountered */ - public static String reverseDns(InetAddress hostIp, String ns) + public static String reverseDns(InetAddress hostIp, @Nullable String ns) throws NamingException { // // Builds the reverse IP lookup form @@ -228,28 +232,44 @@ public static String getDefaultIP(String strInterface) * (e.g. eth0 or eth0:0) * @param nameserver * The DNS host name + * @param tryfallbackResolution + * if true and if reverse DNS resolution fails then attempt to + * resolve the hostname with + * {@link InetAddress#getCanonicalHostName()} which includes + * hosts file resolution. * @return A string vector of all host names associated with the IPs tied to * the specified interface * @throws UnknownHostException if the given interface is invalid */ - public static String[] getHosts(String strInterface, String nameserver) - throws UnknownHostException { - String[] ips = getIPs(strInterface); - Vector hosts = new Vector(); - for (int ctr = 0; ctr < ips.length; ctr++) { + public static String[] getHosts(String strInterface, + @Nullable String nameserver, + boolean tryfallbackResolution) + throws UnknownHostException { + final List hosts = new Vector(); + final List addresses = + getIPsAsInetAddressList(strInterface, true); + for (InetAddress address : addresses) { try { - hosts.add(reverseDns(InetAddress.getByName(ips[ctr]), - nameserver)); - } catch (UnknownHostException ignored) { + hosts.add(reverseDns(address, nameserver)); } catch (NamingException ignored) { } } - if (hosts.isEmpty()) { - LOG.warn("Unable to determine hostname for interface " + strInterface); - return new String[] { cachedHostname }; - } else { - return hosts.toArray(new String[hosts.size()]); + if (hosts.isEmpty() && tryfallbackResolution) { + for (InetAddress address : addresses) { + final String canonicalHostName = address.getCanonicalHostName(); + // Don't use the result if it looks like an IP address. + if (!InetAddresses.isInetAddress(canonicalHostName)) { + hosts.add(canonicalHostName); + } + } } + + if (hosts.isEmpty()) { + LOG.warn("Unable to determine hostname for interface " + + strInterface); + hosts.add(cachedHostname); + } + return hosts.toArray(new String[hosts.size()]); } @@ -315,7 +335,7 @@ private static String resolveLocalHostIPAddress() { */ public static String[] getHosts(String strInterface) throws UnknownHostException { - return getHosts(strInterface, null); + return getHosts(strInterface, null, false); } /** @@ -331,17 +351,19 @@ public static String[] getHosts(String strInterface) * @throws UnknownHostException * If one is encountered while querying the default interface */ - public static String getDefaultHost(String strInterface, String nameserver) + public static String getDefaultHost(@Nullable String strInterface, + @Nullable String nameserver, + boolean tryfallbackResolution) throws UnknownHostException { - if ("default".equals(strInterface)) { + if (strInterface == null || "default".equals(strInterface)) { return cachedHostname; } - if ("default".equals(nameserver)) { - return getDefaultHost(strInterface); + if (nameserver != null && "default".equals(nameserver)) { + nameserver = null; } - String[] hosts = getHosts(strInterface, nameserver); + String[] hosts = getHosts(strInterface, nameserver, tryfallbackResolution); return hosts[0]; } @@ -357,9 +379,74 @@ public static String getDefaultHost(String strInterface, String nameserver) * @throws UnknownHostException * If one is encountered while querying the default interface */ - public static String getDefaultHost(String strInterface) + public static String getDefaultHost(@Nullable String strInterface) throws UnknownHostException { - return getDefaultHost(strInterface, null); + return getDefaultHost(strInterface, null, false); } + /** + * Returns the default (first) host name associated by the provided + * nameserver with the address bound to the specified network interface. + * + * @param strInterface + * The name of the network interface to query (e.g. eth0) + * @param nameserver + * The DNS host name + * @throws UnknownHostException + * If one is encountered while querying the default interface + */ + public static String getDefaultHost(@Nullable String strInterface, + @Nullable String nameserver) + throws UnknownHostException { + return getDefaultHost(strInterface, nameserver, false); + } + + /** + * Returns all the IPs associated with the provided interface, if any, as + * a list of InetAddress objects. + * + * @param strInterface + * The name of the network interface or sub-interface to query + * (eg eth0 or eth0:0) or the string "default" + * @param returnSubinterfaces + * Whether to return IPs associated with subinterfaces of + * the given interface + * @return A list of all the IPs associated with the provided + * interface. The local host IP is returned if the interface + * name "default" is specified or there is an I/O error looking + * for the given interface. + * @throws UnknownHostException + * If the given interface is invalid + * + */ + public static List getIPsAsInetAddressList(String strInterface, + boolean returnSubinterfaces) throws UnknownHostException { + if ("default".equals(strInterface)) { + return Arrays.asList(InetAddress.getByName(cachedHostAddress)); + } + NetworkInterface netIf; + try { + netIf = NetworkInterface.getByName(strInterface); + if (netIf == null) { + netIf = getSubinterface(strInterface); + } + } catch (SocketException e) { + LOG.warn("I/O error finding interface " + strInterface + + ": " + e.getMessage()); + return Arrays.asList(InetAddress.getByName(cachedHostAddress)); + } + if (netIf == null) { + throw new UnknownHostException("No such interface " + strInterface); + } + + // NB: Using a LinkedHashSet to preserve the order for callers + // that depend on a particular element being 1st in the array. + // For example, getDefaultIP always returns the first element. + LinkedHashSet allAddrs = new LinkedHashSet(); + allAddrs.addAll(Collections.list(netIf.getInetAddresses())); + if (!returnSubinterfaces) { + allAddrs.removeAll(getSubinterfaceInetAddrs(netIf)); + } + return new Vector(allAddrs); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java index eddf98d07ff..38096ab4715 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java @@ -17,6 +17,8 @@ package org.apache.hadoop.security; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_DNS_INTERFACE_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_DNS_NAMESERVER_KEY; import java.io.IOException; import java.net.InetAddress; @@ -29,6 +31,7 @@ import java.util.List; import java.util.ServiceLoader; +import javax.annotation.Nullable; import javax.security.auth.kerberos.KerberosPrincipal; import javax.security.auth.kerberos.KerberosTicket; @@ -39,6 +42,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.Text; +import org.apache.hadoop.net.DNS; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; @@ -180,13 +184,38 @@ private static String replacePattern(String[] components, String hostname) throws IOException { String fqdn = hostname; if (fqdn == null || fqdn.isEmpty() || fqdn.equals("0.0.0.0")) { - fqdn = getLocalHostName(); + fqdn = getLocalHostName(null); } return components[0] + "/" + StringUtils.toLowerCase(fqdn) + "@" + components[2]; } - - static String getLocalHostName() throws UnknownHostException { + + /** + * Retrieve the name of the current host. Multihomed hosts may restrict the + * hostname lookup to a specific interface and nameserver with {@link + * org.apache.hadoop.fs.CommonConfigurationKeysPublic#HADOOP_SECURITY_DNS_INTERFACE_KEY} + * and {@link org.apache.hadoop.fs.CommonConfigurationKeysPublic#HADOOP_SECURITY_DNS_NAMESERVER_KEY} + * + * @param conf Configuration object. May be null. + * @return + * @throws UnknownHostException + */ + static String getLocalHostName(@Nullable Configuration conf) + throws UnknownHostException { + if (conf != null) { + String dnsInterface = conf.get(HADOOP_SECURITY_DNS_INTERFACE_KEY); + String nameServer = conf.get(HADOOP_SECURITY_DNS_NAMESERVER_KEY); + + if (dnsInterface != null) { + return DNS.getDefaultHost(dnsInterface, nameServer, true); + } else if (nameServer != null) { + throw new IllegalArgumentException(HADOOP_SECURITY_DNS_NAMESERVER_KEY + + " requires " + HADOOP_SECURITY_DNS_INTERFACE_KEY + ". Check your" + + "configuration."); + } + } + + // Fallback to querying the default hostname as we did before. return InetAddress.getLocalHost().getCanonicalHostName(); } @@ -207,7 +236,7 @@ static String getLocalHostName() throws UnknownHostException { @InterfaceStability.Evolving public static void login(final Configuration conf, final String keytabFileKey, final String userNameKey) throws IOException { - login(conf, keytabFileKey, userNameKey, getLocalHostName()); + login(conf, keytabFileKey, userNameKey, getLocalHostName(conf)); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index b5d646347ae..a12f15c76f4 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -97,6 +97,31 @@ + + hadoop.security.dns.interface + + The name of the Network Interface from which the service should determine + its host name for Kerberos login. e.g. eth2. In a multi-homed environment, + the setting can be used to affect the _HOST subsitution in the service + Kerberos principal. If this configuration value is not set, the service + will use its default hostname as returned by + InetAddress.getLocalHost().getCanonicalHostName(). + + Most clusters will not require this setting. + + + + + hadoop.security.dns.nameserver + + The host name or IP address of the name server (DNS) which a service Node + should use to determine its own host name for Kerberos Login. Requires + hadoop.security.dns.interface. + + Most clusters will not require this setting. + + +