HADOOP-7314. Add support for throwing UnknownHostException when a host doesn't resolve. Contributed by Jeffrey Naisbitt.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1147317 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
f582aa4492
commit
31f1a674e5
|
@ -266,6 +266,9 @@ Trunk (unreleased changes)
|
|||
|
||||
HADOOP-7305. Eclipse project files are incomplete. (Niels Basjes via eli)
|
||||
|
||||
HADOOP-7314. Add support for throwing UnknownHostException when a host doesn't
|
||||
resolve. (Jeffrey Naisbitt via jitendra)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
|
||||
|
|
|
@ -17,6 +17,9 @@
|
|||
*/
|
||||
package org.apache.hadoop.net;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.SocketException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -42,16 +45,11 @@ public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
|
|||
this.rawMapping = rawMapping;
|
||||
}
|
||||
|
||||
public List<String> resolve(List<String> names) {
|
||||
// normalize all input names to be in the form of IP addresses
|
||||
names = NetUtils.normalizeHostNames(names);
|
||||
|
||||
List <String> result = new ArrayList<String>(names.size());
|
||||
if (names.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the hosts from 'names' that have not been cached previously
|
||||
*/
|
||||
private List<String> getUncachedHosts(List<String> names) {
|
||||
// find out all names without cached resolved location
|
||||
List<String> unCachedHosts = new ArrayList<String>(names.size());
|
||||
for (String name : names) {
|
||||
|
@ -59,27 +57,81 @@ public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
|
|||
unCachedHosts.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve those names
|
||||
List<String> rNames = rawMapping.resolve(unCachedHosts);
|
||||
|
||||
return unCachedHosts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches the resolved hosts
|
||||
*/
|
||||
private void cacheResolvedHosts(List<String> uncachedHosts,
|
||||
List<String> resolvedHosts) {
|
||||
// Cache the result
|
||||
if (rNames != null) {
|
||||
for (int i=0; i<unCachedHosts.size(); i++) {
|
||||
cache.put(unCachedHosts.get(i), rNames.get(i));
|
||||
if (resolvedHosts != null) {
|
||||
for (int i=0; i<uncachedHosts.size(); i++) {
|
||||
cache.put(uncachedHosts.get(i), resolvedHosts.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the cached resolution of the list of hostnames/addresses.
|
||||
* Returns null if any of the names are not currently in the cache
|
||||
*/
|
||||
private List<String> getCachedHosts(List<String> names) {
|
||||
List<String> result = new ArrayList<String>(names.size());
|
||||
// Construct the result
|
||||
for (String name : names) {
|
||||
//now everything is in the cache
|
||||
String networkLocation = cache.get(name);
|
||||
if (networkLocation != null) {
|
||||
result.add(networkLocation);
|
||||
} else { //resolve all or nothing
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves host names and adds them to the cache.
|
||||
* Unlike the 'resolve" method, this won't hide UnknownHostExceptions
|
||||
*
|
||||
* @param names to resolve
|
||||
* @return List of resolved names
|
||||
* @throws UnknownHostException if any hosts cannot be resolved
|
||||
*/
|
||||
public List<String> resolveValidHosts(List<String> names)
|
||||
throws UnknownHostException {
|
||||
if (names.isEmpty()) {
|
||||
return new ArrayList<String>();
|
||||
}
|
||||
List<String> addresses = new ArrayList<String>(names.size());
|
||||
for (String name : names) {
|
||||
addresses.add(InetAddress.getByName(name).getHostAddress());
|
||||
}
|
||||
|
||||
List<String> uncachedHosts = this.getUncachedHosts(names);
|
||||
|
||||
// Resolve the uncached hosts
|
||||
List<String> resolvedHosts = rawMapping.resolveValidHosts(uncachedHosts);
|
||||
this.cacheResolvedHosts(uncachedHosts, resolvedHosts);
|
||||
return this.getCachedHosts(addresses);
|
||||
}
|
||||
|
||||
public List<String> resolve(List<String> names) {
|
||||
// normalize all input names to be in the form of IP addresses
|
||||
names = NetUtils.normalizeHostNames(names);
|
||||
|
||||
List <String> result = new ArrayList<String>(names.size());
|
||||
if (names.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
List<String> uncachedHosts = this.getUncachedHosts(names);
|
||||
|
||||
// Resolve the uncached hosts
|
||||
List<String> resolvedHosts = rawMapping.resolve(uncachedHosts);
|
||||
this.cacheResolvedHosts(uncachedHosts, resolvedHosts);
|
||||
return this.getCachedHosts(names);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
package org.apache.hadoop.net;
|
||||
|
||||
import java.util.List;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
@ -44,4 +45,23 @@ public interface DNSToSwitchMapping {
|
|||
* @return list of resolved network paths
|
||||
*/
|
||||
public List<String> resolve(List<String> names);
|
||||
|
||||
/**
|
||||
* Resolves a list of DNS-names/IP-addresses and returns back a list of
|
||||
* switch information (network paths). One-to-one correspondence must be
|
||||
* maintained between the elements in the lists.
|
||||
* Consider an element in the argument list - x.y.com. The switch information
|
||||
* that is returned must be a network path of the form /foo/rack,
|
||||
* where / is the root, and 'foo' is the switch where 'rack' is connected.
|
||||
* Note the hostname/ip-address is not part of the returned path.
|
||||
* The network topology of the cluster would determine the number of
|
||||
* components in the network path. Unlike 'resolve', names must be
|
||||
* resolvable
|
||||
* @param names
|
||||
* @return list of resolved network paths
|
||||
* @throws UnknownHostException if any hosts are not resolvable
|
||||
*/
|
||||
public List<String> resolveValidHosts(List<String> names)
|
||||
throws UnknownHostException;
|
||||
|
||||
}
|
||||
|
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.net;
|
|||
|
||||
import java.util.*;
|
||||
import java.io.*;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
@ -123,6 +124,17 @@ implements Configurable
|
|||
return m;
|
||||
}
|
||||
|
||||
public List<String> resolveValidHosts(List<String> names)
|
||||
throws UnknownHostException {
|
||||
List<String> result = this.resolve(names);
|
||||
if (result != null) {
|
||||
return result;
|
||||
} else {
|
||||
throw new UnknownHostException(
|
||||
"Unknown host(s) returned from ScriptBasedMapping");
|
||||
}
|
||||
}
|
||||
|
||||
private String runResolveCommand(List<String> args) {
|
||||
int loopCount = 0;
|
||||
if (args.size() == 0) {
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
package org.apache.hadoop.net;
|
||||
|
||||
import java.util.*;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.conf.Configured;
|
||||
|
@ -59,4 +60,19 @@ public class StaticMapping extends Configured implements DNSToSwitchMapping {
|
|||
return m;
|
||||
}
|
||||
}
|
||||
public List<String> resolveValidHosts(List<String> names)
|
||||
throws UnknownHostException {
|
||||
List<String> m = new ArrayList<String>();
|
||||
synchronized (nameToRackMap) {
|
||||
for (String name : names) {
|
||||
String rackId;
|
||||
if ((rackId = nameToRackMap.get(name)) != null) {
|
||||
m.add(rackId);
|
||||
} else {
|
||||
throw new UnknownHostException(name);
|
||||
}
|
||||
}
|
||||
return m;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.hadoop.net;
|
|||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
||||
|
@ -26,21 +27,59 @@ import junit.framework.TestCase;
|
|||
|
||||
public class TestScriptBasedMapping extends TestCase {
|
||||
|
||||
public void testNoArgsMeansNoResult() {
|
||||
ScriptBasedMapping mapping = new ScriptBasedMapping();
|
||||
private ScriptBasedMapping mapping;
|
||||
private Configuration conf;
|
||||
private List<String> names;
|
||||
|
||||
public TestScriptBasedMapping() {
|
||||
mapping = new ScriptBasedMapping();
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf = new Configuration();
|
||||
conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
|
||||
ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1);
|
||||
conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
|
||||
|
||||
mapping.setConf(conf);
|
||||
mapping.setConf(conf);
|
||||
}
|
||||
|
||||
List<String> names = new ArrayList<String>();
|
||||
public void testNoArgsMeansNoResult() {
|
||||
names = new ArrayList<String>();
|
||||
names.add("some.machine.name");
|
||||
names.add("other.machine.name");
|
||||
|
||||
List<String> result = mapping.resolve(names);
|
||||
assertNull(result);
|
||||
}
|
||||
|
||||
public void testResolveValidInvalidHostException() {
|
||||
names = new ArrayList<String>();
|
||||
names.add("1.com"); // Add invalid hostname that doesn't resolve
|
||||
boolean exceptionThrown = false;
|
||||
try {
|
||||
mapping.resolveValidHosts(names);
|
||||
} catch (UnknownHostException e) {
|
||||
exceptionThrown = true;
|
||||
}
|
||||
assertTrue(
|
||||
"resolveValidHosts did not throw UnknownHostException for invalid host",
|
||||
exceptionThrown);
|
||||
}
|
||||
|
||||
public void testResolveValidHostNoException() {
|
||||
conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
|
||||
ScriptBasedMapping.MIN_ALLOWABLE_ARGS);
|
||||
conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "echo");
|
||||
mapping.setConf(conf);
|
||||
|
||||
names = new ArrayList<String>();
|
||||
names.add("some.machine.name");
|
||||
names.add("other.machine.name");
|
||||
|
||||
boolean exceptionThrown = false;
|
||||
try {
|
||||
mapping.resolveValidHosts(names);
|
||||
} catch (UnknownHostException e) {
|
||||
exceptionThrown = true;
|
||||
}
|
||||
assertFalse("resolveValidHosts threw Exception for valid host", exceptionThrown);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue