HADOOP-7314. Add support for throwing UnknownHostException when a host doesn't resolve. Contributed by Jeffrey Naisbitt.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1147317 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jitendra Nath Pandey 2011-07-15 20:58:00 +00:00
parent f582aa4492
commit 31f1a674e5
6 changed files with 167 additions and 25 deletions

View File

@ -266,6 +266,9 @@ Trunk (unreleased changes)
HADOOP-7305. Eclipse project files are incomplete. (Niels Basjes via eli)
HADOOP-7314. Add support for throwing UnknownHostException when a host doesn't
resolve. (Jeffrey Naisbitt via jitendra)
OPTIMIZATIONS
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole

View File

@ -17,6 +17,9 @@
*/
package org.apache.hadoop.net;
import java.net.InetAddress;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -42,16 +45,11 @@ public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
this.rawMapping = rawMapping;
}
public List<String> resolve(List<String> names) {
// normalize all input names to be in the form of IP addresses
names = NetUtils.normalizeHostNames(names);
List <String> result = new ArrayList<String>(names.size());
if (names.isEmpty()) {
return result;
}
/**
* Returns the hosts from 'names' that have not been cached previously
*/
private List<String> getUncachedHosts(List<String> names) {
// find out all names without cached resolved location
List<String> unCachedHosts = new ArrayList<String>(names.size());
for (String name : names) {
@ -59,27 +57,81 @@ public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
unCachedHosts.add(name);
}
}
// Resolve those names
List<String> rNames = rawMapping.resolve(unCachedHosts);
return unCachedHosts;
}
/**
* Caches the resolved hosts
*/
private void cacheResolvedHosts(List<String> uncachedHosts,
List<String> resolvedHosts) {
// Cache the result
if (rNames != null) {
for (int i=0; i<unCachedHosts.size(); i++) {
cache.put(unCachedHosts.get(i), rNames.get(i));
if (resolvedHosts != null) {
for (int i=0; i<uncachedHosts.size(); i++) {
cache.put(uncachedHosts.get(i), resolvedHosts.get(i));
}
}
}
/**
* Returns the cached resolution of the list of hostnames/addresses.
* Returns null if any of the names are not currently in the cache
*/
private List<String> getCachedHosts(List<String> names) {
List<String> result = new ArrayList<String>(names.size());
// Construct the result
for (String name : names) {
//now everything is in the cache
String networkLocation = cache.get(name);
if (networkLocation != null) {
result.add(networkLocation);
} else { //resolve all or nothing
} else {
return null;
}
}
return result;
}
/**
* Resolves host names and adds them to the cache.
* Unlike the 'resolve" method, this won't hide UnknownHostExceptions
*
* @param names to resolve
* @return List of resolved names
* @throws UnknownHostException if any hosts cannot be resolved
*/
public List<String> resolveValidHosts(List<String> names)
throws UnknownHostException {
if (names.isEmpty()) {
return new ArrayList<String>();
}
List<String> addresses = new ArrayList<String>(names.size());
for (String name : names) {
addresses.add(InetAddress.getByName(name).getHostAddress());
}
List<String> uncachedHosts = this.getUncachedHosts(names);
// Resolve the uncached hosts
List<String> resolvedHosts = rawMapping.resolveValidHosts(uncachedHosts);
this.cacheResolvedHosts(uncachedHosts, resolvedHosts);
return this.getCachedHosts(addresses);
}
public List<String> resolve(List<String> names) {
// normalize all input names to be in the form of IP addresses
names = NetUtils.normalizeHostNames(names);
List <String> result = new ArrayList<String>(names.size());
if (names.isEmpty()) {
return result;
}
List<String> uncachedHosts = this.getUncachedHosts(names);
// Resolve the uncached hosts
List<String> resolvedHosts = rawMapping.resolve(uncachedHosts);
this.cacheResolvedHosts(uncachedHosts, resolvedHosts);
return this.getCachedHosts(names);
}
}

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.net;
import java.util.List;
import java.net.UnknownHostException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -44,4 +45,23 @@ public interface DNSToSwitchMapping {
* @return list of resolved network paths
*/
public List<String> resolve(List<String> names);
/**
* Resolves a list of DNS-names/IP-addresses and returns back a list of
* switch information (network paths). One-to-one correspondence must be
* maintained between the elements in the lists.
* Consider an element in the argument list - x.y.com. The switch information
* that is returned must be a network path of the form /foo/rack,
* where / is the root, and 'foo' is the switch where 'rack' is connected.
* Note the hostname/ip-address is not part of the returned path.
* The network topology of the cluster would determine the number of
* components in the network path. Unlike 'resolve', names must be
* resolvable
* @param names
* @return list of resolved network paths
* @throws UnknownHostException if any hosts are not resolvable
*/
public List<String> resolveValidHosts(List<String> names)
throws UnknownHostException;
}

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.net;
import java.util.*;
import java.io.*;
import java.net.UnknownHostException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -123,6 +124,17 @@ implements Configurable
return m;
}
public List<String> resolveValidHosts(List<String> names)
throws UnknownHostException {
List<String> result = this.resolve(names);
if (result != null) {
return result;
} else {
throw new UnknownHostException(
"Unknown host(s) returned from ScriptBasedMapping");
}
}
private String runResolveCommand(List<String> args) {
int loopCount = 0;
if (args.size() == 0) {

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.net;
import java.util.*;
import java.net.UnknownHostException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@ -59,4 +60,19 @@ public class StaticMapping extends Configured implements DNSToSwitchMapping {
return m;
}
}
public List<String> resolveValidHosts(List<String> names)
throws UnknownHostException {
List<String> m = new ArrayList<String>();
synchronized (nameToRackMap) {
for (String name : names) {
String rackId;
if ((rackId = nameToRackMap.get(name)) != null) {
m.add(rackId);
} else {
throw new UnknownHostException(name);
}
}
return m;
}
}
}

View File

@ -19,6 +19,7 @@ package org.apache.hadoop.net;
import java.util.ArrayList;
import java.util.List;
import java.net.UnknownHostException;
import org.apache.hadoop.conf.Configuration;
@ -26,21 +27,59 @@ import junit.framework.TestCase;
public class TestScriptBasedMapping extends TestCase {
public void testNoArgsMeansNoResult() {
ScriptBasedMapping mapping = new ScriptBasedMapping();
private ScriptBasedMapping mapping;
private Configuration conf;
private List<String> names;
public TestScriptBasedMapping() {
mapping = new ScriptBasedMapping();
Configuration conf = new Configuration();
conf = new Configuration();
conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1);
conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
mapping.setConf(conf);
mapping.setConf(conf);
}
List<String> names = new ArrayList<String>();
public void testNoArgsMeansNoResult() {
names = new ArrayList<String>();
names.add("some.machine.name");
names.add("other.machine.name");
List<String> result = mapping.resolve(names);
assertNull(result);
}
public void testResolveValidInvalidHostException() {
names = new ArrayList<String>();
names.add("1.com"); // Add invalid hostname that doesn't resolve
boolean exceptionThrown = false;
try {
mapping.resolveValidHosts(names);
} catch (UnknownHostException e) {
exceptionThrown = true;
}
assertTrue(
"resolveValidHosts did not throw UnknownHostException for invalid host",
exceptionThrown);
}
public void testResolveValidHostNoException() {
conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
ScriptBasedMapping.MIN_ALLOWABLE_ARGS);
conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "echo");
mapping.setConf(conf);
names = new ArrayList<String>();
names.add("some.machine.name");
names.add("other.machine.name");
boolean exceptionThrown = false;
try {
mapping.resolveValidHosts(names);
} catch (UnknownHostException e) {
exceptionThrown = true;
}
assertFalse("resolveValidHosts threw Exception for valid host", exceptionThrown);
}
}