From fa2a3c94f28b2a3c5e54ab88e89f73e65ba4ddcd Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 26 Sep 2013 15:24:40 +0000 Subject: [PATCH] HADOOP-9998. Provide methods to clear only part of the DNSToSwitchMapping. (Junping Du via Colin Patrick McCabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1526568 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/net/CachedDNSToSwitchMapping.java | 7 +++++++ .../java/org/apache/hadoop/net/DNSToSwitchMapping.java | 8 ++++++++ .../java/org/apache/hadoop/net/ScriptBasedMapping.java | 6 ++++++ .../src/main/java/org/apache/hadoop/net/TableMapping.java | 7 +++++++ .../test/java/org/apache/hadoop/net/StaticMapping.java | 6 ++++++ .../java/org/apache/hadoop/net/TestSwitchMapping.java | 4 ++++ .../hdfs/server/blockmanagement/DatanodeManager.java | 7 ++++++- .../hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java | 4 ++++ .../client/api/impl/TestAMRMClientContainerRequest.java | 4 ++++ .../org/apache/hadoop/yarn/util/TestRackResolver.java | 4 ++++ 11 files changed, 59 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 462490b5859..ec522dc4d5e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -49,6 +49,9 @@ Release 2.3.0 - UNRELEASED HADOOP-9915. o.a.h.fs.Stat support on Mac OS X (Binglin Chang via Colin Patrick McCabe) + HADOOP-9998. Provide methods to clear only part of the DNSToSwitchMapping. + (Junping Du via Colin Patrick McCabe) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java index e152c549ffc..af487ed5c61 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java @@ -154,4 +154,11 @@ public class CachedDNSToSwitchMapping extends AbstractDNSToSwitchMapping { public void reloadCachedMappings() { cache.clear(); } + + @Override + public void reloadCachedMappings(List names) { + for (String name : names) { + cache.remove(name); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java index ccc109302fd..7b1b332b9b7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java @@ -59,4 +59,12 @@ public interface DNSToSwitchMapping { * will get a chance to see the new data. */ public void reloadCachedMappings(); + + /** + * Reload cached mappings on specific nodes. + * + * If there is a cache on these nodes, this method will clear it, so that + * future accesses will see updated data. + */ + public void reloadCachedMappings(List names); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java index c9e62e8580c..2d02e133618 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java @@ -269,5 +269,11 @@ public final class ScriptBasedMapping extends CachedDNSToSwitchMapping { // Nothing to do here, since RawScriptBasedMapping has no cache, and // does not inherit from CachedDNSToSwitchMapping } + + @Override + public void reloadCachedMappings(List names) { + // Nothing to do here, since RawScriptBasedMapping has no cache, and + // does not inherit from CachedDNSToSwitchMapping + } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java index fa96476ff6e..2662108124d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java @@ -162,5 +162,12 @@ public class TableMapping extends CachedDNSToSwitchMapping { } } } + + @Override + public void reloadCachedMappings(List names) { + // TableMapping has to reload all mappings at once, so no chance to + // reload mappings on specific nodes + reloadCachedMappings(); + } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java index 5492b47c8b7..493d86000ee 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java @@ -152,4 +152,10 @@ public class StaticMapping extends AbstractDNSToSwitchMapping { // reloadCachedMappings does nothing for StaticMapping; there is // nowhere to reload from since all data is in memory. } + + @Override + public void reloadCachedMappings(List names) { + // reloadCachedMappings does nothing for StaticMapping; there is + // nowhere to reload from since all data is in memory. + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSwitchMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSwitchMapping.java index 1caa454243f..b5de661caca 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSwitchMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSwitchMapping.java @@ -120,5 +120,9 @@ public class TestSwitchMapping extends Assert { @Override public void reloadCachedMappings() { } + + @Override + public void reloadCachedMappings(List names) { + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index a5e2f6ada77..c345208e3f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -886,7 +886,12 @@ public class DatanodeManager { // If the network location is invalid, clear the cached mappings // so that we have a chance to re-add this DataNode with the // correct network location later. - dnsToSwitchMapping.reloadCachedMappings(); + List invalidNodeNames = new ArrayList(3); + // clear cache for nodes in IP or Hostname + invalidNodeNames.add(nodeReg.getIpAddr()); + invalidNodeNames.add(nodeReg.getHostName()); + invalidNodeNames.add(nodeReg.getPeerHostName()); + dnsToSwitchMapping.reloadCachedMappings(invalidNodeNames); throw e; } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java index 87611ee99d9..599f6af49ab 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java @@ -99,6 +99,10 @@ public class TestJobHistoryParsing { @Override public void reloadCachedMappings() { } + + @Override + public void reloadCachedMappings(List names) { + } } @Test(timeout = 50000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java index 71ee9f9f5a7..cb8c86aaea3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java @@ -215,6 +215,10 @@ public class TestAMRMClientContainerRequest { @Override public void reloadCachedMappings() {} + + @Override + public void reloadCachedMappings(List names) { + } } private void verifyResourceRequest( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java index d3ccfcaa92a..42f7b9b5091 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java @@ -67,6 +67,10 @@ public class TestRackResolver { public void reloadCachedMappings() { // nothing to do here, since RawScriptBasedMapping has no cache. } + + @Override + public void reloadCachedMappings(List names) { + } } @Test