From 2e6f49f4e4cdbd708eb172e6e1ebeff944b52097 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 6 Jul 2012 17:20:45 +0000 Subject: [PATCH] svn merge -c 1358305 FIXES: MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1358308 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/fs/LocalDirAllocator.java | 13 +++++++++++++ .../org/apache/hadoop/fs/TestLocalDirAllocator.java | 13 +++++++++++++ hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../localizer/ContainerLocalizer.java | 12 +++++++----- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java index dab7ab4e9c9..3e305930fdd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java @@ -207,6 +207,19 @@ public static boolean isContextValid(String contextCfgItemName) { return contexts.containsKey(contextCfgItemName); } } + + /** + * Removes the context from the context config items + * + * @param contextCfgItemName + */ + @Deprecated + @InterfaceAudience.LimitedPrivate({"MapReduce"}) + public static void removeContext(String contextCfgItemName) { + synchronized (contexts) { + contexts.remove(contextCfgItemName); + } + } /** We search through all the configured dirs for the file's existence * and return true when we find diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index ad319709bfa..51e57dba81e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -339,5 +339,18 @@ public void testGetLocalPathToRead() throws IOException { } } + + @Test + public void testRemoveContext() throws IOException { + String dir = buildBufferDir(ROOT, 0); + String contextCfgItemName = "application_1340842292563_0004.app.cache.dirs"; + conf.set(contextCfgItemName, dir); + LocalDirAllocator localDirAllocator = new LocalDirAllocator( + contextCfgItemName); + localDirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf); + assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); + LocalDirAllocator.removeContext(contextCfgItemName); + assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); + } } diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 1a5f7d2b016..5ca9c5b696a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -519,6 +519,10 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4387. RM gets fatal error and exits during TestRM (Kihwal Lee via tgraves) + MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap + space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K + via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 0da83d91c70..f57739d0211 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -91,6 +91,7 @@ public class ContainerLocalizer { private final LocalDirAllocator userDirs; private final RecordFactory recordFactory; private final Map> pendingResources; + private final String appCacheDirContextName; public ContainerLocalizer(FileContext lfs, String user, String appId, String localizerId, List localDirs, @@ -108,10 +109,9 @@ public ContainerLocalizer(FileContext lfs, String user, String appId, this.localizerId = localizerId; this.recordFactory = recordFactory; this.conf = new Configuration(); - this.appDirs = - new LocalDirAllocator(String.format(APPCACHE_CTXT_FMT, appId)); - this.userDirs = - new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, appId)); + this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId); + this.appDirs = new LocalDirAllocator(appCacheDirContextName); + this.userDirs = new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, user)); this.pendingResources = new HashMap>(); } @@ -121,6 +121,7 @@ LocalizationProtocol getProxy(final InetSocketAddress nmAddr) { rpc.getProxy(LocalizationProtocol.class, nmAddr, conf); } + @SuppressWarnings("deprecation") public int runLocalization(final InetSocketAddress nmAddr) throws IOException, InterruptedException { // load credentials @@ -177,6 +178,7 @@ public LocalizationProtocol run() { if (exec != null) { exec.shutdownNow(); } + LocalDirAllocator.removeContext(appCacheDirContextName); } } @@ -373,7 +375,7 @@ private static void initDirs(Configuration conf, String user, String appId, lfs.mkdir(appFileCacheDir, null, false); } conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs); - conf.setStrings(String.format(USERCACHE_CTXT_FMT, appId), usersFileCacheDirs); + conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs); } }