svn merge -c 1358305 FIXES: MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K via bobby)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1358308 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
ca23ea7cf5
commit
2e6f49f4e4
|
@ -208,6 +208,19 @@ public class LocalDirAllocator {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Removes the context from the context config items
|
||||||
|
*
|
||||||
|
* @param contextCfgItemName
|
||||||
|
*/
|
||||||
|
@Deprecated
|
||||||
|
@InterfaceAudience.LimitedPrivate({"MapReduce"})
|
||||||
|
public static void removeContext(String contextCfgItemName) {
|
||||||
|
synchronized (contexts) {
|
||||||
|
contexts.remove(contextCfgItemName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** We search through all the configured dirs for the file's existence
|
/** We search through all the configured dirs for the file's existence
|
||||||
* and return true when we find
|
* and return true when we find
|
||||||
* @param pathStr the requested file (this will be searched)
|
* @param pathStr the requested file (this will be searched)
|
||||||
|
|
|
@ -340,4 +340,17 @@ public class TestLocalDirAllocator {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRemoveContext() throws IOException {
|
||||||
|
String dir = buildBufferDir(ROOT, 0);
|
||||||
|
String contextCfgItemName = "application_1340842292563_0004.app.cache.dirs";
|
||||||
|
conf.set(contextCfgItemName, dir);
|
||||||
|
LocalDirAllocator localDirAllocator = new LocalDirAllocator(
|
||||||
|
contextCfgItemName);
|
||||||
|
localDirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf);
|
||||||
|
assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName));
|
||||||
|
LocalDirAllocator.removeContext(contextCfgItemName);
|
||||||
|
assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName));
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -519,6 +519,10 @@ Release 0.23.3 - UNRELEASED
|
||||||
MAPREDUCE-4387. RM gets fatal error and exits during TestRM
|
MAPREDUCE-4387. RM gets fatal error and exits during TestRM
|
||||||
(Kihwal Lee via tgraves)
|
(Kihwal Lee via tgraves)
|
||||||
|
|
||||||
|
MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap
|
||||||
|
space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K
|
||||||
|
via bobby)
|
||||||
|
|
||||||
Release 0.23.2 - UNRELEASED
|
Release 0.23.2 - UNRELEASED
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
|
@ -91,6 +91,7 @@ public class ContainerLocalizer {
|
||||||
private final LocalDirAllocator userDirs;
|
private final LocalDirAllocator userDirs;
|
||||||
private final RecordFactory recordFactory;
|
private final RecordFactory recordFactory;
|
||||||
private final Map<LocalResource,Future<Path>> pendingResources;
|
private final Map<LocalResource,Future<Path>> pendingResources;
|
||||||
|
private final String appCacheDirContextName;
|
||||||
|
|
||||||
public ContainerLocalizer(FileContext lfs, String user, String appId,
|
public ContainerLocalizer(FileContext lfs, String user, String appId,
|
||||||
String localizerId, List<Path> localDirs,
|
String localizerId, List<Path> localDirs,
|
||||||
|
@ -108,10 +109,9 @@ public class ContainerLocalizer {
|
||||||
this.localizerId = localizerId;
|
this.localizerId = localizerId;
|
||||||
this.recordFactory = recordFactory;
|
this.recordFactory = recordFactory;
|
||||||
this.conf = new Configuration();
|
this.conf = new Configuration();
|
||||||
this.appDirs =
|
this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
|
||||||
new LocalDirAllocator(String.format(APPCACHE_CTXT_FMT, appId));
|
this.appDirs = new LocalDirAllocator(appCacheDirContextName);
|
||||||
this.userDirs =
|
this.userDirs = new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, user));
|
||||||
new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, appId));
|
|
||||||
this.pendingResources = new HashMap<LocalResource,Future<Path>>();
|
this.pendingResources = new HashMap<LocalResource,Future<Path>>();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -121,6 +121,7 @@ public class ContainerLocalizer {
|
||||||
rpc.getProxy(LocalizationProtocol.class, nmAddr, conf);
|
rpc.getProxy(LocalizationProtocol.class, nmAddr, conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public int runLocalization(final InetSocketAddress nmAddr)
|
public int runLocalization(final InetSocketAddress nmAddr)
|
||||||
throws IOException, InterruptedException {
|
throws IOException, InterruptedException {
|
||||||
// load credentials
|
// load credentials
|
||||||
|
@ -177,6 +178,7 @@ public class ContainerLocalizer {
|
||||||
if (exec != null) {
|
if (exec != null) {
|
||||||
exec.shutdownNow();
|
exec.shutdownNow();
|
||||||
}
|
}
|
||||||
|
LocalDirAllocator.removeContext(appCacheDirContextName);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -373,7 +375,7 @@ public class ContainerLocalizer {
|
||||||
lfs.mkdir(appFileCacheDir, null, false);
|
lfs.mkdir(appFileCacheDir, null, false);
|
||||||
}
|
}
|
||||||
conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs);
|
conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs);
|
||||||
conf.setStrings(String.format(USERCACHE_CTXT_FMT, appId), usersFileCacheDirs);
|
conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue