From 1884c3a0b973efd9d841ef73717535fb7e50a405 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Wed, 13 Nov 2013 03:26:20 +0000 Subject: [PATCH] YARN-1386. NodeManager mistakenly loses resources and relocalizes them (Jason Lowe via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1541376 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../localizer/ContainerLocalizer.java | 14 ++++++++++++-- .../localizer/TestContainerLocalizer.java | 5 +++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3fac4d29939..ec88e25b7a7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1621,6 +1621,9 @@ Release 0.23.10 - UNRELEASED YARN-1176. RM web services ClusterMetricsInfo total nodes doesn't include unhealthy nodes (Jonathan Eagles via tgraves) + YARN-1386. NodeManager mistakenly loses resources and relocalizes them + (Jason Lowe via jeagles) + Release 0.23.9 - 2013-07-08 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 51ce9e2fb19..f05f49cd4f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -82,6 +83,8 @@ public class ContainerLocalizer { public static final String WORKDIR = "work"; private static final String APPCACHE_CTXT_FMT = "%s.app.cache.dirs"; private static final String USERCACHE_CTXT_FMT = "%s.user.cache.dirs"; + private static final FsPermission FILECACHE_PERMS = + new FsPermission((short)0710); private final String user; private final String appId; @@ -363,16 +366,23 @@ public class ContainerLocalizer { // $x/usercache/$user/filecache Path userFileCacheDir = new Path(base, FILECACHE); usersFileCacheDirs[i] = userFileCacheDir.toString(); - lfs.mkdir(userFileCacheDir, null, false); + createDir(lfs, userFileCacheDir, FILECACHE_PERMS, false); // $x/usercache/$user/appcache/$appId Path appBase = new Path(base, new Path(APPCACHE, appId)); // $x/usercache/$user/appcache/$appId/filecache Path appFileCacheDir = new Path(appBase, FILECACHE); appsFileCacheDirs[i] = appFileCacheDir.toString(); - lfs.mkdir(appFileCacheDir, null, false); + createDir(lfs, appFileCacheDir, FILECACHE_PERMS, false); } conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs); conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs); } + private static void createDir(FileContext lfs, Path dirPath, + FsPermission perms, boolean createParent) throws IOException { + lfs.mkdir(dirPath, perms, createParent); + if (!perms.equals(perms.applyUMask(lfs.getUMask()))) { + lfs.setPermission(dirPath, perms); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java index 10da1bee13d..22fad6f0c80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java @@ -83,6 +83,7 @@ public class TestContainerLocalizer { static final Log LOG = LogFactory.getLog(TestContainerLocalizer.class); static final Path basedir = new Path("target", TestContainerLocalizer.class.getName()); + static final FsPermission CACHE_DIR_PERM = new FsPermission((short)0710); static final String appUser = "yak"; static final String appId = "app_RM_0"; @@ -171,12 +172,12 @@ public class TestContainerLocalizer { Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser); Path privcache = new Path(base, ContainerLocalizer.FILECACHE); // $x/usercache/$user/filecache - verify(spylfs).mkdir(eq(privcache), isA(FsPermission.class), eq(false)); + verify(spylfs).mkdir(eq(privcache), eq(CACHE_DIR_PERM), eq(false)); Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId)); // $x/usercache/$user/appcache/$appId/filecache Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE); - verify(spylfs).mkdir(eq(appcache), isA(FsPermission.class), eq(false)); + verify(spylfs).mkdir(eq(appcache), eq(CACHE_DIR_PERM), eq(false)); } // verify tokens read at expected location verify(spylfs).open(tokenPath);