From 5107a967fa2558deba11c33a326d4d2e5748f452 Mon Sep 17 00:00:00 2001 From: Sangjin Lee Date: Mon, 20 Jun 2016 09:56:53 -0700 Subject: [PATCH] YARN-4958. The file localization process should allow for wildcards to reduce the application footprint in the state store (Daniel Templeton via sjlee) --- .../server/nodemanager/ContainerExecutor.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 98171af2768..98d45f41cc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -61,7 +61,7 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; public abstract class ContainerExecutor implements Configurable { - + private static final String WILDCARD = "*"; private static final Log LOG = LogFactory.getLog(ContainerExecutor.class); final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION = FsPermission.createImmutable((short) 0700); @@ -281,7 +281,18 @@ public abstract class ContainerExecutor implements Configurable { if (resources != null) { for (Map.Entry> entry : resources.entrySet()) { for (String linkName : entry.getValue()) { - sb.symlink(entry.getKey(), new Path(linkName)); + if (new Path(linkName).getName().equals(WILDCARD)) { + // If this is a wildcarded path, link to everything in the + // directory from the working directory + File directory = new File(entry.getKey().toString()); + + for (File wildLink : directory.listFiles()) { + sb.symlink(new Path(wildLink.toString()), + new Path(wildLink.getName())); + } + } else { + sb.symlink(entry.getKey(), new Path(linkName)); + } } } }