Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4958. The file localization process should allow for wildcards to reduce the application footprint in the state store (Daniel Templeton via sjlee) 

(cherry picked from commit 5107a967fa)
This commit is contained in:
Sangjin Lee 2016-06-20 09:56:53 -07:00
parent 39ea0891d2
commit 2a79910c22
1 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
View File

@ -61,7 +61,7 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
public abstract class ContainerExecutor implements Configurable {
private static final String WILDCARD = "*";
private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
FsPermission.createImmutable((short) 0700);
@ -281,7 +281,18 @@ public abstract class ContainerExecutor implements Configurable {
if (resources != null) {
for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
for (String linkName : entry.getValue()) {
sb.symlink(entry.getKey(), new Path(linkName));
if (new Path(linkName).getName().equals(WILDCARD)) {
// If this is a wildcarded path, link to everything in the
// directory from the working directory
File directory = new File(entry.getKey().toString());
for (File wildLink : directory.listFiles()) {
sb.symlink(new Path(wildLink.toString()),
new Path(wildLink.getName()));
}
} else {
sb.symlink(entry.getKey(), new Path(linkName));
}
}
}
}