Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4958. The file localization process should allow for wildcards to reduce the application footprint in the state store (Daniel Templeton via sjlee)

This commit is contained in:
Sangjin Lee 2016-06-20 09:56:53 -07:00
parent fc6b50cc57
commit 5107a967fa
1 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
View File

@ -61,7 +61,7 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
public abstract class ContainerExecutor implements Configurable {
private static final String WILDCARD = "*";
private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
FsPermission.createImmutable((short) 0700);
@ -281,10 +281,21 @@ public abstract class ContainerExecutor implements Configurable {
if (resources != null) {
for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
for (String linkName : entry.getValue()) {
if (new Path(linkName).getName().equals(WILDCARD)) {
// If this is a wildcarded path, link to everything in the
// directory from the working directory
File directory = new File(entry.getKey().toString());
for (File wildLink : directory.listFiles()) {
sb.symlink(new Path(wildLink.toString()),
new Path(wildLink.getName()));
}
} else {
sb.symlink(entry.getKey(), new Path(linkName));
}
}
}
}
// dump debugging information if configured
if (getConf() != null && getConf().getBoolean(