Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1926. Changed DistributedShell to use appIDs as unique identifiers for HDFS paths and thus avoid test failures on Windows. Contributed by Varun Vasudev. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1586562 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-04-11 04:06:40 +00:00
parent 8d41b363b8
commit d6a6e982a8
3 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/CHANGES.txt
View File

@ -111,6 +111,10 @@ Release 2.4.1 - UNRELEASED
when Application(Attempt) state hasn't been stored before. (Jian He via when Application(Attempt) state hasn't been stored before. (Jian He via
zjshen) zjshen)
YARN-1926. Changed DistributedShell to use appIDs as unique identifiers for
HDFS paths and thus avoid test failures on Windows. (Varun Vasudev via
vinodkv)
Release 2.4.0 - 2014-04-07 Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
View File

@ -468,12 +468,12 @@ public class Client {
// Copy the application master jar to the filesystem // Copy the application master jar to the filesystem
// Create a local resource to point to the destination jar path // Create a local resource to point to the destination jar path
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(),
localResources, null); localResources, null);
// Set the log4j properties if needed // Set the log4j properties if needed
if (!log4jPropFile.isEmpty()) { if (!log4jPropFile.isEmpty()) {
addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(),
localResources, null); localResources, null);
} }
@ -489,7 +489,7 @@ public class Client {
if (!shellScriptPath.isEmpty()) { if (!shellScriptPath.isEmpty()) {
Path shellSrc = new Path(shellScriptPath); Path shellSrc = new Path(shellScriptPath);
String shellPathSuffix = String shellPathSuffix =
appName + "/" + appId.getId() + "/" + SCRIPT_PATH; appName + "/" + appId.toString() + "/" + SCRIPT_PATH;
Path shellDst = Path shellDst =
new Path(fs.getHomeDirectory(), shellPathSuffix); new Path(fs.getHomeDirectory(), shellPathSuffix);
fs.copyFromLocalFile(false, true, shellSrc, shellDst); fs.copyFromLocalFile(false, true, shellSrc, shellDst);
@ -500,12 +500,12 @@ public class Client {
} }
if (!shellCommand.isEmpty()) { if (!shellCommand.isEmpty()) {
addToLocalResources(fs, null, shellCommandPath, appId.getId(), addToLocalResources(fs, null, shellCommandPath, appId.toString(),
localResources, shellCommand); localResources, shellCommand);
} }
if (shellArgs.length > 0) { if (shellArgs.length > 0) {
addToLocalResources(fs, null, shellArgsPath, appId.getId(), addToLocalResources(fs, null, shellArgsPath, appId.toString(),
localResources, StringUtils.join(shellArgs, " ")); localResources, StringUtils.join(shellArgs, " "));
} }
// Set local resource info into app master container launch context // Set local resource info into app master container launch context
@ -737,7 +737,7 @@ public class Client {
} }
private void addToLocalResources(FileSystem fs, String fileSrcPath, private void addToLocalResources(FileSystem fs, String fileSrcPath,
String fileDstPath, int appId, Map<String, LocalResource> localResources, String fileDstPath, String appId, Map<String, LocalResource> localResources,
String resources) throws IOException { String resources) throws IOException {
String suffix = String suffix =
appName + "/" + appId + "/" + fileDstPath; appName + "/" + appId + "/" + fileDstPath;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
View File

@ -219,7 +219,7 @@ public class TestDistributedShell {
"--num_containers", "--num_containers",
"1", "1",
"--shell_command", "--shell_command",
Shell.WINDOWS ? "timeout 8" : "sleep 8", "sleep 8",
"--master_memory", "--master_memory",
"512", "512",
"--container_memory", "--container_memory",