From 83e907c32c9abbdeafd6a0df30b1698db0dc8b7c Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 21 Nov 2013 04:11:22 +0000 Subject: [PATCH] YARN-1303. Reverted the wrong patch committed earlier and committing the correct patch now. In one go. svn merge --ignore-ancestry -c 1544029 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1544030 13f79535-47bb-0310-9956-ffa450edef68 --- .../distributedshell/ApplicationMaster.java | 11 ++++---- .../applications/distributedshell/Client.java | 19 +++++-------- .../TestDistributedShell.java | 27 ++++++++++++------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 6cf3e2dfa18..ac0c6cd343b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.applications.distributedshell; import java.io.BufferedReader; import java.io.DataInputStream; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.StringReader; @@ -219,6 +220,8 @@ public class ApplicationMaster { // Hardcoded path to shell script in launch container's local env private final String ExecShellStringPath = "ExecShellScript.sh"; + private final String shellCommandPath = "shellCommands"; + private volatile boolean done; private volatile boolean success; @@ -301,8 +304,6 @@ public class ApplicationMaster { Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); - opts.addOption("shell_command", true, - "Shell command to be executed by the Application Master"); opts.addOption("shell_script", true, "Location of the shell script to be executed"); opts.addOption("shell_args", true, "Command line args for the shell script"); @@ -373,15 +374,15 @@ public class ApplicationMaster { + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); - if (!cliParser.hasOption("shell_command")) { + File shellCommandFile = new File(shellCommandPath); + if (!shellCommandFile.exists()) { throw new IllegalArgumentException( "No shell command specified to be executed by application master"); } - String shellCommandPath = cliParser.getOptionValue("shell_command"); FileInputStream fs = null; DataInputStream ds = null; try { - ds = new DataInputStream(new FileInputStream(shellCommandPath)); + ds = new DataInputStream(new FileInputStream(shellCommandFile)); shellCommand = ds.readUTF(); } finally { org.apache.commons.io.IOUtils.closeQuietly(ds); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index b40a0f936d4..d1ac2fea3ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -134,7 +134,6 @@ public class Client { // Shell command to be executed private String shellCommand = ""; - private final String shellCommandPath = "shellCommands"; // Location of shell script private String shellScriptPath = ""; // Args to be passed to the shell command @@ -166,6 +165,7 @@ public class Client { // Command line options private Options opts; + private final String shellCommandPath = "shellCommands"; /** * @param args Command line arguments */ @@ -501,14 +501,12 @@ public class Client { IOUtils.closeQuietly(ostream); } FileStatus scFileStatus = fs.getFileStatus(shellCommandDst); - LocalResource scRsrc = Records.newRecord(LocalResource.class); - scRsrc.setType(LocalResourceType.FILE); - scRsrc.setVisibility(LocalResourceVisibility.APPLICATION); - scRsrc.setResource(ConverterUtils.getYarnUrlFromURI(shellCommandDst - .toUri())); - scRsrc.setTimestamp(scFileStatus.getModificationTime()); - scRsrc.setSize(scFileStatus.getLen()); - localResources.put("shellCommands", scRsrc); + LocalResource scRsrc = + LocalResource.newInstance( + ConverterUtils.getYarnUrlFromURI(shellCommandDst.toUri()), + LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, + scFileStatus.getLen(), scFileStatus.getModificationTime()); + localResources.put(shellCommandPath, scRsrc); } // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); @@ -569,9 +567,6 @@ public class Client { vargs.add("--num_containers " + String.valueOf(numContainers)); vargs.add("--priority " + String.valueOf(shellCmdPriority)); - if (!shellCommand.isEmpty()) { - vargs.add("--shell_command " + shellCommandPath + ""); - } if (!shellArgs.isEmpty()) { vargs.add("--shell_args " + shellArgs + ""); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 59148c70938..a5bbbb4ec2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -175,13 +175,14 @@ public class TestDistributedShell { @Test(timeout=90000) public void testDSShellWithCommands() throws Exception { + String[] args = { "--jar", APPMASTER_JAR, "--num_containers", "2", "--shell_command", - "echo HADOOP YARN MAPREDUCE|wc -w", + "\"echo output_ignored;echo output_expected\"", "--master_memory", "512", "--master_vcores", @@ -201,8 +202,8 @@ public class TestDistributedShell { boolean result = client.run(); LOG.info("Client run completed. Result=" + result); List expectedContent = new ArrayList(); - expectedContent.add("3"); - verifyContainerLog(2, expectedContent); + expectedContent.add("output_expected"); + verifyContainerLog(2, expectedContent, false, ""); } @Test(timeout=90000) @@ -368,8 +369,8 @@ public class TestDistributedShell { Assert.assertTrue(client.run()); } - private void - verifyContainerLog(int containerNum, List expectedContent) { + private int verifyContainerLog(int containerNum, + List expectedContent, boolean count, String expectedWord) { File logFolder = new File(yarnCluster.getNodeManager(0).getConfig() .get(YarnConfiguration.NM_LOG_DIRS, @@ -387,9 +388,10 @@ public class TestDistributedShell { File[] containerFiles = listOfFiles[currentContainerLogFileIndex].listFiles(); + int numOfWords = 0; for (int i = 0; i < containerFiles.length; i++) { for (File output : containerFiles[i].listFiles()) { - if (output.getName().trim().equalsIgnoreCase("stdout")) { + if (output.getName().trim().contains("stdout")) { BufferedReader br = null; try { @@ -398,9 +400,15 @@ public class TestDistributedShell { br = new BufferedReader(new FileReader(output)); int numOfline = 0; while ((sCurrentLine = br.readLine()) != null) { - Assert.assertEquals("The current is" + sCurrentLine, - expectedContent.get(numOfline), sCurrentLine.trim()); - numOfline++; + if (count) { + if (sCurrentLine.contains(expectedWord)) { + numOfWords++; + } + } else if (output.getName().trim().equals("stdout")){ + Assert.assertEquals("The current is" + sCurrentLine, + expectedContent.get(numOfline), sCurrentLine.trim()); + numOfline++; + } } } catch (IOException e) { @@ -416,6 +424,7 @@ public class TestDistributedShell { } } } + return numOfWords; } }