From 5f68f640a5b17cfb0385462f63d86d4d109cec7e Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Tue, 23 Feb 2016 12:29:25 +0530 Subject: [PATCH] YARN-4709. NMWebServices produces incorrect JSON for containers. Contributed by Varun Saxena. (cherry picked from commit 140cb5d74565aad848b40a251ec02b597a3a20d4) (cherry picked from commit 2c218ca8a8da9bdea3ba052da10d87e5716c1d16) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../yarn/webapp/WebServicesTestUtils.java | 28 +++++++++++++ .../nodemanager/webapp/dao/ContainerInfo.java | 1 - .../webapp/TestNMWebServicesContainers.java | 39 ++++++++++++++++--- 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 430627a3a74..579c22eb38c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1217,6 +1217,9 @@ Release 2.8.0 - UNRELEASED YARN-4386. refreshNodesGracefully() should send recommission event to active RMNodes only. (Kuhu Shukla via junping_du) + YARN-4709. NMWebServices produces incorrect JSON for containers. + (Varun Saxena via vvasudev) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java index aa211a62f96..0454ce6832c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.webapp; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; +import java.util.List; + import org.w3c.dom.Attr; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -47,6 +50,31 @@ public class WebServicesTestUtils { return Float.parseFloat(val); } + public static List getXmlStrings(Element element, String name) { + NodeList id = element.getElementsByTagName(name); + List strings = new ArrayList<>(); + int len = id.getLength(); + if (id.getLength() == 0) { + return strings; + } + for (int i = 0; i < len; i++) { + Element line = (Element) id.item(i); + if (line == null) { + continue; + } + Node first = line.getFirstChild(); + if (first == null) { + continue; + } + String val = first.getNodeValue(); + if (val == null) { + continue; + } + strings.add(val); + } + return strings; + } + public static String getXmlString(Element element, String name) { NodeList id = element.getElementsByTagName(name); Element line = (Element) id.item(0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index e462eba4b80..cf022b985ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -53,7 +53,6 @@ public class ContainerInfo { @XmlTransient protected String exitStatus; - @XmlElementWrapper protected List containerLogFiles; public ContainerInfo() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 0ed56d3f0b1..3c4a660c88a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -20,13 +20,16 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; import static org.apache.hadoop.yarn.util.StringHelper.ujoin; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.io.StringReader; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; @@ -48,6 +51,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.ResourceView; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -203,12 +207,27 @@ public class TestNMWebServicesContainers extends JerseyTestBase { app.getAppId(), 1); Container container1 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 1); + ((MockContainer)container1).setState(ContainerState.RUNNING); Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2); + ((MockContainer)container2).setState(ContainerState.RUNNING); nmContext.getContainers() .put(container1.getContainerId(), container1); nmContext.getContainers() .put(container2.getContainerId(), container2); + File appDir = new File(testLogDir + "/" + app.getAppId().toString()); + appDir.mkdir(); + File container1Dir = + new File(appDir + "/" + container1.getContainerId().toString()); + container1Dir.mkdir(); + // Create log files for containers. + new File(container1Dir + "/" + "syslog").createNewFile(); + new File(container1Dir + "/" + "stdout").createNewFile(); + File container2Dir = + new File(appDir + "/" + container2.getContainerId().toString()); + container2Dir.mkdir(); + new File(container2Dir + "/" + "syslog").createNewFile(); + new File(container2Dir + "/" + "stdout").createNewFile(); app.getContainers().put(container1.getContainerId(), container1); app.getContainers().put(container2.getContainerId(), container2); @@ -475,9 +494,13 @@ public class TestNMWebServicesContainers extends JerseyTestBase { WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"), WebServicesTestUtils.getXmlString(element, "containerLogsLink")); // verify that the container log files element exists - assertTrue("containerLogFiles missing", - WebServicesTestUtils.getXmlString(element, "containerLogFiles") - != null); + List containerLogFiles = + WebServicesTestUtils.getXmlStrings(element, "containerLogFiles"); + assertFalse("containerLogFiles missing",containerLogFiles.isEmpty()); + assertEquals(2, containerLogFiles.size()); + assertTrue("syslog and stdout expected", + containerLogFiles.contains("syslog") && + containerLogFiles.contains("stdout")); } } @@ -492,8 +515,14 @@ public class TestNMWebServicesContainers extends JerseyTestBase { info.getInt("totalVCoresNeeded"), info.getString("containerLogsLink")); // verify that the container log files element exists - assertTrue("containerLogFiles missing", - info.getJSONArray("containerLogFiles") != null); + JSONArray containerLogFilesArr = info.getJSONArray("containerLogFiles"); + assertTrue("containerLogFiles missing", containerLogFilesArr != null); + assertEquals(2, containerLogFilesArr.length()); + for (int i = 0; i < 2; i++) { + assertTrue("syslog and stdout expected", + containerLogFilesArr.get(i).equals("syslog") || + containerLogFilesArr.get(i).equals("stdout")); + } } public void verifyNodeContainerInfoGeneric(Container cont, String id,