YARN-4709. NMWebServices produces incorrect JSON for containers. Contributed by Varun Saxena.

(cherry picked from commit 140cb5d745)
(cherry picked from commit 2c218ca8a8)

hadoop-yarn-project/CHANGES.txt

@@ -1217,6 +1217,9 @@ Release 2.8.0 - UNRELEASED
     YARN-4386. refreshNodesGracefully() should send recommission event to active
     RMNodes only. (Kuhu Shukla via junping_du)
 
+    YARN-4709. NMWebServices produces incorrect JSON for containers.
+    (Varun Saxena via vvasudev)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java

@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.webapp;
 
 import static org.junit.Assert.assertTrue;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.w3c.dom.Attr;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
@@ -47,6 +50,31 @@ public class WebServicesTestUtils {
     return Float.parseFloat(val);
   }
 
+  public static List<String> getXmlStrings(Element element, String name) {
+    NodeList id = element.getElementsByTagName(name);
+    List<String> strings = new ArrayList<>();
+    int len = id.getLength();
+    if (id.getLength() == 0) {
+      return strings;
+    }
+    for (int i = 0; i < len; i++) {
+      Element line = (Element) id.item(i);
+      if (line == null) {
+        continue;
+      }
+      Node first = line.getFirstChild();
+      if (first == null) {
+        continue;
+      }
+      String val = first.getNodeValue();
+      if (val == null) {
+        continue;
+      }
+      strings.add(val);
+    }
+    return strings;
+  }
+
   public static String getXmlString(Element element, String name) {
     NodeList id = element.getElementsByTagName(name);
     Element line = (Element) id.item(0);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java

@@ -53,7 +53,6 @@ public class ContainerInfo {
   @XmlTransient
   protected String exitStatus;
 
-  @XmlElementWrapper
   protected List<String> containerLogFiles;
 
   public ContainerInfo() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java

@@ -20,13 +20,16 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 
 import javax.ws.rs.core.MediaType;
 import javax.xml.parsers.DocumentBuilder;
@@ -48,6 +51,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
 import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -203,12 +207,27 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
         app.getAppId(), 1);
     Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
         app.getUser(), app.getAppId(), 1);
+    ((MockContainer)container1).setState(ContainerState.RUNNING);
     Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
         app.getUser(), app.getAppId(), 2);
+    ((MockContainer)container2).setState(ContainerState.RUNNING);
     nmContext.getContainers()
         .put(container1.getContainerId(), container1);
     nmContext.getContainers()
         .put(container2.getContainerId(), container2);
+    File appDir = new File(testLogDir + "/" + app.getAppId().toString());
+    appDir.mkdir();
+    File container1Dir =
+        new File(appDir + "/" + container1.getContainerId().toString());
+    container1Dir.mkdir();
+    // Create log files for containers.
+    new File(container1Dir + "/" + "syslog").createNewFile();
+    new File(container1Dir + "/" + "stdout").createNewFile();
+    File container2Dir =
+        new File(appDir + "/" + container2.getContainerId().toString());
+    container2Dir.mkdir();
+    new File(container2Dir + "/" + "syslog").createNewFile();
+    new File(container2Dir + "/" + "stdout").createNewFile();
 
     app.getContainers().put(container1.getContainerId(), container1);
     app.getContainers().put(container2.getContainerId(), container2);
@@ -475,9 +494,13 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
           WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
           WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
       // verify that the container log files element exists
-      assertTrue("containerLogFiles missing",
+      List<String> containerLogFiles =
-          WebServicesTestUtils.getXmlString(element, "containerLogFiles")
+          WebServicesTestUtils.getXmlStrings(element, "containerLogFiles");
-              != null);
+      assertFalse("containerLogFiles missing",containerLogFiles.isEmpty());
+      assertEquals(2, containerLogFiles.size());
+      assertTrue("syslog and stdout expected",
+          containerLogFiles.contains("syslog") &&
+          containerLogFiles.contains("stdout"));
     }
   }
 
@@ -492,8 +515,14 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
         info.getInt("totalVCoresNeeded"),
         info.getString("containerLogsLink"));
     // verify that the container log files element exists
-    assertTrue("containerLogFiles missing",
+    JSONArray containerLogFilesArr = info.getJSONArray("containerLogFiles");
-        info.getJSONArray("containerLogFiles") != null);
+    assertTrue("containerLogFiles missing", containerLogFilesArr != null);
+    assertEquals(2, containerLogFilesArr.length());
+    for (int i = 0; i < 2; i++) {
+      assertTrue("syslog and stdout expected",
+          containerLogFilesArr.get(i).equals("syslog") ||
+          containerLogFilesArr.get(i).equals("stdout"));
+    }
   }
 
   public void verifyNodeContainerInfoGeneric(Container cont, String id,