From 28d6a453a9d5b1ec12a1b5ec4f21cf275f01d3d4 Mon Sep 17 00:00:00 2001
From: Szilard Nemeth <snemeth@apache.org>
Date: Mon, 15 Jul 2019 11:59:11 +0200
Subject: [PATCH] YARN-9127. Create more tests to verify
 GpuDeviceInformationParser. Contributed by Peter Bacsko

(cherry picked from commit 18ee1092b471c5337f05809f8f01dae415e51a3a)
---
 ...esource-types-error-redefine-fpga-unit.xml |  45 ++
 ...resource-types-error-redefine-gpu-unit.xml |  45 ++
 .../resourceplugin/gpu/GpuDiscoverer.java     |   2 +-
 .../webapp/dao/gpu/GpuDeviceInformation.java  |  18 +-
 .../dao/gpu/GpuDeviceInformationParser.java   |  48 +-
 .../dao/gpu/PerGpuDeviceInformation.java      |   1 -
 .../webapp/dao/gpu/PerGpuMemoryUsage.java     |   7 +-
 .../gpu/TestGpuDeviceInformationParser.java   | 159 ++++-
 .../resources/nvidia-smi-output-excerpt.xml   |  71 +++
 .../nvidia-smi-output-missing-tags.xml        |  28 +
 .../nvidia-smi-output-missing-tags2.xml       |  61 ++
 .../resources/nvidia-smi-sample-output.xml    | 547 ++++++++++++++++++
 .../resources/nvidia-smi-sample-xml-output    | 547 ------------------
 13 files changed, 978 insertions(+), 601 deletions(-)
 create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-fpga-unit.xml
 create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-gpu-unit.xml
 create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-excerpt.xml
 create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-missing-tags.xml
 create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-missing-tags2.xml
 create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-output.xml
 delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-xml-output

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-fpga-unit.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-fpga-unit.xml
new file mode 100644
index 00000000000..72cfb9888a9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-fpga-unit.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>yarn.io/fpga</value>
+ </property>
+
+ <property>
+   <name>yarn.resource-types.yarn.io/fpga.units</name>
+   <value>G</value>
+ </property>
+
+</configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-gpu-unit.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-gpu-unit.xml
new file mode 100644
index 00000000000..aa61b5f40bb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-redefine-gpu-unit.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>yarn.io/gpu</value>
+ </property>
+
+ <property>
+   <name>yarn.resource-types.yarn.io/gpu.units</name>
+   <value>G</value>
+ </property>
+
+</configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
index 27a4ea1e4a1..ce767229e50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
@@ -64,7 +64,6 @@ public class GpuDiscoverer {
   private Configuration conf = null;
   private String pathOfGpuBinary = null;
   private Map<String, String> environment = new HashMap<>();
-  private GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
 
   private int numOfErrorExecutionSinceLastSucceed = 0;
   private GpuDeviceInformation lastDiscoveredGpuInformation = null;
@@ -125,6 +124,7 @@ public class GpuDiscoverer {
     try {
       output = Shell.execCommand(environment,
           new String[] { pathOfGpuBinary, "-x", "-q" }, MAX_EXEC_TIMEOUT_MS);
+      GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
       lastDiscoveredGpuInformation = parser.parseXml(output);
       numOfErrorExecutionSinceLastSucceed = 0;
       return lastDiscoveredGpuInformation;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
index 837d5cc99cd..c830d432a6b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import java.util.List;
 
@@ -31,14 +32,10 @@ import java.util.List;
 @InterfaceStability.Unstable
 @XmlRootElement(name = "nvidia_smi_log")
 public class GpuDeviceInformation {
-  List<PerGpuDeviceInformation> gpus;
+  private List<PerGpuDeviceInformation> gpus;
+  private String driverVersion = "N/A";
 
-  String driverVersion = "N/A";
-
-  // More fields like topology information could be added when needed.
-  // ...
-
-  @javax.xml.bind.annotation.XmlElement(name = "gpu")
+  @XmlElement(name = "gpu")
   public List<PerGpuDeviceInformation> getGpus() {
     return gpus;
   }
@@ -47,7 +44,7 @@ public class GpuDeviceInformation {
     this.gpus = gpus;
   }
 
-  @javax.xml.bind.annotation.XmlElement(name = "driver_version")
+  @XmlElement(name = "driver_version")
   public String getDriverVersion() {
     return driverVersion;
   }
@@ -59,8 +56,9 @@ public class GpuDeviceInformation {
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();
-    sb.append("=== Gpus in the system ===\n").append("\tDriver Version:").append(
-        getDriverVersion()).append("\n");
+    sb.append("=== GPUs in the system ===\n")
+        .append("\tDriver Version:")
+        .append(getDriverVersion()).append("\n");
 
     if (gpus != null) {
       for (PerGpuDeviceInformation gpu : gpus) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
index 894cb70ca61..5c166571ccc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
@@ -24,13 +24,11 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
 import org.xml.sax.XMLReader;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.Unmarshaller;
-import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParserFactory;
 import javax.xml.transform.sax.SAXSource;
 import java.io.StringReader;
@@ -46,39 +44,39 @@ public class GpuDeviceInformationParser {
   public static final String GPU_SCRIPT_REFERENCE = "GPU device detection " +
       "script";
 
-  private Unmarshaller unmarshaller = null;
-  private XMLReader xmlReader = null;
+  private final Unmarshaller unmarshaller;
+  private final XMLReader xmlReader;
 
-  private void init()
-      throws SAXException, ParserConfigurationException, JAXBException {
+  public GpuDeviceInformationParser() throws YarnException {
+    try {
+      final SAXParserFactory parserFactory = initSaxParserFactory();
+      final JAXBContext jaxbContext = JAXBContext.newInstance(
+          GpuDeviceInformation.class);
+      this.xmlReader = parserFactory.newSAXParser().getXMLReader();
+      this.unmarshaller = jaxbContext.createUnmarshaller();
+    } catch (Exception e) {
+      String msg = "Exception while initializing parser for " +
+          GPU_SCRIPT_REFERENCE;
+      LOG.error(msg, e);
+      throw new YarnException(msg, e);
+    }
+  }
+
+  /**
+   * Disable external-dtd since by default nvidia-smi output contains
+   * &lt;!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd"> in header.
+   */
+  private SAXParserFactory initSaxParserFactory() throws Exception {
     SAXParserFactory spf = SAXParserFactory.newInstance();
-    // Disable external-dtd since by default nvidia-smi output contains
-    // <!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd"> in header
     spf.setFeature(
         "http://apache.org/xml/features/nonvalidating/load-external-dtd",
         false);
     spf.setFeature("http://xml.org/sax/features/validation", false);
-
-    JAXBContext jaxbContext = JAXBContext.newInstance(
-        GpuDeviceInformation.class);
-
-    this.xmlReader = spf.newSAXParser().getXMLReader();
-    this.unmarshaller = jaxbContext.createUnmarshaller();
+    return spf;
   }
 
   public synchronized GpuDeviceInformation parseXml(String xmlContent)
       throws YarnException {
-    if (unmarshaller == null) {
-      try {
-        init();
-      } catch (SAXException | ParserConfigurationException | JAXBException e) {
-        String msg = "Exception while initializing parser for " +
-            GPU_SCRIPT_REFERENCE;
-        LOG.error(msg, e);
-        throw new YarnException(e);
-      }
-    }
-
     InputSource inputSource = new InputSource(new StringReader(xmlContent));
     SAXSource source = new SAXSource(xmlReader, inputSource);
     try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
index 25c2e3a1f1d..11ff2a4c49c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
@@ -33,7 +33,6 @@ import javax.xml.bind.annotation.adapters.XmlAdapter;
 @InterfaceStability.Unstable
 @XmlRootElement(name = "gpu")
 public class PerGpuDeviceInformation {
-
   private String productName = "N/A";
   private String uuid = "N/A";
   private int minorNumber = -1;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
index afc1a9679b7..1c2c0c49b84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
@@ -29,8 +29,8 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @InterfaceStability.Unstable
 @XmlRootElement(name = "fb_memory_usage")
 public class PerGpuMemoryUsage {
-  long usedMemoryMiB = -1L;
-  long availMemoryMiB = -1L;
+  private long usedMemoryMiB = -1L;
+  private long availMemoryMiB = -1L;
 
   @XmlJavaTypeAdapter(PerGpuDeviceInformation.StrToMemAdapter.class)
   @XmlElement(name = "used")
@@ -53,6 +53,9 @@ public class PerGpuMemoryUsage {
   }
 
   public long getTotalMemoryMiB() {
+    if (usedMemoryMiB == -1 && availMemoryMiB == -1) {
+      return -1;
+    }
     return usedMemoryMiB + availMemoryMiB;
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
index dc96746cf5d..28e06b72bc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
@@ -20,31 +20,160 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.junit.Assert;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import java.io.File;
 import java.io.IOException;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
 public class TestGpuDeviceInformationParser {
+  private static final String UTF_8 = "UTF-8";
+  private static final double DELTA = 1e-6;
+
+  @Rule
+  public ExpectedException expected = ExpectedException.none();
+
   @Test
   public void testParse() throws IOException, YarnException {
-    File f = new File("src/test/resources/nvidia-smi-sample-xml-output");
-    String s = FileUtils.readFileToString(f, "UTF-8");
+    File f = new File("src/test/resources/nvidia-smi-sample-output.xml");
+    String s = FileUtils.readFileToString(f, UTF_8);
 
     GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
-
     GpuDeviceInformation info = parser.parseXml(s);
-    Assert.assertEquals("375.66", info.getDriverVersion());
-    Assert.assertEquals(2, info.getGpus().size());
-    PerGpuDeviceInformation gpu1 = info.getGpus().get(1);
-    Assert.assertEquals("Tesla P100-PCIE-12GB", gpu1.getProductName());
-    Assert.assertEquals(12193, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
-    Assert.assertEquals(10.3f,
-        gpu1.getGpuUtilizations().getOverallGpuUtilization(), 1e-6);
-    Assert.assertEquals(34f, gpu1.getTemperature().getCurrentGpuTemp(), 1e-6);
-    Assert.assertEquals(85f, gpu1.getTemperature().getMaxGpuTemp(), 1e-6);
-    Assert.assertEquals(82f, gpu1.getTemperature().getSlowThresholdGpuTemp(),
-        1e-6);
+
+    assertEquals("375.66", info.getDriverVersion());
+    assertEquals(2, info.getGpus().size());
+    assertFirstGpu(info.getGpus().get(0));
+    assertSecondGpu(info.getGpus().get(1));
+  }
+
+  @Test
+  public void testParseExcerpt() throws IOException, YarnException {
+    File f = new File("src/test/resources/nvidia-smi-output-excerpt.xml");
+    String s = FileUtils.readFileToString(f, UTF_8);
+
+    GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
+    GpuDeviceInformation info = parser.parseXml(s);
+
+    assertEquals("375.66", info.getDriverVersion());
+    assertEquals(2, info.getGpus().size());
+    assertFirstGpu(info.getGpus().get(0));
+    assertSecondGpu(info.getGpus().get(1));
+  }
+
+  @Test
+  public void testParseConsecutivelyWithSameParser()
+      throws IOException, YarnException {
+    File f = new File("src/test/resources/nvidia-smi-sample-output.xml");
+    String s = FileUtils.readFileToString(f, UTF_8);
+
+    for (int i = 0; i < 3; i++) {
+      GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
+      GpuDeviceInformation info = parser.parseXml(s);
+
+      assertEquals("375.66", info.getDriverVersion());
+      assertEquals(2, info.getGpus().size());
+      assertFirstGpu(info.getGpus().get(0));
+      assertSecondGpu(info.getGpus().get(1));
+    }
+  }
+
+  @Test
+  public void testParseEmptyString() throws YarnException {
+    expected.expect(YarnException.class);
+    GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
+    parser.parseXml("");
+  }
+
+  @Test
+  public void testParseInvalidRootElement() throws YarnException {
+    expected.expect(YarnException.class);
+    GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
+    parser.parseXml("<nvidia_smiiiii></nvidia_smiiiii");
+  }
+
+  @Test
+  public void testParseMissingTags() throws IOException, YarnException {
+    File f = new File("src/test/resources/nvidia-smi-output-missing-tags.xml");
+    String s = FileUtils.readFileToString(f, UTF_8);
+
+    GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
+    GpuDeviceInformation info = parser.parseXml(s);
+
+    assertEquals("375.66", info.getDriverVersion());
+    assertEquals(1, info.getGpus().size());
+
+    PerGpuDeviceInformation gpu = info.getGpus().get(0);
+    assertEquals("N/A", gpu.getProductName());
+    assertEquals("N/A", gpu.getUuid());
+    assertEquals(-1, gpu.getMinorNumber());
+    assertNull(gpu.getGpuMemoryUsage());
+    assertNull(gpu.getTemperature());
+    assertNull(gpu.getGpuUtilizations());
+  }
+
+  @Test
+  public void testParseMissingInnerTags() throws IOException, YarnException {
+    File f =new File("src/test/resources/nvidia-smi-output-missing-tags2.xml");
+    String s = FileUtils.readFileToString(f, UTF_8);
+
+    GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
+    GpuDeviceInformation info = parser.parseXml(s);
+
+    assertEquals("375.66", info.getDriverVersion());
+    assertEquals(2, info.getGpus().size());
+
+    PerGpuDeviceInformation gpu = info.getGpus().get(0);
+    assertEquals("Tesla P100-PCIE-12GB", gpu.getProductName());
+    assertEquals("GPU-28604e81-21ec-cc48-6759-bf2648b22e16", gpu.getUuid());
+    assertEquals(0, gpu.getMinorNumber());
+    assertEquals(-1, gpu.getGpuMemoryUsage().getTotalMemoryMiB());
+    assertEquals(-1, (long) gpu.getGpuMemoryUsage().getUsedMemoryMiB());
+    assertEquals(-1, (long) gpu.getGpuMemoryUsage().getAvailMemoryMiB());
+    assertEquals(0f,
+        gpu.getGpuUtilizations().getOverallGpuUtilization(), DELTA);
+    assertEquals(Float.MIN_VALUE,
+        gpu.getTemperature().getCurrentGpuTemp(), DELTA);
+    assertEquals(Float.MIN_VALUE,
+        gpu.getTemperature().getMaxGpuTemp(), DELTA);
+    assertEquals(Float.MIN_VALUE,
+        gpu.getTemperature().getSlowThresholdGpuTemp(),
+        DELTA);
+
+    assertSecondGpu(info.getGpus().get(1));
+  }
+
+  private void assertFirstGpu(PerGpuDeviceInformation gpu) {
+    assertEquals("Tesla P100-PCIE-12GB", gpu.getProductName());
+    assertEquals("GPU-28604e81-21ec-cc48-6759-bf2648b22e16", gpu.getUuid());
+    assertEquals(0, gpu.getMinorNumber());
+    assertEquals(11567, gpu.getGpuMemoryUsage().getTotalMemoryMiB());
+    assertEquals(11400, (long) gpu.getGpuMemoryUsage().getUsedMemoryMiB());
+    assertEquals(167, (long) gpu.getGpuMemoryUsage().getAvailMemoryMiB());
+    assertEquals(33.4f,
+        gpu.getGpuUtilizations().getOverallGpuUtilization(), DELTA);
+    assertEquals(31f, gpu.getTemperature().getCurrentGpuTemp(), DELTA);
+    assertEquals(80f, gpu.getTemperature().getMaxGpuTemp(), DELTA);
+    assertEquals(88f, gpu.getTemperature().getSlowThresholdGpuTemp(),
+        DELTA);
+  }
+
+  private void assertSecondGpu(PerGpuDeviceInformation gpu) {
+    assertEquals("Tesla P100-PCIE-12GB_2", gpu.getProductName());
+    assertEquals("GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3", gpu.getUuid());
+    assertEquals(1, gpu.getMinorNumber());
+    assertEquals(12290, gpu.getGpuMemoryUsage().getTotalMemoryMiB());
+    assertEquals(11800, (long) gpu.getGpuMemoryUsage().getUsedMemoryMiB());
+    assertEquals(490, (long) gpu.getGpuMemoryUsage().getAvailMemoryMiB());
+    assertEquals(10.3f,
+        gpu.getGpuUtilizations().getOverallGpuUtilization(), DELTA);
+    assertEquals(34f, gpu.getTemperature().getCurrentGpuTemp(), DELTA);
+    assertEquals(85f, gpu.getTemperature().getMaxGpuTemp(), DELTA);
+    assertEquals(82f, gpu.getTemperature().getSlowThresholdGpuTemp(),
+        DELTA);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-excerpt.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-excerpt.xml
new file mode 100644
index 00000000000..84b3abf7ed3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-excerpt.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
+
+<nvidia_smi_log>
+    <timestamp>Wed Sep 6 21:52:51 2017</timestamp>
+    <driver_version>375.66</driver_version>
+    <attached_gpus>2</attached_gpus>
+    <gpu id="0000:04:00.0">
+        <product_name>Tesla P100-PCIE-12GB</product_name>
+        <product_brand>Tesla</product_brand>
+        <uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
+        <minor_number>0</minor_number>
+        <fb_memory_usage>
+            <total>11567 MiB</total>
+            <used>11400 MiB</used>
+            <free>167 MiB</free>
+        </fb_memory_usage>
+        <utilization>
+            <gpu_util>33.4 %</gpu_util>
+            <memory_util>0 %</memory_util>
+            <encoder_util>0 %</encoder_util>
+            <decoder_util>0 %</decoder_util>
+        </utilization>
+        <temperature>
+            <gpu_temp>31 C</gpu_temp>
+            <gpu_temp_max_threshold>80 C</gpu_temp_max_threshold>
+            <gpu_temp_slow_threshold>88 C</gpu_temp_slow_threshold>
+        </temperature>
+    </gpu>
+
+    <gpu id="0000:82:00.0">
+        <product_name>Tesla P100-PCIE-12GB_2</product_name>
+        <product_brand>Tesla</product_brand>
+        <uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
+        <minor_number>1</minor_number>
+        <fb_memory_usage>
+            <total>12290 MiB</total>
+            <used>11800 MiB</used>
+            <free>490 MiB</free>
+        </fb_memory_usage>
+        <compute_mode>Default</compute_mode>
+        <utilization>
+            <gpu_util>10.3 %</gpu_util>
+            <memory_util>0 %</memory_util>
+            <encoder_util>0 %</encoder_util>
+            <decoder_util>0 %</decoder_util>
+        </utilization>
+        <temperature>
+            <gpu_temp>34 C</gpu_temp>
+            <gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
+            <gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
+        </temperature>
+    </gpu>
+
+</nvidia_smi_log>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-missing-tags.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-missing-tags.xml
new file mode 100644
index 00000000000..df341d36a15
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-missing-tags.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
+
+<nvidia_smi_log>
+    <timestamp>Wed Sep  6 21:52:51 2017</timestamp>
+    <driver_version>375.66</driver_version>
+    <attached_gpus>2</attached_gpus>
+    <gpu id="">
+        <product_brand>Tesla</product_brand>
+    </gpu>
+
+</nvidia_smi_log>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-missing-tags2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-missing-tags2.xml
new file mode 100644
index 00000000000..add92a83d7d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-output-missing-tags2.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
+
+<nvidia_smi_log>
+    <timestamp>Wed Sep  6 21:52:51 2017</timestamp>
+    <driver_version>375.66</driver_version>
+    <attached_gpus>2</attached_gpus>
+    <gpu id="0000:04:00.0">
+        <product_name>Tesla P100-PCIE-12GB</product_name>
+        <product_brand>Tesla</product_brand>
+        <uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
+        <minor_number>0</minor_number>
+        <fb_memory_usage>
+        </fb_memory_usage>
+        <utilization>
+        </utilization>
+        <temperature>
+        </temperature>
+    </gpu>
+
+    <gpu id="0000:82:00.0">
+        <product_name>Tesla P100-PCIE-12GB_2</product_name>
+        <product_brand>Tesla</product_brand>
+        <uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
+        <minor_number>1</minor_number>
+        <fb_memory_usage>
+            <total>12290 MiB</total>
+            <used>11800 MiB</used>
+            <free>490 MiB</free>
+        </fb_memory_usage>
+        <compute_mode>Default</compute_mode>
+        <utilization>
+            <gpu_util>10.3 %</gpu_util>
+            <memory_util>0 %</memory_util>
+            <encoder_util>0 %</encoder_util>
+            <decoder_util>0 %</decoder_util>
+        </utilization>
+        <temperature>
+            <gpu_temp>34 C</gpu_temp>
+            <gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
+            <gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
+        </temperature>
+    </gpu>
+
+</nvidia_smi_log>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-output.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-output.xml
new file mode 100644
index 00000000000..cdd5404ea90
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-output.xml
@@ -0,0 +1,547 @@
+<?xml version="1.0" ?>
+<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<nvidia_smi_log>
+    <timestamp>Wed Sep  6 21:52:51 2017</timestamp>
+    <driver_version>375.66</driver_version>
+    <attached_gpus>2</attached_gpus>
+    <gpu id="0000:04:00.0">
+        <product_name>Tesla P100-PCIE-12GB</product_name>
+        <product_brand>Tesla</product_brand>
+        <display_mode>Disabled</display_mode>
+        <display_active>Disabled</display_active>
+        <persistence_mode>Disabled</persistence_mode>
+        <accounting_mode>Disabled</accounting_mode>
+        <accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
+        <driver_model>
+            <current_dm>N/A</current_dm>
+            <pending_dm>N/A</pending_dm>
+        </driver_model>
+        <serial>0320717030197</serial>
+        <uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
+        <minor_number>0</minor_number>
+        <vbios_version>86.00.3A.00.02</vbios_version>
+        <multigpu_board>No</multigpu_board>
+        <board_id>0x400</board_id>
+        <gpu_part_number>900-2H400-0110-030</gpu_part_number>
+        <inforom_version>
+            <img_version>H400.0202.00.01</img_version>
+            <oem_object>1.1</oem_object>
+            <ecc_object>4.1</ecc_object>
+            <pwr_object>N/A</pwr_object>
+        </inforom_version>
+        <gpu_operation_mode>
+            <current_gom>N/A</current_gom>
+            <pending_gom>N/A</pending_gom>
+        </gpu_operation_mode>
+        <gpu_virtualization_mode>
+            <virtualization_mode>None</virtualization_mode>
+        </gpu_virtualization_mode>
+        <pci>
+            <pci_bus>04</pci_bus>
+            <pci_device>00</pci_device>
+            <pci_domain>0000</pci_domain>
+            <pci_device_id>15F710DE</pci_device_id>
+            <pci_bus_id>0000:04:00.0</pci_bus_id>
+            <pci_sub_system_id>11DA10DE</pci_sub_system_id>
+            <pci_gpu_link_info>
+                <pcie_gen>
+                    <max_link_gen>3</max_link_gen>
+                    <current_link_gen>3</current_link_gen>
+                </pcie_gen>
+                <link_widths>
+                    <max_link_width>16x</max_link_width>
+                    <current_link_width>16x</current_link_width>
+                </link_widths>
+            </pci_gpu_link_info>
+            <pci_bridge_chip>
+                <bridge_chip_type>N/A</bridge_chip_type>
+                <bridge_chip_fw>N/A</bridge_chip_fw>
+            </pci_bridge_chip>
+            <replay_counter>0</replay_counter>
+            <tx_util>0 KB/s</tx_util>
+            <rx_util>0 KB/s</rx_util>
+        </pci>
+        <fan_speed>N/A</fan_speed>
+        <performance_state>P0</performance_state>
+        <clocks_throttle_reasons>
+            <clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
+            <clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
+            <clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
+            <clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
+            <clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
+            <clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
+        </clocks_throttle_reasons>
+        <fb_memory_usage>
+            <total>11567 MiB</total>
+            <used>11400 MiB</used>
+            <free>167 MiB</free>
+        </fb_memory_usage>
+        <bar1_memory_usage>
+            <total>16384 MiB</total>
+            <used>2 MiB</used>
+            <free>16382 MiB</free>
+        </bar1_memory_usage>
+        <compute_mode>Default</compute_mode>
+        <utilization>
+            <gpu_util>33.4 %</gpu_util>
+            <memory_util>0 %</memory_util>
+            <encoder_util>0 %</encoder_util>
+            <decoder_util>0 %</decoder_util>
+        </utilization>
+        <encoder_stats>
+            <session_count>0</session_count>
+            <average_fps>0</average_fps>
+            <average_latency>0 ms</average_latency>
+        </encoder_stats>
+        <ecc_mode>
+            <current_ecc>Enabled</current_ecc>
+            <pending_ecc>Enabled</pending_ecc>
+        </ecc_mode>
+        <ecc_errors>
+            <volatile>
+                <single_bit>
+                    <device_memory>0</device_memory>
+                    <register_file>0</register_file>
+                    <l1_cache>N/A</l1_cache>
+                    <l2_cache>0</l2_cache>
+                    <texture_memory>0</texture_memory>
+                    <texture_shm>0</texture_shm>
+                    <total>0</total>
+                </single_bit>
+                <double_bit>
+                    <device_memory>0</device_memory>
+                    <register_file>0</register_file>
+                    <l1_cache>N/A</l1_cache>
+                    <l2_cache>0</l2_cache>
+                    <texture_memory>0</texture_memory>
+                    <texture_shm>0</texture_shm>
+                    <total>0</total>
+                </double_bit>
+            </volatile>
+            <aggregate>
+                <single_bit>
+                    <device_memory>0</device_memory>
+                    <register_file>0</register_file>
+                    <l1_cache>N/A</l1_cache>
+                    <l2_cache>0</l2_cache>
+                    <texture_memory>0</texture_memory>
+                    <texture_shm>0</texture_shm>
+                    <total>0</total>
+                </single_bit>
+                <double_bit>
+                    <device_memory>0</device_memory>
+                    <register_file>0</register_file>
+                    <l1_cache>N/A</l1_cache>
+                    <l2_cache>0</l2_cache>
+                    <texture_memory>0</texture_memory>
+                    <texture_shm>0</texture_shm>
+                    <total>0</total>
+                </double_bit>
+            </aggregate>
+        </ecc_errors>
+        <retired_pages>
+            <multiple_single_bit_retirement>
+                <retired_count>0</retired_count>
+                <retired_page_addresses>
+                </retired_page_addresses>
+            </multiple_single_bit_retirement>
+            <double_bit_retirement>
+                <retired_count>0</retired_count>
+                <retired_page_addresses>
+                </retired_page_addresses>
+            </double_bit_retirement>
+            <pending_retirement>No</pending_retirement>
+        </retired_pages>
+        <temperature>
+            <gpu_temp>31 C</gpu_temp>
+            <gpu_temp_max_threshold>80 C</gpu_temp_max_threshold>
+            <gpu_temp_slow_threshold>88 C</gpu_temp_slow_threshold>
+        </temperature>
+        <power_readings>
+            <power_state>P0</power_state>
+            <power_management>Supported</power_management>
+            <power_draw>24.84 W</power_draw>
+            <power_limit>250.00 W</power_limit>
+            <default_power_limit>250.00 W</default_power_limit>
+            <enforced_power_limit>250.00 W</enforced_power_limit>
+            <min_power_limit>125.00 W</min_power_limit>
+            <max_power_limit>250.00 W</max_power_limit>
+        </power_readings>
+        <clocks>
+            <graphics_clock>405 MHz</graphics_clock>
+            <sm_clock>405 MHz</sm_clock>
+            <mem_clock>715 MHz</mem_clock>
+            <video_clock>835 MHz</video_clock>
+        </clocks>
+        <applications_clocks>
+            <graphics_clock>1189 MHz</graphics_clock>
+            <mem_clock>715 MHz</mem_clock>
+        </applications_clocks>
+        <default_applications_clocks>
+            <graphics_clock>1189 MHz</graphics_clock>
+            <mem_clock>715 MHz</mem_clock>
+        </default_applications_clocks>
+        <max_clocks>
+            <graphics_clock>1328 MHz</graphics_clock>
+            <sm_clock>1328 MHz</sm_clock>
+            <mem_clock>715 MHz</mem_clock>
+            <video_clock>1328 MHz</video_clock>
+        </max_clocks>
+        <clock_policy>
+            <auto_boost>N/A</auto_boost>
+            <auto_boost_default>N/A</auto_boost_default>
+        </clock_policy>
+        <supported_clocks>
+            <supported_mem_clock>
+                <value>715 MHz</value>
+                <supported_graphics_clock>1328 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1316 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1303 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1290 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1278 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1265 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1252 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1240 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1227 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1215 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1202 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1189 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1177 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1164 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1151 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1139 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1126 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1113 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1101 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1088 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1075 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1063 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1050 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1037 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1025 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1012 MHz</supported_graphics_clock>
+                <supported_graphics_clock>999 MHz</supported_graphics_clock>
+                <supported_graphics_clock>987 MHz</supported_graphics_clock>
+                <supported_graphics_clock>974 MHz</supported_graphics_clock>
+                <supported_graphics_clock>961 MHz</supported_graphics_clock>
+                <supported_graphics_clock>949 MHz</supported_graphics_clock>
+                <supported_graphics_clock>936 MHz</supported_graphics_clock>
+                <supported_graphics_clock>923 MHz</supported_graphics_clock>
+                <supported_graphics_clock>911 MHz</supported_graphics_clock>
+                <supported_graphics_clock>898 MHz</supported_graphics_clock>
+                <supported_graphics_clock>885 MHz</supported_graphics_clock>
+                <supported_graphics_clock>873 MHz</supported_graphics_clock>
+                <supported_graphics_clock>860 MHz</supported_graphics_clock>
+                <supported_graphics_clock>847 MHz</supported_graphics_clock>
+                <supported_graphics_clock>835 MHz</supported_graphics_clock>
+                <supported_graphics_clock>822 MHz</supported_graphics_clock>
+                <supported_graphics_clock>810 MHz</supported_graphics_clock>
+                <supported_graphics_clock>797 MHz</supported_graphics_clock>
+                <supported_graphics_clock>784 MHz</supported_graphics_clock>
+                <supported_graphics_clock>772 MHz</supported_graphics_clock>
+                <supported_graphics_clock>759 MHz</supported_graphics_clock>
+                <supported_graphics_clock>746 MHz</supported_graphics_clock>
+                <supported_graphics_clock>734 MHz</supported_graphics_clock>
+                <supported_graphics_clock>721 MHz</supported_graphics_clock>
+                <supported_graphics_clock>708 MHz</supported_graphics_clock>
+                <supported_graphics_clock>696 MHz</supported_graphics_clock>
+                <supported_graphics_clock>683 MHz</supported_graphics_clock>
+                <supported_graphics_clock>670 MHz</supported_graphics_clock>
+                <supported_graphics_clock>658 MHz</supported_graphics_clock>
+                <supported_graphics_clock>645 MHz</supported_graphics_clock>
+                <supported_graphics_clock>632 MHz</supported_graphics_clock>
+                <supported_graphics_clock>620 MHz</supported_graphics_clock>
+                <supported_graphics_clock>607 MHz</supported_graphics_clock>
+                <supported_graphics_clock>594 MHz</supported_graphics_clock>
+                <supported_graphics_clock>582 MHz</supported_graphics_clock>
+                <supported_graphics_clock>569 MHz</supported_graphics_clock>
+                <supported_graphics_clock>556 MHz</supported_graphics_clock>
+                <supported_graphics_clock>544 MHz</supported_graphics_clock>
+            </supported_mem_clock>
+        </supported_clocks>
+        <processes>
+        </processes>
+        <accounted_processes>
+        </accounted_processes>
+    </gpu>
+
+    <gpu id="0000:82:00.0">
+        <product_name>Tesla P100-PCIE-12GB_2</product_name>
+        <product_brand>Tesla</product_brand>
+        <display_mode>Disabled</display_mode>
+        <display_active>Disabled</display_active>
+        <persistence_mode>Disabled</persistence_mode>
+        <accounting_mode>Disabled</accounting_mode>
+        <accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
+        <driver_model>
+            <current_dm>N/A</current_dm>
+            <pending_dm>N/A</pending_dm>
+        </driver_model>
+        <serial>0320717031755</serial>
+        <uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
+        <minor_number>1</minor_number>
+        <vbios_version>86.00.3A.00.02</vbios_version>
+        <multigpu_board>No</multigpu_board>
+        <board_id>0x8200</board_id>
+        <gpu_part_number>900-2H400-0110-030</gpu_part_number>
+        <inforom_version>
+            <img_version>H400.0202.00.01</img_version>
+            <oem_object>1.1</oem_object>
+            <ecc_object>4.1</ecc_object>
+            <pwr_object>N/A</pwr_object>
+        </inforom_version>
+        <gpu_operation_mode>
+            <current_gom>N/A</current_gom>
+            <pending_gom>N/A</pending_gom>
+        </gpu_operation_mode>
+        <gpu_virtualization_mode>
+            <virtualization_mode>None</virtualization_mode>
+        </gpu_virtualization_mode>
+        <pci>
+            <pci_bus>82</pci_bus>
+            <pci_device>00</pci_device>
+            <pci_domain>0000</pci_domain>
+            <pci_device_id>15F710DE</pci_device_id>
+            <pci_bus_id>0000:82:00.0</pci_bus_id>
+            <pci_sub_system_id>11DA10DE</pci_sub_system_id>
+            <pci_gpu_link_info>
+                <pcie_gen>
+                    <max_link_gen>3</max_link_gen>
+                    <current_link_gen>3</current_link_gen>
+                </pcie_gen>
+                <link_widths>
+                    <max_link_width>16x</max_link_width>
+                    <current_link_width>16x</current_link_width>
+                </link_widths>
+            </pci_gpu_link_info>
+            <pci_bridge_chip>
+                <bridge_chip_type>N/A</bridge_chip_type>
+                <bridge_chip_fw>N/A</bridge_chip_fw>
+            </pci_bridge_chip>
+            <replay_counter>0</replay_counter>
+            <tx_util>0 KB/s</tx_util>
+            <rx_util>0 KB/s</rx_util>
+        </pci>
+        <fan_speed>N/A</fan_speed>
+        <performance_state>P0</performance_state>
+        <clocks_throttle_reasons>
+            <clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
+            <clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
+            <clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
+            <clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
+            <clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
+            <clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
+        </clocks_throttle_reasons>
+        <fb_memory_usage>
+            <total>12290 MiB</total>
+            <used>11800 MiB</used>
+            <free>490 MiB</free>
+        </fb_memory_usage>
+        <bar1_memory_usage>
+            <total>16384 MiB</total>
+            <used>2 MiB</used>
+            <free>16382 MiB</free>
+        </bar1_memory_usage>
+        <compute_mode>Default</compute_mode>
+        <utilization>
+            <gpu_util>10.3 %</gpu_util>
+            <memory_util>0 %</memory_util>
+            <encoder_util>0 %</encoder_util>
+            <decoder_util>0 %</decoder_util>
+        </utilization>
+        <encoder_stats>
+            <session_count>0</session_count>
+            <average_fps>0</average_fps>
+            <average_latency>0 ms</average_latency>
+        </encoder_stats>
+        <ecc_mode>
+            <current_ecc>Enabled</current_ecc>
+            <pending_ecc>Enabled</pending_ecc>
+        </ecc_mode>
+        <ecc_errors>
+            <volatile>
+                <single_bit>
+                    <device_memory>0</device_memory>
+                    <register_file>0</register_file>
+                    <l1_cache>N/A</l1_cache>
+                    <l2_cache>0</l2_cache>
+                    <texture_memory>0</texture_memory>
+                    <texture_shm>0</texture_shm>
+                    <total>0</total>
+                </single_bit>
+                <double_bit>
+                    <device_memory>0</device_memory>
+                    <register_file>0</register_file>
+                    <l1_cache>N/A</l1_cache>
+                    <l2_cache>0</l2_cache>
+                    <texture_memory>0</texture_memory>
+                    <texture_shm>0</texture_shm>
+                    <total>0</total>
+                </double_bit>
+            </volatile>
+            <aggregate>
+                <single_bit>
+                    <device_memory>0</device_memory>
+                    <register_file>0</register_file>
+                    <l1_cache>N/A</l1_cache>
+                    <l2_cache>0</l2_cache>
+                    <texture_memory>0</texture_memory>
+                    <texture_shm>0</texture_shm>
+                    <total>0</total>
+                </single_bit>
+                <double_bit>
+                    <device_memory>0</device_memory>
+                    <register_file>0</register_file>
+                    <l1_cache>N/A</l1_cache>
+                    <l2_cache>0</l2_cache>
+                    <texture_memory>0</texture_memory>
+                    <texture_shm>0</texture_shm>
+                    <total>0</total>
+                </double_bit>
+            </aggregate>
+        </ecc_errors>
+        <retired_pages>
+            <multiple_single_bit_retirement>
+                <retired_count>0</retired_count>
+                <retired_page_addresses>
+                </retired_page_addresses>
+            </multiple_single_bit_retirement>
+            <double_bit_retirement>
+                <retired_count>0</retired_count>
+                <retired_page_addresses>
+                </retired_page_addresses>
+            </double_bit_retirement>
+            <pending_retirement>No</pending_retirement>
+        </retired_pages>
+        <temperature>
+            <gpu_temp>34 C</gpu_temp>
+            <gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
+            <gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
+        </temperature>
+        <power_readings>
+            <power_state>P0</power_state>
+            <power_management>Supported</power_management>
+            <power_draw>25.54 W</power_draw>
+            <power_limit>250.00 W</power_limit>
+            <default_power_limit>250.00 W</default_power_limit>
+            <enforced_power_limit>250.00 W</enforced_power_limit>
+            <min_power_limit>125.00 W</min_power_limit>
+            <max_power_limit>250.00 W</max_power_limit>
+        </power_readings>
+        <clocks>
+            <graphics_clock>405 MHz</graphics_clock>
+            <sm_clock>405 MHz</sm_clock>
+            <mem_clock>715 MHz</mem_clock>
+            <video_clock>835 MHz</video_clock>
+        </clocks>
+        <applications_clocks>
+            <graphics_clock>1189 MHz</graphics_clock>
+            <mem_clock>715 MHz</mem_clock>
+        </applications_clocks>
+        <default_applications_clocks>
+            <graphics_clock>1189 MHz</graphics_clock>
+            <mem_clock>715 MHz</mem_clock>
+        </default_applications_clocks>
+        <max_clocks>
+            <graphics_clock>1328 MHz</graphics_clock>
+            <sm_clock>1328 MHz</sm_clock>
+            <mem_clock>715 MHz</mem_clock>
+            <video_clock>1328 MHz</video_clock>
+        </max_clocks>
+        <clock_policy>
+            <auto_boost>N/A</auto_boost>
+            <auto_boost_default>N/A</auto_boost_default>
+        </clock_policy>
+        <supported_clocks>
+            <supported_mem_clock>
+                <value>715 MHz</value>
+                <supported_graphics_clock>1328 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1316 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1303 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1290 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1278 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1265 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1252 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1240 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1227 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1215 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1202 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1189 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1177 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1164 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1151 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1139 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1126 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1113 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1101 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1088 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1075 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1063 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1050 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1037 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1025 MHz</supported_graphics_clock>
+                <supported_graphics_clock>1012 MHz</supported_graphics_clock>
+                <supported_graphics_clock>999 MHz</supported_graphics_clock>
+                <supported_graphics_clock>987 MHz</supported_graphics_clock>
+                <supported_graphics_clock>974 MHz</supported_graphics_clock>
+                <supported_graphics_clock>961 MHz</supported_graphics_clock>
+                <supported_graphics_clock>949 MHz</supported_graphics_clock>
+                <supported_graphics_clock>936 MHz</supported_graphics_clock>
+                <supported_graphics_clock>923 MHz</supported_graphics_clock>
+                <supported_graphics_clock>911 MHz</supported_graphics_clock>
+                <supported_graphics_clock>898 MHz</supported_graphics_clock>
+                <supported_graphics_clock>885 MHz</supported_graphics_clock>
+                <supported_graphics_clock>873 MHz</supported_graphics_clock>
+                <supported_graphics_clock>860 MHz</supported_graphics_clock>
+                <supported_graphics_clock>847 MHz</supported_graphics_clock>
+                <supported_graphics_clock>835 MHz</supported_graphics_clock>
+                <supported_graphics_clock>822 MHz</supported_graphics_clock>
+                <supported_graphics_clock>810 MHz</supported_graphics_clock>
+                <supported_graphics_clock>797 MHz</supported_graphics_clock>
+                <supported_graphics_clock>784 MHz</supported_graphics_clock>
+                <supported_graphics_clock>772 MHz</supported_graphics_clock>
+                <supported_graphics_clock>759 MHz</supported_graphics_clock>
+                <supported_graphics_clock>746 MHz</supported_graphics_clock>
+                <supported_graphics_clock>734 MHz</supported_graphics_clock>
+                <supported_graphics_clock>721 MHz</supported_graphics_clock>
+                <supported_graphics_clock>708 MHz</supported_graphics_clock>
+                <supported_graphics_clock>696 MHz</supported_graphics_clock>
+                <supported_graphics_clock>683 MHz</supported_graphics_clock>
+                <supported_graphics_clock>670 MHz</supported_graphics_clock>
+                <supported_graphics_clock>658 MHz</supported_graphics_clock>
+                <supported_graphics_clock>645 MHz</supported_graphics_clock>
+                <supported_graphics_clock>632 MHz</supported_graphics_clock>
+                <supported_graphics_clock>620 MHz</supported_graphics_clock>
+                <supported_graphics_clock>607 MHz</supported_graphics_clock>
+                <supported_graphics_clock>594 MHz</supported_graphics_clock>
+                <supported_graphics_clock>582 MHz</supported_graphics_clock>
+                <supported_graphics_clock>569 MHz</supported_graphics_clock>
+                <supported_graphics_clock>556 MHz</supported_graphics_clock>
+                <supported_graphics_clock>544 MHz</supported_graphics_clock>
+            </supported_mem_clock>
+        </supported_clocks>
+        <processes>
+        </processes>
+        <accounted_processes>
+        </accounted_processes>
+    </gpu>
+
+</nvidia_smi_log>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-xml-output b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-xml-output
deleted file mode 100644
index 5ccb72265b5..00000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-xml-output
+++ /dev/null
@@ -1,547 +0,0 @@
-<?xml version="1.0" ?>
-<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<nvidia_smi_log>
-	<timestamp>Wed Sep  6 21:52:51 2017</timestamp>
-	<driver_version>375.66</driver_version>
-	<attached_gpus>2</attached_gpus>
-	<gpu id="0000:04:00.0">
-		<product_name>Tesla P100-PCIE-12GB</product_name>
-		<product_brand>Tesla</product_brand>
-		<display_mode>Disabled</display_mode>
-		<display_active>Disabled</display_active>
-		<persistence_mode>Disabled</persistence_mode>
-		<accounting_mode>Disabled</accounting_mode>
-		<accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
-		<driver_model>
-			<current_dm>N/A</current_dm>
-			<pending_dm>N/A</pending_dm>
-		</driver_model>
-		<serial>0320717030197</serial>
-		<uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
-		<minor_number>0</minor_number>
-		<vbios_version>86.00.3A.00.02</vbios_version>
-		<multigpu_board>No</multigpu_board>
-		<board_id>0x400</board_id>
-		<gpu_part_number>900-2H400-0110-030</gpu_part_number>
-		<inforom_version>
-			<img_version>H400.0202.00.01</img_version>
-			<oem_object>1.1</oem_object>
-			<ecc_object>4.1</ecc_object>
-			<pwr_object>N/A</pwr_object>
-		</inforom_version>
-		<gpu_operation_mode>
-			<current_gom>N/A</current_gom>
-			<pending_gom>N/A</pending_gom>
-		</gpu_operation_mode>
-		<gpu_virtualization_mode>
-			<virtualization_mode>None</virtualization_mode>
-		</gpu_virtualization_mode>
-		<pci>
-			<pci_bus>04</pci_bus>
-			<pci_device>00</pci_device>
-			<pci_domain>0000</pci_domain>
-			<pci_device_id>15F710DE</pci_device_id>
-			<pci_bus_id>0000:04:00.0</pci_bus_id>
-			<pci_sub_system_id>11DA10DE</pci_sub_system_id>
-			<pci_gpu_link_info>
-				<pcie_gen>
-					<max_link_gen>3</max_link_gen>
-					<current_link_gen>3</current_link_gen>
-				</pcie_gen>
-				<link_widths>
-					<max_link_width>16x</max_link_width>
-					<current_link_width>16x</current_link_width>
-				</link_widths>
-			</pci_gpu_link_info>
-			<pci_bridge_chip>
-				<bridge_chip_type>N/A</bridge_chip_type>
-				<bridge_chip_fw>N/A</bridge_chip_fw>
-			</pci_bridge_chip>
-			<replay_counter>0</replay_counter>
-			<tx_util>0 KB/s</tx_util>
-			<rx_util>0 KB/s</rx_util>
-		</pci>
-		<fan_speed>N/A</fan_speed>
-		<performance_state>P0</performance_state>
-		<clocks_throttle_reasons>
-			<clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
-			<clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
-			<clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
-			<clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
-			<clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
-			<clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
-		</clocks_throttle_reasons>
-		<fb_memory_usage>
-			<total>12193 MiB</total>
-			<used>0 MiB</used>
-			<free>12193 MiB</free>
-		</fb_memory_usage>
-		<bar1_memory_usage>
-			<total>16384 MiB</total>
-			<used>2 MiB</used>
-			<free>16382 MiB</free>
-		</bar1_memory_usage>
-		<compute_mode>Default</compute_mode>
-		<utilization>
-			<gpu_util>0 %</gpu_util>
-			<memory_util>0 %</memory_util>
-			<encoder_util>0 %</encoder_util>
-			<decoder_util>0 %</decoder_util>
-		</utilization>
-		<encoder_stats>
-			<session_count>0</session_count>
-			<average_fps>0</average_fps>
-			<average_latency>0 ms</average_latency>
-		</encoder_stats>
-		<ecc_mode>
-			<current_ecc>Enabled</current_ecc>
-			<pending_ecc>Enabled</pending_ecc>
-		</ecc_mode>
-		<ecc_errors>
-			<volatile>
-				<single_bit>
-					<device_memory>0</device_memory>
-					<register_file>0</register_file>
-					<l1_cache>N/A</l1_cache>
-					<l2_cache>0</l2_cache>
-					<texture_memory>0</texture_memory>
-					<texture_shm>0</texture_shm>
-					<total>0</total>
-				</single_bit>
-				<double_bit>
-					<device_memory>0</device_memory>
-					<register_file>0</register_file>
-					<l1_cache>N/A</l1_cache>
-					<l2_cache>0</l2_cache>
-					<texture_memory>0</texture_memory>
-					<texture_shm>0</texture_shm>
-					<total>0</total>
-				</double_bit>
-			</volatile>
-			<aggregate>
-				<single_bit>
-					<device_memory>0</device_memory>
-					<register_file>0</register_file>
-					<l1_cache>N/A</l1_cache>
-					<l2_cache>0</l2_cache>
-					<texture_memory>0</texture_memory>
-					<texture_shm>0</texture_shm>
-					<total>0</total>
-				</single_bit>
-				<double_bit>
-					<device_memory>0</device_memory>
-					<register_file>0</register_file>
-					<l1_cache>N/A</l1_cache>
-					<l2_cache>0</l2_cache>
-					<texture_memory>0</texture_memory>
-					<texture_shm>0</texture_shm>
-					<total>0</total>
-				</double_bit>
-			</aggregate>
-		</ecc_errors>
-		<retired_pages>
-			<multiple_single_bit_retirement>
-				<retired_count>0</retired_count>
-				<retired_page_addresses>
-				</retired_page_addresses>
-			</multiple_single_bit_retirement>
-			<double_bit_retirement>
-				<retired_count>0</retired_count>
-				<retired_page_addresses>
-				</retired_page_addresses>
-			</double_bit_retirement>
-			<pending_retirement>No</pending_retirement>
-		</retired_pages>
-		<temperature>
-			<gpu_temp>31 C</gpu_temp>
-			<gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
-			<gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
-		</temperature>
-		<power_readings>
-			<power_state>P0</power_state>
-			<power_management>Supported</power_management>
-			<power_draw>24.84 W</power_draw>
-			<power_limit>250.00 W</power_limit>
-			<default_power_limit>250.00 W</default_power_limit>
-			<enforced_power_limit>250.00 W</enforced_power_limit>
-			<min_power_limit>125.00 W</min_power_limit>
-			<max_power_limit>250.00 W</max_power_limit>
-		</power_readings>
-		<clocks>
-			<graphics_clock>405 MHz</graphics_clock>
-			<sm_clock>405 MHz</sm_clock>
-			<mem_clock>715 MHz</mem_clock>
-			<video_clock>835 MHz</video_clock>
-		</clocks>
-		<applications_clocks>
-			<graphics_clock>1189 MHz</graphics_clock>
-			<mem_clock>715 MHz</mem_clock>
-		</applications_clocks>
-		<default_applications_clocks>
-			<graphics_clock>1189 MHz</graphics_clock>
-			<mem_clock>715 MHz</mem_clock>
-		</default_applications_clocks>
-		<max_clocks>
-			<graphics_clock>1328 MHz</graphics_clock>
-			<sm_clock>1328 MHz</sm_clock>
-			<mem_clock>715 MHz</mem_clock>
-			<video_clock>1328 MHz</video_clock>
-		</max_clocks>
-		<clock_policy>
-			<auto_boost>N/A</auto_boost>
-			<auto_boost_default>N/A</auto_boost_default>
-		</clock_policy>
-		<supported_clocks>
-			<supported_mem_clock>
-				<value>715 MHz</value>
-				<supported_graphics_clock>1328 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1316 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1303 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1290 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1278 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1265 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1252 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1240 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1227 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1215 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1202 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1189 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1177 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1164 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1151 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1139 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1126 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1113 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1101 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1088 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1075 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1063 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1050 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1037 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1025 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1012 MHz</supported_graphics_clock>
-				<supported_graphics_clock>999 MHz</supported_graphics_clock>
-				<supported_graphics_clock>987 MHz</supported_graphics_clock>
-				<supported_graphics_clock>974 MHz</supported_graphics_clock>
-				<supported_graphics_clock>961 MHz</supported_graphics_clock>
-				<supported_graphics_clock>949 MHz</supported_graphics_clock>
-				<supported_graphics_clock>936 MHz</supported_graphics_clock>
-				<supported_graphics_clock>923 MHz</supported_graphics_clock>
-				<supported_graphics_clock>911 MHz</supported_graphics_clock>
-				<supported_graphics_clock>898 MHz</supported_graphics_clock>
-				<supported_graphics_clock>885 MHz</supported_graphics_clock>
-				<supported_graphics_clock>873 MHz</supported_graphics_clock>
-				<supported_graphics_clock>860 MHz</supported_graphics_clock>
-				<supported_graphics_clock>847 MHz</supported_graphics_clock>
-				<supported_graphics_clock>835 MHz</supported_graphics_clock>
-				<supported_graphics_clock>822 MHz</supported_graphics_clock>
-				<supported_graphics_clock>810 MHz</supported_graphics_clock>
-				<supported_graphics_clock>797 MHz</supported_graphics_clock>
-				<supported_graphics_clock>784 MHz</supported_graphics_clock>
-				<supported_graphics_clock>772 MHz</supported_graphics_clock>
-				<supported_graphics_clock>759 MHz</supported_graphics_clock>
-				<supported_graphics_clock>746 MHz</supported_graphics_clock>
-				<supported_graphics_clock>734 MHz</supported_graphics_clock>
-				<supported_graphics_clock>721 MHz</supported_graphics_clock>
-				<supported_graphics_clock>708 MHz</supported_graphics_clock>
-				<supported_graphics_clock>696 MHz</supported_graphics_clock>
-				<supported_graphics_clock>683 MHz</supported_graphics_clock>
-				<supported_graphics_clock>670 MHz</supported_graphics_clock>
-				<supported_graphics_clock>658 MHz</supported_graphics_clock>
-				<supported_graphics_clock>645 MHz</supported_graphics_clock>
-				<supported_graphics_clock>632 MHz</supported_graphics_clock>
-				<supported_graphics_clock>620 MHz</supported_graphics_clock>
-				<supported_graphics_clock>607 MHz</supported_graphics_clock>
-				<supported_graphics_clock>594 MHz</supported_graphics_clock>
-				<supported_graphics_clock>582 MHz</supported_graphics_clock>
-				<supported_graphics_clock>569 MHz</supported_graphics_clock>
-				<supported_graphics_clock>556 MHz</supported_graphics_clock>
-				<supported_graphics_clock>544 MHz</supported_graphics_clock>
-			</supported_mem_clock>
-		</supported_clocks>
-		<processes>
-		</processes>
-		<accounted_processes>
-		</accounted_processes>
-	</gpu>
-
-	<gpu id="0000:82:00.0">
-		<product_name>Tesla P100-PCIE-12GB</product_name>
-		<product_brand>Tesla</product_brand>
-		<display_mode>Disabled</display_mode>
-		<display_active>Disabled</display_active>
-		<persistence_mode>Disabled</persistence_mode>
-		<accounting_mode>Disabled</accounting_mode>
-		<accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
-		<driver_model>
-			<current_dm>N/A</current_dm>
-			<pending_dm>N/A</pending_dm>
-		</driver_model>
-		<serial>0320717031755</serial>
-		<uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
-		<minor_number>1</minor_number>
-		<vbios_version>86.00.3A.00.02</vbios_version>
-		<multigpu_board>No</multigpu_board>
-		<board_id>0x8200</board_id>
-		<gpu_part_number>900-2H400-0110-030</gpu_part_number>
-		<inforom_version>
-			<img_version>H400.0202.00.01</img_version>
-			<oem_object>1.1</oem_object>
-			<ecc_object>4.1</ecc_object>
-			<pwr_object>N/A</pwr_object>
-		</inforom_version>
-		<gpu_operation_mode>
-			<current_gom>N/A</current_gom>
-			<pending_gom>N/A</pending_gom>
-		</gpu_operation_mode>
-		<gpu_virtualization_mode>
-			<virtualization_mode>None</virtualization_mode>
-		</gpu_virtualization_mode>
-		<pci>
-			<pci_bus>82</pci_bus>
-			<pci_device>00</pci_device>
-			<pci_domain>0000</pci_domain>
-			<pci_device_id>15F710DE</pci_device_id>
-			<pci_bus_id>0000:82:00.0</pci_bus_id>
-			<pci_sub_system_id>11DA10DE</pci_sub_system_id>
-			<pci_gpu_link_info>
-				<pcie_gen>
-					<max_link_gen>3</max_link_gen>
-					<current_link_gen>3</current_link_gen>
-				</pcie_gen>
-				<link_widths>
-					<max_link_width>16x</max_link_width>
-					<current_link_width>16x</current_link_width>
-				</link_widths>
-			</pci_gpu_link_info>
-			<pci_bridge_chip>
-				<bridge_chip_type>N/A</bridge_chip_type>
-				<bridge_chip_fw>N/A</bridge_chip_fw>
-			</pci_bridge_chip>
-			<replay_counter>0</replay_counter>
-			<tx_util>0 KB/s</tx_util>
-			<rx_util>0 KB/s</rx_util>
-		</pci>
-		<fan_speed>N/A</fan_speed>
-		<performance_state>P0</performance_state>
-		<clocks_throttle_reasons>
-			<clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
-			<clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
-			<clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
-			<clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
-			<clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
-			<clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
-		</clocks_throttle_reasons>
-		<fb_memory_usage>
-			<total>12193 MiB</total>
-			<used>0 MiB</used>
-			<free>12193 MiB</free>
-		</fb_memory_usage>
-		<bar1_memory_usage>
-			<total>16384 MiB</total>
-			<used>2 MiB</used>
-			<free>16382 MiB</free>
-		</bar1_memory_usage>
-		<compute_mode>Default</compute_mode>
-		<utilization>
-			<gpu_util>10.3 %</gpu_util>
-			<memory_util>0 %</memory_util>
-			<encoder_util>0 %</encoder_util>
-			<decoder_util>0 %</decoder_util>
-		</utilization>
-		<encoder_stats>
-			<session_count>0</session_count>
-			<average_fps>0</average_fps>
-			<average_latency>0 ms</average_latency>
-		</encoder_stats>
-		<ecc_mode>
-			<current_ecc>Enabled</current_ecc>
-			<pending_ecc>Enabled</pending_ecc>
-		</ecc_mode>
-		<ecc_errors>
-			<volatile>
-				<single_bit>
-					<device_memory>0</device_memory>
-					<register_file>0</register_file>
-					<l1_cache>N/A</l1_cache>
-					<l2_cache>0</l2_cache>
-					<texture_memory>0</texture_memory>
-					<texture_shm>0</texture_shm>
-					<total>0</total>
-				</single_bit>
-				<double_bit>
-					<device_memory>0</device_memory>
-					<register_file>0</register_file>
-					<l1_cache>N/A</l1_cache>
-					<l2_cache>0</l2_cache>
-					<texture_memory>0</texture_memory>
-					<texture_shm>0</texture_shm>
-					<total>0</total>
-				</double_bit>
-			</volatile>
-			<aggregate>
-				<single_bit>
-					<device_memory>0</device_memory>
-					<register_file>0</register_file>
-					<l1_cache>N/A</l1_cache>
-					<l2_cache>0</l2_cache>
-					<texture_memory>0</texture_memory>
-					<texture_shm>0</texture_shm>
-					<total>0</total>
-				</single_bit>
-				<double_bit>
-					<device_memory>0</device_memory>
-					<register_file>0</register_file>
-					<l1_cache>N/A</l1_cache>
-					<l2_cache>0</l2_cache>
-					<texture_memory>0</texture_memory>
-					<texture_shm>0</texture_shm>
-					<total>0</total>
-				</double_bit>
-			</aggregate>
-		</ecc_errors>
-		<retired_pages>
-			<multiple_single_bit_retirement>
-				<retired_count>0</retired_count>
-				<retired_page_addresses>
-				</retired_page_addresses>
-			</multiple_single_bit_retirement>
-			<double_bit_retirement>
-				<retired_count>0</retired_count>
-				<retired_page_addresses>
-				</retired_page_addresses>
-			</double_bit_retirement>
-			<pending_retirement>No</pending_retirement>
-		</retired_pages>
-		<temperature>
-			<gpu_temp>34 C</gpu_temp>
-			<gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
-			<gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
-		</temperature>
-		<power_readings>
-			<power_state>P0</power_state>
-			<power_management>Supported</power_management>
-			<power_draw>25.54 W</power_draw>
-			<power_limit>250.00 W</power_limit>
-			<default_power_limit>250.00 W</default_power_limit>
-			<enforced_power_limit>250.00 W</enforced_power_limit>
-			<min_power_limit>125.00 W</min_power_limit>
-			<max_power_limit>250.00 W</max_power_limit>
-		</power_readings>
-		<clocks>
-			<graphics_clock>405 MHz</graphics_clock>
-			<sm_clock>405 MHz</sm_clock>
-			<mem_clock>715 MHz</mem_clock>
-			<video_clock>835 MHz</video_clock>
-		</clocks>
-		<applications_clocks>
-			<graphics_clock>1189 MHz</graphics_clock>
-			<mem_clock>715 MHz</mem_clock>
-		</applications_clocks>
-		<default_applications_clocks>
-			<graphics_clock>1189 MHz</graphics_clock>
-			<mem_clock>715 MHz</mem_clock>
-		</default_applications_clocks>
-		<max_clocks>
-			<graphics_clock>1328 MHz</graphics_clock>
-			<sm_clock>1328 MHz</sm_clock>
-			<mem_clock>715 MHz</mem_clock>
-			<video_clock>1328 MHz</video_clock>
-		</max_clocks>
-		<clock_policy>
-			<auto_boost>N/A</auto_boost>
-			<auto_boost_default>N/A</auto_boost_default>
-		</clock_policy>
-		<supported_clocks>
-			<supported_mem_clock>
-				<value>715 MHz</value>
-				<supported_graphics_clock>1328 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1316 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1303 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1290 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1278 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1265 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1252 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1240 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1227 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1215 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1202 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1189 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1177 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1164 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1151 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1139 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1126 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1113 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1101 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1088 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1075 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1063 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1050 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1037 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1025 MHz</supported_graphics_clock>
-				<supported_graphics_clock>1012 MHz</supported_graphics_clock>
-				<supported_graphics_clock>999 MHz</supported_graphics_clock>
-				<supported_graphics_clock>987 MHz</supported_graphics_clock>
-				<supported_graphics_clock>974 MHz</supported_graphics_clock>
-				<supported_graphics_clock>961 MHz</supported_graphics_clock>
-				<supported_graphics_clock>949 MHz</supported_graphics_clock>
-				<supported_graphics_clock>936 MHz</supported_graphics_clock>
-				<supported_graphics_clock>923 MHz</supported_graphics_clock>
-				<supported_graphics_clock>911 MHz</supported_graphics_clock>
-				<supported_graphics_clock>898 MHz</supported_graphics_clock>
-				<supported_graphics_clock>885 MHz</supported_graphics_clock>
-				<supported_graphics_clock>873 MHz</supported_graphics_clock>
-				<supported_graphics_clock>860 MHz</supported_graphics_clock>
-				<supported_graphics_clock>847 MHz</supported_graphics_clock>
-				<supported_graphics_clock>835 MHz</supported_graphics_clock>
-				<supported_graphics_clock>822 MHz</supported_graphics_clock>
-				<supported_graphics_clock>810 MHz</supported_graphics_clock>
-				<supported_graphics_clock>797 MHz</supported_graphics_clock>
-				<supported_graphics_clock>784 MHz</supported_graphics_clock>
-				<supported_graphics_clock>772 MHz</supported_graphics_clock>
-				<supported_graphics_clock>759 MHz</supported_graphics_clock>
-				<supported_graphics_clock>746 MHz</supported_graphics_clock>
-				<supported_graphics_clock>734 MHz</supported_graphics_clock>
-				<supported_graphics_clock>721 MHz</supported_graphics_clock>
-				<supported_graphics_clock>708 MHz</supported_graphics_clock>
-				<supported_graphics_clock>696 MHz</supported_graphics_clock>
-				<supported_graphics_clock>683 MHz</supported_graphics_clock>
-				<supported_graphics_clock>670 MHz</supported_graphics_clock>
-				<supported_graphics_clock>658 MHz</supported_graphics_clock>
-				<supported_graphics_clock>645 MHz</supported_graphics_clock>
-				<supported_graphics_clock>632 MHz</supported_graphics_clock>
-				<supported_graphics_clock>620 MHz</supported_graphics_clock>
-				<supported_graphics_clock>607 MHz</supported_graphics_clock>
-				<supported_graphics_clock>594 MHz</supported_graphics_clock>
-				<supported_graphics_clock>582 MHz</supported_graphics_clock>
-				<supported_graphics_clock>569 MHz</supported_graphics_clock>
-				<supported_graphics_clock>556 MHz</supported_graphics_clock>
-				<supported_graphics_clock>544 MHz</supported_graphics_clock>
-			</supported_mem_clock>
-		</supported_clocks>
-		<processes>
-		</processes>
-		<accounted_processes>
-		</accounted_processes>
-	</gpu>
-
-</nvidia_smi_log>
\ No newline at end of file