YARN-9127. Create more tests to verify GpuDeviceInformationParser. Contributed by Peter Bacsko

(cherry picked from commit 18ee1092b4)
This commit is contained in:
Szilard Nemeth 2019-07-15 11:59:11 +02:00
parent 2fcbdf4131
commit 28d6a453a9
13 changed files with 978 additions and 601 deletions

View File

@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~ http://www.apache.org/licenses/LICENSE-2.0
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<configuration>
<property>
<name>yarn.resource-types</name>
<value>yarn.io/fpga</value>
</property>
<property>
<name>yarn.resource-types.yarn.io/fpga.units</name>
<value>G</value>
</property>
</configuration>

View File

@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~ http://www.apache.org/licenses/LICENSE-2.0
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<configuration>
<property>
<name>yarn.resource-types</name>
<value>yarn.io/gpu</value>
</property>
<property>
<name>yarn.resource-types.yarn.io/gpu.units</name>
<value>G</value>
</property>
</configuration>

View File

@ -64,7 +64,6 @@ public class GpuDiscoverer {
private Configuration conf = null;
private String pathOfGpuBinary = null;
private Map<String, String> environment = new HashMap<>();
private GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
private int numOfErrorExecutionSinceLastSucceed = 0;
private GpuDeviceInformation lastDiscoveredGpuInformation = null;
@ -125,6 +124,7 @@ public class GpuDiscoverer {
try {
output = Shell.execCommand(environment,
new String[] { pathOfGpuBinary, "-x", "-q" }, MAX_EXEC_TIMEOUT_MS);
GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
lastDiscoveredGpuInformation = parser.parseXml(output);
numOfErrorExecutionSinceLastSucceed = 0;
return lastDiscoveredGpuInformation;

View File

@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
@ -31,14 +32,10 @@ import java.util.List;
@InterfaceStability.Unstable
@XmlRootElement(name = "nvidia_smi_log")
public class GpuDeviceInformation {
List<PerGpuDeviceInformation> gpus;
private List<PerGpuDeviceInformation> gpus;
private String driverVersion = "N/A";
String driverVersion = "N/A";
// More fields like topology information could be added when needed.
// ...
@javax.xml.bind.annotation.XmlElement(name = "gpu")
@XmlElement(name = "gpu")
public List<PerGpuDeviceInformation> getGpus() {
return gpus;
}
@ -47,7 +44,7 @@ public class GpuDeviceInformation {
this.gpus = gpus;
}
@javax.xml.bind.annotation.XmlElement(name = "driver_version")
@XmlElement(name = "driver_version")
public String getDriverVersion() {
return driverVersion;
}
@ -59,8 +56,9 @@ public class GpuDeviceInformation {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("=== Gpus in the system ===\n").append("\tDriver Version:").append(
getDriverVersion()).append("\n");
sb.append("=== GPUs in the system ===\n")
.append("\tDriver Version:")
.append(getDriverVersion()).append("\n");
if (gpus != null) {
for (PerGpuDeviceInformation gpu : gpus) {

View File

@ -24,13 +24,11 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.sax.SAXSource;
import java.io.StringReader;
@ -46,39 +44,39 @@ public class GpuDeviceInformationParser {
public static final String GPU_SCRIPT_REFERENCE = "GPU device detection " +
"script";
private Unmarshaller unmarshaller = null;
private XMLReader xmlReader = null;
private final Unmarshaller unmarshaller;
private final XMLReader xmlReader;
private void init()
throws SAXException, ParserConfigurationException, JAXBException {
public GpuDeviceInformationParser() throws YarnException {
try {
final SAXParserFactory parserFactory = initSaxParserFactory();
final JAXBContext jaxbContext = JAXBContext.newInstance(
GpuDeviceInformation.class);
this.xmlReader = parserFactory.newSAXParser().getXMLReader();
this.unmarshaller = jaxbContext.createUnmarshaller();
} catch (Exception e) {
String msg = "Exception while initializing parser for " +
GPU_SCRIPT_REFERENCE;
LOG.error(msg, e);
throw new YarnException(msg, e);
}
}
/**
* Disable external-dtd since by default nvidia-smi output contains
* &lt;!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd"> in header.
*/
private SAXParserFactory initSaxParserFactory() throws Exception {
SAXParserFactory spf = SAXParserFactory.newInstance();
// Disable external-dtd since by default nvidia-smi output contains
// <!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd"> in header
spf.setFeature(
"http://apache.org/xml/features/nonvalidating/load-external-dtd",
false);
spf.setFeature("http://xml.org/sax/features/validation", false);
JAXBContext jaxbContext = JAXBContext.newInstance(
GpuDeviceInformation.class);
this.xmlReader = spf.newSAXParser().getXMLReader();
this.unmarshaller = jaxbContext.createUnmarshaller();
return spf;
}
public synchronized GpuDeviceInformation parseXml(String xmlContent)
throws YarnException {
if (unmarshaller == null) {
try {
init();
} catch (SAXException | ParserConfigurationException | JAXBException e) {
String msg = "Exception while initializing parser for " +
GPU_SCRIPT_REFERENCE;
LOG.error(msg, e);
throw new YarnException(e);
}
}
InputSource inputSource = new InputSource(new StringReader(xmlContent));
SAXSource source = new SAXSource(xmlReader, inputSource);
try {

View File

@ -33,7 +33,6 @@ import javax.xml.bind.annotation.adapters.XmlAdapter;
@InterfaceStability.Unstable
@XmlRootElement(name = "gpu")
public class PerGpuDeviceInformation {
private String productName = "N/A";
private String uuid = "N/A";
private int minorNumber = -1;

View File

@ -29,8 +29,8 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
@InterfaceStability.Unstable
@XmlRootElement(name = "fb_memory_usage")
public class PerGpuMemoryUsage {
long usedMemoryMiB = -1L;
long availMemoryMiB = -1L;
private long usedMemoryMiB = -1L;
private long availMemoryMiB = -1L;
@XmlJavaTypeAdapter(PerGpuDeviceInformation.StrToMemAdapter.class)
@XmlElement(name = "used")
@ -53,6 +53,9 @@ public class PerGpuMemoryUsage {
}
public long getTotalMemoryMiB() {
if (usedMemoryMiB == -1 && availMemoryMiB == -1) {
return -1;
}
return usedMemoryMiB + availMemoryMiB;
}
}

View File

@ -20,31 +20,160 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.io.File;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
public class TestGpuDeviceInformationParser {
private static final String UTF_8 = "UTF-8";
private static final double DELTA = 1e-6;
@Rule
public ExpectedException expected = ExpectedException.none();
@Test
public void testParse() throws IOException, YarnException {
File f = new File("src/test/resources/nvidia-smi-sample-xml-output");
String s = FileUtils.readFileToString(f, "UTF-8");
File f = new File("src/test/resources/nvidia-smi-sample-output.xml");
String s = FileUtils.readFileToString(f, UTF_8);
GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
GpuDeviceInformation info = parser.parseXml(s);
Assert.assertEquals("375.66", info.getDriverVersion());
Assert.assertEquals(2, info.getGpus().size());
PerGpuDeviceInformation gpu1 = info.getGpus().get(1);
Assert.assertEquals("Tesla P100-PCIE-12GB", gpu1.getProductName());
Assert.assertEquals(12193, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
Assert.assertEquals(10.3f,
gpu1.getGpuUtilizations().getOverallGpuUtilization(), 1e-6);
Assert.assertEquals(34f, gpu1.getTemperature().getCurrentGpuTemp(), 1e-6);
Assert.assertEquals(85f, gpu1.getTemperature().getMaxGpuTemp(), 1e-6);
Assert.assertEquals(82f, gpu1.getTemperature().getSlowThresholdGpuTemp(),
1e-6);
assertEquals("375.66", info.getDriverVersion());
assertEquals(2, info.getGpus().size());
assertFirstGpu(info.getGpus().get(0));
assertSecondGpu(info.getGpus().get(1));
}
@Test
public void testParseExcerpt() throws IOException, YarnException {
File f = new File("src/test/resources/nvidia-smi-output-excerpt.xml");
String s = FileUtils.readFileToString(f, UTF_8);
GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
GpuDeviceInformation info = parser.parseXml(s);
assertEquals("375.66", info.getDriverVersion());
assertEquals(2, info.getGpus().size());
assertFirstGpu(info.getGpus().get(0));
assertSecondGpu(info.getGpus().get(1));
}
@Test
public void testParseConsecutivelyWithSameParser()
throws IOException, YarnException {
File f = new File("src/test/resources/nvidia-smi-sample-output.xml");
String s = FileUtils.readFileToString(f, UTF_8);
for (int i = 0; i < 3; i++) {
GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
GpuDeviceInformation info = parser.parseXml(s);
assertEquals("375.66", info.getDriverVersion());
assertEquals(2, info.getGpus().size());
assertFirstGpu(info.getGpus().get(0));
assertSecondGpu(info.getGpus().get(1));
}
}
@Test
public void testParseEmptyString() throws YarnException {
expected.expect(YarnException.class);
GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
parser.parseXml("");
}
@Test
public void testParseInvalidRootElement() throws YarnException {
expected.expect(YarnException.class);
GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
parser.parseXml("<nvidia_smiiiii></nvidia_smiiiii");
}
@Test
public void testParseMissingTags() throws IOException, YarnException {
File f = new File("src/test/resources/nvidia-smi-output-missing-tags.xml");
String s = FileUtils.readFileToString(f, UTF_8);
GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
GpuDeviceInformation info = parser.parseXml(s);
assertEquals("375.66", info.getDriverVersion());
assertEquals(1, info.getGpus().size());
PerGpuDeviceInformation gpu = info.getGpus().get(0);
assertEquals("N/A", gpu.getProductName());
assertEquals("N/A", gpu.getUuid());
assertEquals(-1, gpu.getMinorNumber());
assertNull(gpu.getGpuMemoryUsage());
assertNull(gpu.getTemperature());
assertNull(gpu.getGpuUtilizations());
}
@Test
public void testParseMissingInnerTags() throws IOException, YarnException {
File f =new File("src/test/resources/nvidia-smi-output-missing-tags2.xml");
String s = FileUtils.readFileToString(f, UTF_8);
GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
GpuDeviceInformation info = parser.parseXml(s);
assertEquals("375.66", info.getDriverVersion());
assertEquals(2, info.getGpus().size());
PerGpuDeviceInformation gpu = info.getGpus().get(0);
assertEquals("Tesla P100-PCIE-12GB", gpu.getProductName());
assertEquals("GPU-28604e81-21ec-cc48-6759-bf2648b22e16", gpu.getUuid());
assertEquals(0, gpu.getMinorNumber());
assertEquals(-1, gpu.getGpuMemoryUsage().getTotalMemoryMiB());
assertEquals(-1, (long) gpu.getGpuMemoryUsage().getUsedMemoryMiB());
assertEquals(-1, (long) gpu.getGpuMemoryUsage().getAvailMemoryMiB());
assertEquals(0f,
gpu.getGpuUtilizations().getOverallGpuUtilization(), DELTA);
assertEquals(Float.MIN_VALUE,
gpu.getTemperature().getCurrentGpuTemp(), DELTA);
assertEquals(Float.MIN_VALUE,
gpu.getTemperature().getMaxGpuTemp(), DELTA);
assertEquals(Float.MIN_VALUE,
gpu.getTemperature().getSlowThresholdGpuTemp(),
DELTA);
assertSecondGpu(info.getGpus().get(1));
}
private void assertFirstGpu(PerGpuDeviceInformation gpu) {
assertEquals("Tesla P100-PCIE-12GB", gpu.getProductName());
assertEquals("GPU-28604e81-21ec-cc48-6759-bf2648b22e16", gpu.getUuid());
assertEquals(0, gpu.getMinorNumber());
assertEquals(11567, gpu.getGpuMemoryUsage().getTotalMemoryMiB());
assertEquals(11400, (long) gpu.getGpuMemoryUsage().getUsedMemoryMiB());
assertEquals(167, (long) gpu.getGpuMemoryUsage().getAvailMemoryMiB());
assertEquals(33.4f,
gpu.getGpuUtilizations().getOverallGpuUtilization(), DELTA);
assertEquals(31f, gpu.getTemperature().getCurrentGpuTemp(), DELTA);
assertEquals(80f, gpu.getTemperature().getMaxGpuTemp(), DELTA);
assertEquals(88f, gpu.getTemperature().getSlowThresholdGpuTemp(),
DELTA);
}
private void assertSecondGpu(PerGpuDeviceInformation gpu) {
assertEquals("Tesla P100-PCIE-12GB_2", gpu.getProductName());
assertEquals("GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3", gpu.getUuid());
assertEquals(1, gpu.getMinorNumber());
assertEquals(12290, gpu.getGpuMemoryUsage().getTotalMemoryMiB());
assertEquals(11800, (long) gpu.getGpuMemoryUsage().getUsedMemoryMiB());
assertEquals(490, (long) gpu.getGpuMemoryUsage().getAvailMemoryMiB());
assertEquals(10.3f,
gpu.getGpuUtilizations().getOverallGpuUtilization(), DELTA);
assertEquals(34f, gpu.getTemperature().getCurrentGpuTemp(), DELTA);
assertEquals(85f, gpu.getTemperature().getMaxGpuTemp(), DELTA);
assertEquals(82f, gpu.getTemperature().getSlowThresholdGpuTemp(),
DELTA);
}
}

View File

@ -0,0 +1,71 @@
<?xml version="1.0" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~ http://www.apache.org/licenses/LICENSE-2.0
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
<nvidia_smi_log>
<timestamp>Wed Sep 6 21:52:51 2017</timestamp>
<driver_version>375.66</driver_version>
<attached_gpus>2</attached_gpus>
<gpu id="0000:04:00.0">
<product_name>Tesla P100-PCIE-12GB</product_name>
<product_brand>Tesla</product_brand>
<uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
<minor_number>0</minor_number>
<fb_memory_usage>
<total>11567 MiB</total>
<used>11400 MiB</used>
<free>167 MiB</free>
</fb_memory_usage>
<utilization>
<gpu_util>33.4 %</gpu_util>
<memory_util>0 %</memory_util>
<encoder_util>0 %</encoder_util>
<decoder_util>0 %</decoder_util>
</utilization>
<temperature>
<gpu_temp>31 C</gpu_temp>
<gpu_temp_max_threshold>80 C</gpu_temp_max_threshold>
<gpu_temp_slow_threshold>88 C</gpu_temp_slow_threshold>
</temperature>
</gpu>
<gpu id="0000:82:00.0">
<product_name>Tesla P100-PCIE-12GB_2</product_name>
<product_brand>Tesla</product_brand>
<uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
<minor_number>1</minor_number>
<fb_memory_usage>
<total>12290 MiB</total>
<used>11800 MiB</used>
<free>490 MiB</free>
</fb_memory_usage>
<compute_mode>Default</compute_mode>
<utilization>
<gpu_util>10.3 %</gpu_util>
<memory_util>0 %</memory_util>
<encoder_util>0 %</encoder_util>
<decoder_util>0 %</decoder_util>
</utilization>
<temperature>
<gpu_temp>34 C</gpu_temp>
<gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
<gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
</temperature>
</gpu>
</nvidia_smi_log>

View File

@ -0,0 +1,28 @@
<?xml version="1.0" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~ http://www.apache.org/licenses/LICENSE-2.0
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
<nvidia_smi_log>
<timestamp>Wed Sep 6 21:52:51 2017</timestamp>
<driver_version>375.66</driver_version>
<attached_gpus>2</attached_gpus>
<gpu id="">
<product_brand>Tesla</product_brand>
</gpu>
</nvidia_smi_log>

View File

@ -0,0 +1,61 @@
<?xml version="1.0" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~ http://www.apache.org/licenses/LICENSE-2.0
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
<nvidia_smi_log>
<timestamp>Wed Sep 6 21:52:51 2017</timestamp>
<driver_version>375.66</driver_version>
<attached_gpus>2</attached_gpus>
<gpu id="0000:04:00.0">
<product_name>Tesla P100-PCIE-12GB</product_name>
<product_brand>Tesla</product_brand>
<uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
<minor_number>0</minor_number>
<fb_memory_usage>
</fb_memory_usage>
<utilization>
</utilization>
<temperature>
</temperature>
</gpu>
<gpu id="0000:82:00.0">
<product_name>Tesla P100-PCIE-12GB_2</product_name>
<product_brand>Tesla</product_brand>
<uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
<minor_number>1</minor_number>
<fb_memory_usage>
<total>12290 MiB</total>
<used>11800 MiB</used>
<free>490 MiB</free>
</fb_memory_usage>
<compute_mode>Default</compute_mode>
<utilization>
<gpu_util>10.3 %</gpu_util>
<memory_util>0 %</memory_util>
<encoder_util>0 %</encoder_util>
<decoder_util>0 %</decoder_util>
</utilization>
<temperature>
<gpu_temp>34 C</gpu_temp>
<gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
<gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
</temperature>
</gpu>
</nvidia_smi_log>

View File

@ -0,0 +1,547 @@
<?xml version="1.0" ?>
<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<nvidia_smi_log>
<timestamp>Wed Sep 6 21:52:51 2017</timestamp>
<driver_version>375.66</driver_version>
<attached_gpus>2</attached_gpus>
<gpu id="0000:04:00.0">
<product_name>Tesla P100-PCIE-12GB</product_name>
<product_brand>Tesla</product_brand>
<display_mode>Disabled</display_mode>
<display_active>Disabled</display_active>
<persistence_mode>Disabled</persistence_mode>
<accounting_mode>Disabled</accounting_mode>
<accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
<driver_model>
<current_dm>N/A</current_dm>
<pending_dm>N/A</pending_dm>
</driver_model>
<serial>0320717030197</serial>
<uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
<minor_number>0</minor_number>
<vbios_version>86.00.3A.00.02</vbios_version>
<multigpu_board>No</multigpu_board>
<board_id>0x400</board_id>
<gpu_part_number>900-2H400-0110-030</gpu_part_number>
<inforom_version>
<img_version>H400.0202.00.01</img_version>
<oem_object>1.1</oem_object>
<ecc_object>4.1</ecc_object>
<pwr_object>N/A</pwr_object>
</inforom_version>
<gpu_operation_mode>
<current_gom>N/A</current_gom>
<pending_gom>N/A</pending_gom>
</gpu_operation_mode>
<gpu_virtualization_mode>
<virtualization_mode>None</virtualization_mode>
</gpu_virtualization_mode>
<pci>
<pci_bus>04</pci_bus>
<pci_device>00</pci_device>
<pci_domain>0000</pci_domain>
<pci_device_id>15F710DE</pci_device_id>
<pci_bus_id>0000:04:00.0</pci_bus_id>
<pci_sub_system_id>11DA10DE</pci_sub_system_id>
<pci_gpu_link_info>
<pcie_gen>
<max_link_gen>3</max_link_gen>
<current_link_gen>3</current_link_gen>
</pcie_gen>
<link_widths>
<max_link_width>16x</max_link_width>
<current_link_width>16x</current_link_width>
</link_widths>
</pci_gpu_link_info>
<pci_bridge_chip>
<bridge_chip_type>N/A</bridge_chip_type>
<bridge_chip_fw>N/A</bridge_chip_fw>
</pci_bridge_chip>
<replay_counter>0</replay_counter>
<tx_util>0 KB/s</tx_util>
<rx_util>0 KB/s</rx_util>
</pci>
<fan_speed>N/A</fan_speed>
<performance_state>P0</performance_state>
<clocks_throttle_reasons>
<clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
<clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
<clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
<clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
<clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
<clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
</clocks_throttle_reasons>
<fb_memory_usage>
<total>11567 MiB</total>
<used>11400 MiB</used>
<free>167 MiB</free>
</fb_memory_usage>
<bar1_memory_usage>
<total>16384 MiB</total>
<used>2 MiB</used>
<free>16382 MiB</free>
</bar1_memory_usage>
<compute_mode>Default</compute_mode>
<utilization>
<gpu_util>33.4 %</gpu_util>
<memory_util>0 %</memory_util>
<encoder_util>0 %</encoder_util>
<decoder_util>0 %</decoder_util>
</utilization>
<encoder_stats>
<session_count>0</session_count>
<average_fps>0</average_fps>
<average_latency>0 ms</average_latency>
</encoder_stats>
<ecc_mode>
<current_ecc>Enabled</current_ecc>
<pending_ecc>Enabled</pending_ecc>
</ecc_mode>
<ecc_errors>
<volatile>
<single_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</single_bit>
<double_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</double_bit>
</volatile>
<aggregate>
<single_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</single_bit>
<double_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</double_bit>
</aggregate>
</ecc_errors>
<retired_pages>
<multiple_single_bit_retirement>
<retired_count>0</retired_count>
<retired_page_addresses>
</retired_page_addresses>
</multiple_single_bit_retirement>
<double_bit_retirement>
<retired_count>0</retired_count>
<retired_page_addresses>
</retired_page_addresses>
</double_bit_retirement>
<pending_retirement>No</pending_retirement>
</retired_pages>
<temperature>
<gpu_temp>31 C</gpu_temp>
<gpu_temp_max_threshold>80 C</gpu_temp_max_threshold>
<gpu_temp_slow_threshold>88 C</gpu_temp_slow_threshold>
</temperature>
<power_readings>
<power_state>P0</power_state>
<power_management>Supported</power_management>
<power_draw>24.84 W</power_draw>
<power_limit>250.00 W</power_limit>
<default_power_limit>250.00 W</default_power_limit>
<enforced_power_limit>250.00 W</enforced_power_limit>
<min_power_limit>125.00 W</min_power_limit>
<max_power_limit>250.00 W</max_power_limit>
</power_readings>
<clocks>
<graphics_clock>405 MHz</graphics_clock>
<sm_clock>405 MHz</sm_clock>
<mem_clock>715 MHz</mem_clock>
<video_clock>835 MHz</video_clock>
</clocks>
<applications_clocks>
<graphics_clock>1189 MHz</graphics_clock>
<mem_clock>715 MHz</mem_clock>
</applications_clocks>
<default_applications_clocks>
<graphics_clock>1189 MHz</graphics_clock>
<mem_clock>715 MHz</mem_clock>
</default_applications_clocks>
<max_clocks>
<graphics_clock>1328 MHz</graphics_clock>
<sm_clock>1328 MHz</sm_clock>
<mem_clock>715 MHz</mem_clock>
<video_clock>1328 MHz</video_clock>
</max_clocks>
<clock_policy>
<auto_boost>N/A</auto_boost>
<auto_boost_default>N/A</auto_boost_default>
</clock_policy>
<supported_clocks>
<supported_mem_clock>
<value>715 MHz</value>
<supported_graphics_clock>1328 MHz</supported_graphics_clock>
<supported_graphics_clock>1316 MHz</supported_graphics_clock>
<supported_graphics_clock>1303 MHz</supported_graphics_clock>
<supported_graphics_clock>1290 MHz</supported_graphics_clock>
<supported_graphics_clock>1278 MHz</supported_graphics_clock>
<supported_graphics_clock>1265 MHz</supported_graphics_clock>
<supported_graphics_clock>1252 MHz</supported_graphics_clock>
<supported_graphics_clock>1240 MHz</supported_graphics_clock>
<supported_graphics_clock>1227 MHz</supported_graphics_clock>
<supported_graphics_clock>1215 MHz</supported_graphics_clock>
<supported_graphics_clock>1202 MHz</supported_graphics_clock>
<supported_graphics_clock>1189 MHz</supported_graphics_clock>
<supported_graphics_clock>1177 MHz</supported_graphics_clock>
<supported_graphics_clock>1164 MHz</supported_graphics_clock>
<supported_graphics_clock>1151 MHz</supported_graphics_clock>
<supported_graphics_clock>1139 MHz</supported_graphics_clock>
<supported_graphics_clock>1126 MHz</supported_graphics_clock>
<supported_graphics_clock>1113 MHz</supported_graphics_clock>
<supported_graphics_clock>1101 MHz</supported_graphics_clock>
<supported_graphics_clock>1088 MHz</supported_graphics_clock>
<supported_graphics_clock>1075 MHz</supported_graphics_clock>
<supported_graphics_clock>1063 MHz</supported_graphics_clock>
<supported_graphics_clock>1050 MHz</supported_graphics_clock>
<supported_graphics_clock>1037 MHz</supported_graphics_clock>
<supported_graphics_clock>1025 MHz</supported_graphics_clock>
<supported_graphics_clock>1012 MHz</supported_graphics_clock>
<supported_graphics_clock>999 MHz</supported_graphics_clock>
<supported_graphics_clock>987 MHz</supported_graphics_clock>
<supported_graphics_clock>974 MHz</supported_graphics_clock>
<supported_graphics_clock>961 MHz</supported_graphics_clock>
<supported_graphics_clock>949 MHz</supported_graphics_clock>
<supported_graphics_clock>936 MHz</supported_graphics_clock>
<supported_graphics_clock>923 MHz</supported_graphics_clock>
<supported_graphics_clock>911 MHz</supported_graphics_clock>
<supported_graphics_clock>898 MHz</supported_graphics_clock>
<supported_graphics_clock>885 MHz</supported_graphics_clock>
<supported_graphics_clock>873 MHz</supported_graphics_clock>
<supported_graphics_clock>860 MHz</supported_graphics_clock>
<supported_graphics_clock>847 MHz</supported_graphics_clock>
<supported_graphics_clock>835 MHz</supported_graphics_clock>
<supported_graphics_clock>822 MHz</supported_graphics_clock>
<supported_graphics_clock>810 MHz</supported_graphics_clock>
<supported_graphics_clock>797 MHz</supported_graphics_clock>
<supported_graphics_clock>784 MHz</supported_graphics_clock>
<supported_graphics_clock>772 MHz</supported_graphics_clock>
<supported_graphics_clock>759 MHz</supported_graphics_clock>
<supported_graphics_clock>746 MHz</supported_graphics_clock>
<supported_graphics_clock>734 MHz</supported_graphics_clock>
<supported_graphics_clock>721 MHz</supported_graphics_clock>
<supported_graphics_clock>708 MHz</supported_graphics_clock>
<supported_graphics_clock>696 MHz</supported_graphics_clock>
<supported_graphics_clock>683 MHz</supported_graphics_clock>
<supported_graphics_clock>670 MHz</supported_graphics_clock>
<supported_graphics_clock>658 MHz</supported_graphics_clock>
<supported_graphics_clock>645 MHz</supported_graphics_clock>
<supported_graphics_clock>632 MHz</supported_graphics_clock>
<supported_graphics_clock>620 MHz</supported_graphics_clock>
<supported_graphics_clock>607 MHz</supported_graphics_clock>
<supported_graphics_clock>594 MHz</supported_graphics_clock>
<supported_graphics_clock>582 MHz</supported_graphics_clock>
<supported_graphics_clock>569 MHz</supported_graphics_clock>
<supported_graphics_clock>556 MHz</supported_graphics_clock>
<supported_graphics_clock>544 MHz</supported_graphics_clock>
</supported_mem_clock>
</supported_clocks>
<processes>
</processes>
<accounted_processes>
</accounted_processes>
</gpu>
<gpu id="0000:82:00.0">
<product_name>Tesla P100-PCIE-12GB_2</product_name>
<product_brand>Tesla</product_brand>
<display_mode>Disabled</display_mode>
<display_active>Disabled</display_active>
<persistence_mode>Disabled</persistence_mode>
<accounting_mode>Disabled</accounting_mode>
<accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
<driver_model>
<current_dm>N/A</current_dm>
<pending_dm>N/A</pending_dm>
</driver_model>
<serial>0320717031755</serial>
<uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
<minor_number>1</minor_number>
<vbios_version>86.00.3A.00.02</vbios_version>
<multigpu_board>No</multigpu_board>
<board_id>0x8200</board_id>
<gpu_part_number>900-2H400-0110-030</gpu_part_number>
<inforom_version>
<img_version>H400.0202.00.01</img_version>
<oem_object>1.1</oem_object>
<ecc_object>4.1</ecc_object>
<pwr_object>N/A</pwr_object>
</inforom_version>
<gpu_operation_mode>
<current_gom>N/A</current_gom>
<pending_gom>N/A</pending_gom>
</gpu_operation_mode>
<gpu_virtualization_mode>
<virtualization_mode>None</virtualization_mode>
</gpu_virtualization_mode>
<pci>
<pci_bus>82</pci_bus>
<pci_device>00</pci_device>
<pci_domain>0000</pci_domain>
<pci_device_id>15F710DE</pci_device_id>
<pci_bus_id>0000:82:00.0</pci_bus_id>
<pci_sub_system_id>11DA10DE</pci_sub_system_id>
<pci_gpu_link_info>
<pcie_gen>
<max_link_gen>3</max_link_gen>
<current_link_gen>3</current_link_gen>
</pcie_gen>
<link_widths>
<max_link_width>16x</max_link_width>
<current_link_width>16x</current_link_width>
</link_widths>
</pci_gpu_link_info>
<pci_bridge_chip>
<bridge_chip_type>N/A</bridge_chip_type>
<bridge_chip_fw>N/A</bridge_chip_fw>
</pci_bridge_chip>
<replay_counter>0</replay_counter>
<tx_util>0 KB/s</tx_util>
<rx_util>0 KB/s</rx_util>
</pci>
<fan_speed>N/A</fan_speed>
<performance_state>P0</performance_state>
<clocks_throttle_reasons>
<clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
<clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
<clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
<clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
<clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
<clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
</clocks_throttle_reasons>
<fb_memory_usage>
<total>12290 MiB</total>
<used>11800 MiB</used>
<free>490 MiB</free>
</fb_memory_usage>
<bar1_memory_usage>
<total>16384 MiB</total>
<used>2 MiB</used>
<free>16382 MiB</free>
</bar1_memory_usage>
<compute_mode>Default</compute_mode>
<utilization>
<gpu_util>10.3 %</gpu_util>
<memory_util>0 %</memory_util>
<encoder_util>0 %</encoder_util>
<decoder_util>0 %</decoder_util>
</utilization>
<encoder_stats>
<session_count>0</session_count>
<average_fps>0</average_fps>
<average_latency>0 ms</average_latency>
</encoder_stats>
<ecc_mode>
<current_ecc>Enabled</current_ecc>
<pending_ecc>Enabled</pending_ecc>
</ecc_mode>
<ecc_errors>
<volatile>
<single_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</single_bit>
<double_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</double_bit>
</volatile>
<aggregate>
<single_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</single_bit>
<double_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</double_bit>
</aggregate>
</ecc_errors>
<retired_pages>
<multiple_single_bit_retirement>
<retired_count>0</retired_count>
<retired_page_addresses>
</retired_page_addresses>
</multiple_single_bit_retirement>
<double_bit_retirement>
<retired_count>0</retired_count>
<retired_page_addresses>
</retired_page_addresses>
</double_bit_retirement>
<pending_retirement>No</pending_retirement>
</retired_pages>
<temperature>
<gpu_temp>34 C</gpu_temp>
<gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
<gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
</temperature>
<power_readings>
<power_state>P0</power_state>
<power_management>Supported</power_management>
<power_draw>25.54 W</power_draw>
<power_limit>250.00 W</power_limit>
<default_power_limit>250.00 W</default_power_limit>
<enforced_power_limit>250.00 W</enforced_power_limit>
<min_power_limit>125.00 W</min_power_limit>
<max_power_limit>250.00 W</max_power_limit>
</power_readings>
<clocks>
<graphics_clock>405 MHz</graphics_clock>
<sm_clock>405 MHz</sm_clock>
<mem_clock>715 MHz</mem_clock>
<video_clock>835 MHz</video_clock>
</clocks>
<applications_clocks>
<graphics_clock>1189 MHz</graphics_clock>
<mem_clock>715 MHz</mem_clock>
</applications_clocks>
<default_applications_clocks>
<graphics_clock>1189 MHz</graphics_clock>
<mem_clock>715 MHz</mem_clock>
</default_applications_clocks>
<max_clocks>
<graphics_clock>1328 MHz</graphics_clock>
<sm_clock>1328 MHz</sm_clock>
<mem_clock>715 MHz</mem_clock>
<video_clock>1328 MHz</video_clock>
</max_clocks>
<clock_policy>
<auto_boost>N/A</auto_boost>
<auto_boost_default>N/A</auto_boost_default>
</clock_policy>
<supported_clocks>
<supported_mem_clock>
<value>715 MHz</value>
<supported_graphics_clock>1328 MHz</supported_graphics_clock>
<supported_graphics_clock>1316 MHz</supported_graphics_clock>
<supported_graphics_clock>1303 MHz</supported_graphics_clock>
<supported_graphics_clock>1290 MHz</supported_graphics_clock>
<supported_graphics_clock>1278 MHz</supported_graphics_clock>
<supported_graphics_clock>1265 MHz</supported_graphics_clock>
<supported_graphics_clock>1252 MHz</supported_graphics_clock>
<supported_graphics_clock>1240 MHz</supported_graphics_clock>
<supported_graphics_clock>1227 MHz</supported_graphics_clock>
<supported_graphics_clock>1215 MHz</supported_graphics_clock>
<supported_graphics_clock>1202 MHz</supported_graphics_clock>
<supported_graphics_clock>1189 MHz</supported_graphics_clock>
<supported_graphics_clock>1177 MHz</supported_graphics_clock>
<supported_graphics_clock>1164 MHz</supported_graphics_clock>
<supported_graphics_clock>1151 MHz</supported_graphics_clock>
<supported_graphics_clock>1139 MHz</supported_graphics_clock>
<supported_graphics_clock>1126 MHz</supported_graphics_clock>
<supported_graphics_clock>1113 MHz</supported_graphics_clock>
<supported_graphics_clock>1101 MHz</supported_graphics_clock>
<supported_graphics_clock>1088 MHz</supported_graphics_clock>
<supported_graphics_clock>1075 MHz</supported_graphics_clock>
<supported_graphics_clock>1063 MHz</supported_graphics_clock>
<supported_graphics_clock>1050 MHz</supported_graphics_clock>
<supported_graphics_clock>1037 MHz</supported_graphics_clock>
<supported_graphics_clock>1025 MHz</supported_graphics_clock>
<supported_graphics_clock>1012 MHz</supported_graphics_clock>
<supported_graphics_clock>999 MHz</supported_graphics_clock>
<supported_graphics_clock>987 MHz</supported_graphics_clock>
<supported_graphics_clock>974 MHz</supported_graphics_clock>
<supported_graphics_clock>961 MHz</supported_graphics_clock>
<supported_graphics_clock>949 MHz</supported_graphics_clock>
<supported_graphics_clock>936 MHz</supported_graphics_clock>
<supported_graphics_clock>923 MHz</supported_graphics_clock>
<supported_graphics_clock>911 MHz</supported_graphics_clock>
<supported_graphics_clock>898 MHz</supported_graphics_clock>
<supported_graphics_clock>885 MHz</supported_graphics_clock>
<supported_graphics_clock>873 MHz</supported_graphics_clock>
<supported_graphics_clock>860 MHz</supported_graphics_clock>
<supported_graphics_clock>847 MHz</supported_graphics_clock>
<supported_graphics_clock>835 MHz</supported_graphics_clock>
<supported_graphics_clock>822 MHz</supported_graphics_clock>
<supported_graphics_clock>810 MHz</supported_graphics_clock>
<supported_graphics_clock>797 MHz</supported_graphics_clock>
<supported_graphics_clock>784 MHz</supported_graphics_clock>
<supported_graphics_clock>772 MHz</supported_graphics_clock>
<supported_graphics_clock>759 MHz</supported_graphics_clock>
<supported_graphics_clock>746 MHz</supported_graphics_clock>
<supported_graphics_clock>734 MHz</supported_graphics_clock>
<supported_graphics_clock>721 MHz</supported_graphics_clock>
<supported_graphics_clock>708 MHz</supported_graphics_clock>
<supported_graphics_clock>696 MHz</supported_graphics_clock>
<supported_graphics_clock>683 MHz</supported_graphics_clock>
<supported_graphics_clock>670 MHz</supported_graphics_clock>
<supported_graphics_clock>658 MHz</supported_graphics_clock>
<supported_graphics_clock>645 MHz</supported_graphics_clock>
<supported_graphics_clock>632 MHz</supported_graphics_clock>
<supported_graphics_clock>620 MHz</supported_graphics_clock>
<supported_graphics_clock>607 MHz</supported_graphics_clock>
<supported_graphics_clock>594 MHz</supported_graphics_clock>
<supported_graphics_clock>582 MHz</supported_graphics_clock>
<supported_graphics_clock>569 MHz</supported_graphics_clock>
<supported_graphics_clock>556 MHz</supported_graphics_clock>
<supported_graphics_clock>544 MHz</supported_graphics_clock>
</supported_mem_clock>
</supported_clocks>
<processes>
</processes>
<accounted_processes>
</accounted_processes>
</gpu>
</nvidia_smi_log>

View File

@ -1,547 +0,0 @@
<?xml version="1.0" ?>
<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<nvidia_smi_log>
<timestamp>Wed Sep 6 21:52:51 2017</timestamp>
<driver_version>375.66</driver_version>
<attached_gpus>2</attached_gpus>
<gpu id="0000:04:00.0">
<product_name>Tesla P100-PCIE-12GB</product_name>
<product_brand>Tesla</product_brand>
<display_mode>Disabled</display_mode>
<display_active>Disabled</display_active>
<persistence_mode>Disabled</persistence_mode>
<accounting_mode>Disabled</accounting_mode>
<accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
<driver_model>
<current_dm>N/A</current_dm>
<pending_dm>N/A</pending_dm>
</driver_model>
<serial>0320717030197</serial>
<uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
<minor_number>0</minor_number>
<vbios_version>86.00.3A.00.02</vbios_version>
<multigpu_board>No</multigpu_board>
<board_id>0x400</board_id>
<gpu_part_number>900-2H400-0110-030</gpu_part_number>
<inforom_version>
<img_version>H400.0202.00.01</img_version>
<oem_object>1.1</oem_object>
<ecc_object>4.1</ecc_object>
<pwr_object>N/A</pwr_object>
</inforom_version>
<gpu_operation_mode>
<current_gom>N/A</current_gom>
<pending_gom>N/A</pending_gom>
</gpu_operation_mode>
<gpu_virtualization_mode>
<virtualization_mode>None</virtualization_mode>
</gpu_virtualization_mode>
<pci>
<pci_bus>04</pci_bus>
<pci_device>00</pci_device>
<pci_domain>0000</pci_domain>
<pci_device_id>15F710DE</pci_device_id>
<pci_bus_id>0000:04:00.0</pci_bus_id>
<pci_sub_system_id>11DA10DE</pci_sub_system_id>
<pci_gpu_link_info>
<pcie_gen>
<max_link_gen>3</max_link_gen>
<current_link_gen>3</current_link_gen>
</pcie_gen>
<link_widths>
<max_link_width>16x</max_link_width>
<current_link_width>16x</current_link_width>
</link_widths>
</pci_gpu_link_info>
<pci_bridge_chip>
<bridge_chip_type>N/A</bridge_chip_type>
<bridge_chip_fw>N/A</bridge_chip_fw>
</pci_bridge_chip>
<replay_counter>0</replay_counter>
<tx_util>0 KB/s</tx_util>
<rx_util>0 KB/s</rx_util>
</pci>
<fan_speed>N/A</fan_speed>
<performance_state>P0</performance_state>
<clocks_throttle_reasons>
<clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
<clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
<clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
<clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
<clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
<clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
</clocks_throttle_reasons>
<fb_memory_usage>
<total>12193 MiB</total>
<used>0 MiB</used>
<free>12193 MiB</free>
</fb_memory_usage>
<bar1_memory_usage>
<total>16384 MiB</total>
<used>2 MiB</used>
<free>16382 MiB</free>
</bar1_memory_usage>
<compute_mode>Default</compute_mode>
<utilization>
<gpu_util>0 %</gpu_util>
<memory_util>0 %</memory_util>
<encoder_util>0 %</encoder_util>
<decoder_util>0 %</decoder_util>
</utilization>
<encoder_stats>
<session_count>0</session_count>
<average_fps>0</average_fps>
<average_latency>0 ms</average_latency>
</encoder_stats>
<ecc_mode>
<current_ecc>Enabled</current_ecc>
<pending_ecc>Enabled</pending_ecc>
</ecc_mode>
<ecc_errors>
<volatile>
<single_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</single_bit>
<double_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</double_bit>
</volatile>
<aggregate>
<single_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</single_bit>
<double_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</double_bit>
</aggregate>
</ecc_errors>
<retired_pages>
<multiple_single_bit_retirement>
<retired_count>0</retired_count>
<retired_page_addresses>
</retired_page_addresses>
</multiple_single_bit_retirement>
<double_bit_retirement>
<retired_count>0</retired_count>
<retired_page_addresses>
</retired_page_addresses>
</double_bit_retirement>
<pending_retirement>No</pending_retirement>
</retired_pages>
<temperature>
<gpu_temp>31 C</gpu_temp>
<gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
<gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
</temperature>
<power_readings>
<power_state>P0</power_state>
<power_management>Supported</power_management>
<power_draw>24.84 W</power_draw>
<power_limit>250.00 W</power_limit>
<default_power_limit>250.00 W</default_power_limit>
<enforced_power_limit>250.00 W</enforced_power_limit>
<min_power_limit>125.00 W</min_power_limit>
<max_power_limit>250.00 W</max_power_limit>
</power_readings>
<clocks>
<graphics_clock>405 MHz</graphics_clock>
<sm_clock>405 MHz</sm_clock>
<mem_clock>715 MHz</mem_clock>
<video_clock>835 MHz</video_clock>
</clocks>
<applications_clocks>
<graphics_clock>1189 MHz</graphics_clock>
<mem_clock>715 MHz</mem_clock>
</applications_clocks>
<default_applications_clocks>
<graphics_clock>1189 MHz</graphics_clock>
<mem_clock>715 MHz</mem_clock>
</default_applications_clocks>
<max_clocks>
<graphics_clock>1328 MHz</graphics_clock>
<sm_clock>1328 MHz</sm_clock>
<mem_clock>715 MHz</mem_clock>
<video_clock>1328 MHz</video_clock>
</max_clocks>
<clock_policy>
<auto_boost>N/A</auto_boost>
<auto_boost_default>N/A</auto_boost_default>
</clock_policy>
<supported_clocks>
<supported_mem_clock>
<value>715 MHz</value>
<supported_graphics_clock>1328 MHz</supported_graphics_clock>
<supported_graphics_clock>1316 MHz</supported_graphics_clock>
<supported_graphics_clock>1303 MHz</supported_graphics_clock>
<supported_graphics_clock>1290 MHz</supported_graphics_clock>
<supported_graphics_clock>1278 MHz</supported_graphics_clock>
<supported_graphics_clock>1265 MHz</supported_graphics_clock>
<supported_graphics_clock>1252 MHz</supported_graphics_clock>
<supported_graphics_clock>1240 MHz</supported_graphics_clock>
<supported_graphics_clock>1227 MHz</supported_graphics_clock>
<supported_graphics_clock>1215 MHz</supported_graphics_clock>
<supported_graphics_clock>1202 MHz</supported_graphics_clock>
<supported_graphics_clock>1189 MHz</supported_graphics_clock>
<supported_graphics_clock>1177 MHz</supported_graphics_clock>
<supported_graphics_clock>1164 MHz</supported_graphics_clock>
<supported_graphics_clock>1151 MHz</supported_graphics_clock>
<supported_graphics_clock>1139 MHz</supported_graphics_clock>
<supported_graphics_clock>1126 MHz</supported_graphics_clock>
<supported_graphics_clock>1113 MHz</supported_graphics_clock>
<supported_graphics_clock>1101 MHz</supported_graphics_clock>
<supported_graphics_clock>1088 MHz</supported_graphics_clock>
<supported_graphics_clock>1075 MHz</supported_graphics_clock>
<supported_graphics_clock>1063 MHz</supported_graphics_clock>
<supported_graphics_clock>1050 MHz</supported_graphics_clock>
<supported_graphics_clock>1037 MHz</supported_graphics_clock>
<supported_graphics_clock>1025 MHz</supported_graphics_clock>
<supported_graphics_clock>1012 MHz</supported_graphics_clock>
<supported_graphics_clock>999 MHz</supported_graphics_clock>
<supported_graphics_clock>987 MHz</supported_graphics_clock>
<supported_graphics_clock>974 MHz</supported_graphics_clock>
<supported_graphics_clock>961 MHz</supported_graphics_clock>
<supported_graphics_clock>949 MHz</supported_graphics_clock>
<supported_graphics_clock>936 MHz</supported_graphics_clock>
<supported_graphics_clock>923 MHz</supported_graphics_clock>
<supported_graphics_clock>911 MHz</supported_graphics_clock>
<supported_graphics_clock>898 MHz</supported_graphics_clock>
<supported_graphics_clock>885 MHz</supported_graphics_clock>
<supported_graphics_clock>873 MHz</supported_graphics_clock>
<supported_graphics_clock>860 MHz</supported_graphics_clock>
<supported_graphics_clock>847 MHz</supported_graphics_clock>
<supported_graphics_clock>835 MHz</supported_graphics_clock>
<supported_graphics_clock>822 MHz</supported_graphics_clock>
<supported_graphics_clock>810 MHz</supported_graphics_clock>
<supported_graphics_clock>797 MHz</supported_graphics_clock>
<supported_graphics_clock>784 MHz</supported_graphics_clock>
<supported_graphics_clock>772 MHz</supported_graphics_clock>
<supported_graphics_clock>759 MHz</supported_graphics_clock>
<supported_graphics_clock>746 MHz</supported_graphics_clock>
<supported_graphics_clock>734 MHz</supported_graphics_clock>
<supported_graphics_clock>721 MHz</supported_graphics_clock>
<supported_graphics_clock>708 MHz</supported_graphics_clock>
<supported_graphics_clock>696 MHz</supported_graphics_clock>
<supported_graphics_clock>683 MHz</supported_graphics_clock>
<supported_graphics_clock>670 MHz</supported_graphics_clock>
<supported_graphics_clock>658 MHz</supported_graphics_clock>
<supported_graphics_clock>645 MHz</supported_graphics_clock>
<supported_graphics_clock>632 MHz</supported_graphics_clock>
<supported_graphics_clock>620 MHz</supported_graphics_clock>
<supported_graphics_clock>607 MHz</supported_graphics_clock>
<supported_graphics_clock>594 MHz</supported_graphics_clock>
<supported_graphics_clock>582 MHz</supported_graphics_clock>
<supported_graphics_clock>569 MHz</supported_graphics_clock>
<supported_graphics_clock>556 MHz</supported_graphics_clock>
<supported_graphics_clock>544 MHz</supported_graphics_clock>
</supported_mem_clock>
</supported_clocks>
<processes>
</processes>
<accounted_processes>
</accounted_processes>
</gpu>
<gpu id="0000:82:00.0">
<product_name>Tesla P100-PCIE-12GB</product_name>
<product_brand>Tesla</product_brand>
<display_mode>Disabled</display_mode>
<display_active>Disabled</display_active>
<persistence_mode>Disabled</persistence_mode>
<accounting_mode>Disabled</accounting_mode>
<accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
<driver_model>
<current_dm>N/A</current_dm>
<pending_dm>N/A</pending_dm>
</driver_model>
<serial>0320717031755</serial>
<uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
<minor_number>1</minor_number>
<vbios_version>86.00.3A.00.02</vbios_version>
<multigpu_board>No</multigpu_board>
<board_id>0x8200</board_id>
<gpu_part_number>900-2H400-0110-030</gpu_part_number>
<inforom_version>
<img_version>H400.0202.00.01</img_version>
<oem_object>1.1</oem_object>
<ecc_object>4.1</ecc_object>
<pwr_object>N/A</pwr_object>
</inforom_version>
<gpu_operation_mode>
<current_gom>N/A</current_gom>
<pending_gom>N/A</pending_gom>
</gpu_operation_mode>
<gpu_virtualization_mode>
<virtualization_mode>None</virtualization_mode>
</gpu_virtualization_mode>
<pci>
<pci_bus>82</pci_bus>
<pci_device>00</pci_device>
<pci_domain>0000</pci_domain>
<pci_device_id>15F710DE</pci_device_id>
<pci_bus_id>0000:82:00.0</pci_bus_id>
<pci_sub_system_id>11DA10DE</pci_sub_system_id>
<pci_gpu_link_info>
<pcie_gen>
<max_link_gen>3</max_link_gen>
<current_link_gen>3</current_link_gen>
</pcie_gen>
<link_widths>
<max_link_width>16x</max_link_width>
<current_link_width>16x</current_link_width>
</link_widths>
</pci_gpu_link_info>
<pci_bridge_chip>
<bridge_chip_type>N/A</bridge_chip_type>
<bridge_chip_fw>N/A</bridge_chip_fw>
</pci_bridge_chip>
<replay_counter>0</replay_counter>
<tx_util>0 KB/s</tx_util>
<rx_util>0 KB/s</rx_util>
</pci>
<fan_speed>N/A</fan_speed>
<performance_state>P0</performance_state>
<clocks_throttle_reasons>
<clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
<clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
<clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
<clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
<clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
<clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
</clocks_throttle_reasons>
<fb_memory_usage>
<total>12193 MiB</total>
<used>0 MiB</used>
<free>12193 MiB</free>
</fb_memory_usage>
<bar1_memory_usage>
<total>16384 MiB</total>
<used>2 MiB</used>
<free>16382 MiB</free>
</bar1_memory_usage>
<compute_mode>Default</compute_mode>
<utilization>
<gpu_util>10.3 %</gpu_util>
<memory_util>0 %</memory_util>
<encoder_util>0 %</encoder_util>
<decoder_util>0 %</decoder_util>
</utilization>
<encoder_stats>
<session_count>0</session_count>
<average_fps>0</average_fps>
<average_latency>0 ms</average_latency>
</encoder_stats>
<ecc_mode>
<current_ecc>Enabled</current_ecc>
<pending_ecc>Enabled</pending_ecc>
</ecc_mode>
<ecc_errors>
<volatile>
<single_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</single_bit>
<double_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</double_bit>
</volatile>
<aggregate>
<single_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</single_bit>
<double_bit>
<device_memory>0</device_memory>
<register_file>0</register_file>
<l1_cache>N/A</l1_cache>
<l2_cache>0</l2_cache>
<texture_memory>0</texture_memory>
<texture_shm>0</texture_shm>
<total>0</total>
</double_bit>
</aggregate>
</ecc_errors>
<retired_pages>
<multiple_single_bit_retirement>
<retired_count>0</retired_count>
<retired_page_addresses>
</retired_page_addresses>
</multiple_single_bit_retirement>
<double_bit_retirement>
<retired_count>0</retired_count>
<retired_page_addresses>
</retired_page_addresses>
</double_bit_retirement>
<pending_retirement>No</pending_retirement>
</retired_pages>
<temperature>
<gpu_temp>34 C</gpu_temp>
<gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
<gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
</temperature>
<power_readings>
<power_state>P0</power_state>
<power_management>Supported</power_management>
<power_draw>25.54 W</power_draw>
<power_limit>250.00 W</power_limit>
<default_power_limit>250.00 W</default_power_limit>
<enforced_power_limit>250.00 W</enforced_power_limit>
<min_power_limit>125.00 W</min_power_limit>
<max_power_limit>250.00 W</max_power_limit>
</power_readings>
<clocks>
<graphics_clock>405 MHz</graphics_clock>
<sm_clock>405 MHz</sm_clock>
<mem_clock>715 MHz</mem_clock>
<video_clock>835 MHz</video_clock>
</clocks>
<applications_clocks>
<graphics_clock>1189 MHz</graphics_clock>
<mem_clock>715 MHz</mem_clock>
</applications_clocks>
<default_applications_clocks>
<graphics_clock>1189 MHz</graphics_clock>
<mem_clock>715 MHz</mem_clock>
</default_applications_clocks>
<max_clocks>
<graphics_clock>1328 MHz</graphics_clock>
<sm_clock>1328 MHz</sm_clock>
<mem_clock>715 MHz</mem_clock>
<video_clock>1328 MHz</video_clock>
</max_clocks>
<clock_policy>
<auto_boost>N/A</auto_boost>
<auto_boost_default>N/A</auto_boost_default>
</clock_policy>
<supported_clocks>
<supported_mem_clock>
<value>715 MHz</value>
<supported_graphics_clock>1328 MHz</supported_graphics_clock>
<supported_graphics_clock>1316 MHz</supported_graphics_clock>
<supported_graphics_clock>1303 MHz</supported_graphics_clock>
<supported_graphics_clock>1290 MHz</supported_graphics_clock>
<supported_graphics_clock>1278 MHz</supported_graphics_clock>
<supported_graphics_clock>1265 MHz</supported_graphics_clock>
<supported_graphics_clock>1252 MHz</supported_graphics_clock>
<supported_graphics_clock>1240 MHz</supported_graphics_clock>
<supported_graphics_clock>1227 MHz</supported_graphics_clock>
<supported_graphics_clock>1215 MHz</supported_graphics_clock>
<supported_graphics_clock>1202 MHz</supported_graphics_clock>
<supported_graphics_clock>1189 MHz</supported_graphics_clock>
<supported_graphics_clock>1177 MHz</supported_graphics_clock>
<supported_graphics_clock>1164 MHz</supported_graphics_clock>
<supported_graphics_clock>1151 MHz</supported_graphics_clock>
<supported_graphics_clock>1139 MHz</supported_graphics_clock>
<supported_graphics_clock>1126 MHz</supported_graphics_clock>
<supported_graphics_clock>1113 MHz</supported_graphics_clock>
<supported_graphics_clock>1101 MHz</supported_graphics_clock>
<supported_graphics_clock>1088 MHz</supported_graphics_clock>
<supported_graphics_clock>1075 MHz</supported_graphics_clock>
<supported_graphics_clock>1063 MHz</supported_graphics_clock>
<supported_graphics_clock>1050 MHz</supported_graphics_clock>
<supported_graphics_clock>1037 MHz</supported_graphics_clock>
<supported_graphics_clock>1025 MHz</supported_graphics_clock>
<supported_graphics_clock>1012 MHz</supported_graphics_clock>
<supported_graphics_clock>999 MHz</supported_graphics_clock>
<supported_graphics_clock>987 MHz</supported_graphics_clock>
<supported_graphics_clock>974 MHz</supported_graphics_clock>
<supported_graphics_clock>961 MHz</supported_graphics_clock>
<supported_graphics_clock>949 MHz</supported_graphics_clock>
<supported_graphics_clock>936 MHz</supported_graphics_clock>
<supported_graphics_clock>923 MHz</supported_graphics_clock>
<supported_graphics_clock>911 MHz</supported_graphics_clock>
<supported_graphics_clock>898 MHz</supported_graphics_clock>
<supported_graphics_clock>885 MHz</supported_graphics_clock>
<supported_graphics_clock>873 MHz</supported_graphics_clock>
<supported_graphics_clock>860 MHz</supported_graphics_clock>
<supported_graphics_clock>847 MHz</supported_graphics_clock>
<supported_graphics_clock>835 MHz</supported_graphics_clock>
<supported_graphics_clock>822 MHz</supported_graphics_clock>
<supported_graphics_clock>810 MHz</supported_graphics_clock>
<supported_graphics_clock>797 MHz</supported_graphics_clock>
<supported_graphics_clock>784 MHz</supported_graphics_clock>
<supported_graphics_clock>772 MHz</supported_graphics_clock>
<supported_graphics_clock>759 MHz</supported_graphics_clock>
<supported_graphics_clock>746 MHz</supported_graphics_clock>
<supported_graphics_clock>734 MHz</supported_graphics_clock>
<supported_graphics_clock>721 MHz</supported_graphics_clock>
<supported_graphics_clock>708 MHz</supported_graphics_clock>
<supported_graphics_clock>696 MHz</supported_graphics_clock>
<supported_graphics_clock>683 MHz</supported_graphics_clock>
<supported_graphics_clock>670 MHz</supported_graphics_clock>
<supported_graphics_clock>658 MHz</supported_graphics_clock>
<supported_graphics_clock>645 MHz</supported_graphics_clock>
<supported_graphics_clock>632 MHz</supported_graphics_clock>
<supported_graphics_clock>620 MHz</supported_graphics_clock>
<supported_graphics_clock>607 MHz</supported_graphics_clock>
<supported_graphics_clock>594 MHz</supported_graphics_clock>
<supported_graphics_clock>582 MHz</supported_graphics_clock>
<supported_graphics_clock>569 MHz</supported_graphics_clock>
<supported_graphics_clock>556 MHz</supported_graphics_clock>
<supported_graphics_clock>544 MHz</supported_graphics_clock>
</supported_mem_clock>
</supported_clocks>
<processes>
</processes>
<accounted_processes>
</accounted_processes>
</gpu>
</nvidia_smi_log>