YARN-9138. Improve test coverage for nvidia-smi binary execution of GpuDiscoverer. Contributed by Szilard Nemeth.
(cherry picked from commit 46045c5cb3
)
This commit is contained in:
parent
3fe31b36fa
commit
d721634fea
|
@ -42,6 +42,7 @@ import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
|
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public class GpuDiscoverer {
|
public class GpuDiscoverer {
|
||||||
|
@ -75,6 +76,29 @@ public class GpuDiscoverer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String getErrorMessageOfScriptExecution(String msg) {
|
||||||
|
return getFailedToExecuteScriptMessage() +
|
||||||
|
"! Exception message: " + msg;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getErrorMessageOfScriptExecutionThresholdReached() {
|
||||||
|
return getFailedToExecuteScriptMessage() + " for " +
|
||||||
|
MAX_REPEATED_ERROR_ALLOWED + " times, " +
|
||||||
|
"skipping following executions!";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getFailedToExecuteScriptMessage() {
|
||||||
|
return "Failed to execute " +
|
||||||
|
GpuDeviceInformationParser.GPU_SCRIPT_REFERENCE +
|
||||||
|
" (" + pathOfGpuBinary + ")";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getFailedToParseErrorMessage(String msg) {
|
||||||
|
return "Failed to parse XML output of " +
|
||||||
|
GpuDeviceInformationParser.GPU_SCRIPT_REFERENCE
|
||||||
|
+ "( " + pathOfGpuBinary + ")" + msg;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get GPU device information from system.
|
* Get GPU device information from system.
|
||||||
* This need to be called after initialize.
|
* This need to be called after initialize.
|
||||||
|
@ -90,10 +114,7 @@ public class GpuDiscoverer {
|
||||||
validateConfOrThrowException();
|
validateConfOrThrowException();
|
||||||
|
|
||||||
if (numOfErrorExecutionSinceLastSucceed == MAX_REPEATED_ERROR_ALLOWED) {
|
if (numOfErrorExecutionSinceLastSucceed == MAX_REPEATED_ERROR_ALLOWED) {
|
||||||
String msg =
|
String msg = getErrorMessageOfScriptExecutionThresholdReached();
|
||||||
"Failed to execute GPU device information detection script for "
|
|
||||||
+ MAX_REPEATED_ERROR_ALLOWED
|
|
||||||
+ " times, skip following executions.";
|
|
||||||
LOG.error(msg);
|
LOG.error(msg);
|
||||||
throw new YarnException(msg);
|
throw new YarnException(msg);
|
||||||
}
|
}
|
||||||
|
@ -107,16 +128,14 @@ public class GpuDiscoverer {
|
||||||
return lastDiscoveredGpuInformation;
|
return lastDiscoveredGpuInformation;
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
numOfErrorExecutionSinceLastSucceed++;
|
numOfErrorExecutionSinceLastSucceed++;
|
||||||
String msg =
|
String msg = getErrorMessageOfScriptExecution(e.getMessage());
|
||||||
"Failed to execute " + pathOfGpuBinary + " exception message:" + e
|
|
||||||
.getMessage() + ", continue ...";
|
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug(msg);
|
LOG.debug(msg);
|
||||||
}
|
}
|
||||||
throw new YarnException(e);
|
throw new YarnException(msg, e);
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
numOfErrorExecutionSinceLastSucceed++;
|
numOfErrorExecutionSinceLastSucceed++;
|
||||||
String msg = "Failed to parse xml output" + e.getMessage();
|
String msg = getFailedToParseErrorMessage(e.getMessage());
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.warn(msg, e);
|
LOG.warn(msg, e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,6 +43,8 @@ import java.io.StringReader;
|
||||||
public class GpuDeviceInformationParser {
|
public class GpuDeviceInformationParser {
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(
|
private static final Logger LOG = LoggerFactory.getLogger(
|
||||||
GpuDeviceInformationParser.class);
|
GpuDeviceInformationParser.class);
|
||||||
|
public static final String GPU_SCRIPT_REFERENCE = "GPU device detection " +
|
||||||
|
"script";
|
||||||
|
|
||||||
private Unmarshaller unmarshaller = null;
|
private Unmarshaller unmarshaller = null;
|
||||||
private XMLReader xmlReader = null;
|
private XMLReader xmlReader = null;
|
||||||
|
@ -70,7 +72,9 @@ public class GpuDeviceInformationParser {
|
||||||
try {
|
try {
|
||||||
init();
|
init();
|
||||||
} catch (SAXException | ParserConfigurationException | JAXBException e) {
|
} catch (SAXException | ParserConfigurationException | JAXBException e) {
|
||||||
LOG.error("Exception while initialize parser", e);
|
String msg = "Exception while initializing parser for " +
|
||||||
|
GPU_SCRIPT_REFERENCE;
|
||||||
|
LOG.error(msg, e);
|
||||||
throw new YarnException(e);
|
throw new YarnException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -80,8 +84,10 @@ public class GpuDeviceInformationParser {
|
||||||
try {
|
try {
|
||||||
return (GpuDeviceInformation) unmarshaller.unmarshal(source);
|
return (GpuDeviceInformation) unmarshaller.unmarshal(source);
|
||||||
} catch (JAXBException e) {
|
} catch (JAXBException e) {
|
||||||
LOG.error("Exception while parsing xml", e);
|
String msg = "Failed to parse XML output of " +
|
||||||
throw new YarnException(e);
|
GPU_SCRIPT_REFERENCE + "!";
|
||||||
|
LOG.error(msg, e);
|
||||||
|
throw new YarnException(msg, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugi
|
||||||
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.util.Shell;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
|
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
|
||||||
|
@ -28,19 +29,38 @@ import org.junit.Before;
|
||||||
import org.junit.Rule;
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.rules.ExpectedException;
|
import org.junit.rules.ExpectedException;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.PrintWriter;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
|
||||||
|
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer.DEFAULT_BINARY_NAME;
|
||||||
|
import static org.hamcrest.CoreMatchers.containsString;
|
||||||
|
import static org.hamcrest.CoreMatchers.not;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertNotNull;
|
import static org.junit.Assert.assertNotNull;
|
||||||
import static org.junit.Assert.assertNull;
|
import static org.junit.Assert.assertNull;
|
||||||
|
import static org.junit.Assert.assertThat;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
public class TestGpuDiscoverer {
|
public class TestGpuDiscoverer {
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(
|
||||||
|
TestGpuDiscoverer.class);
|
||||||
|
|
||||||
|
private static final String PATH = "PATH";
|
||||||
|
private static final String NVIDIA = "nvidia";
|
||||||
|
private static final String EXEC_PERMISSION = "u+x";
|
||||||
|
private static final String BASH_SHEBANG = "#!/bin/bash\n\n";
|
||||||
|
private static final String TEST_PARENT_DIR = new File("target/temp/" +
|
||||||
|
TestGpuDiscoverer.class.getName()).getAbsolutePath();
|
||||||
|
|
||||||
@Rule
|
@Rule
|
||||||
public ExpectedException exception = ExpectedException.none();
|
public ExpectedException exception = ExpectedException.none();
|
||||||
|
|
||||||
|
@ -68,8 +88,8 @@ public class TestGpuDiscoverer {
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void before() throws IOException {
|
public void before() throws IOException {
|
||||||
String folder = getTestParentFolder();
|
assumeNotWindows();
|
||||||
File f = new File(folder);
|
File f = new File(TEST_PARENT_DIR);
|
||||||
FileUtils.deleteDirectory(f);
|
FileUtils.deleteDirectory(f);
|
||||||
f.mkdirs();
|
f.mkdirs();
|
||||||
}
|
}
|
||||||
|
@ -81,6 +101,55 @@ public class TestGpuDiscoverer {
|
||||||
return conf;
|
return conf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void createNvidiaSmiScript(File file) {
|
||||||
|
writeToFile(file, BASH_SHEBANG +
|
||||||
|
"echo '<nvidia_smi_log></nvidia_smi_log>'");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createFaultyNvidiaSmiScript(File file) {
|
||||||
|
writeToFile(file, BASH_SHEBANG + "echo <<'");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createNvidiaSmiScriptWithInvalidXml(File file) {
|
||||||
|
writeToFile(file, BASH_SHEBANG + "echo '<nvidia_smi_log></bla>'");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void writeToFile(File file, String contents) {
|
||||||
|
try {
|
||||||
|
PrintWriter fileWriter = new PrintWriter(file);
|
||||||
|
fileWriter.write(contents);
|
||||||
|
fileWriter.close();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException("Error while writing nvidia-smi script file!",
|
||||||
|
e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertNvidiaIsOnPath(GpuDiscoverer discoverer) {
|
||||||
|
String path = discoverer.getEnvironmentToRunCommand().get(PATH);
|
||||||
|
assertNotNull(path);
|
||||||
|
assertTrue(path.contains(NVIDIA));
|
||||||
|
}
|
||||||
|
|
||||||
|
private File createFakeNvidiaSmiScriptAsRunnableFile(
|
||||||
|
Consumer<File> scriptFileCreator) throws IOException {
|
||||||
|
File fakeBinary = new File(TEST_PARENT_DIR, DEFAULT_BINARY_NAME);
|
||||||
|
touchFile(fakeBinary);
|
||||||
|
scriptFileCreator.accept(fakeBinary);
|
||||||
|
Shell.execCommand(Shell.getSetPermissionCommand(EXEC_PERMISSION, false,
|
||||||
|
fakeBinary.getAbsolutePath()));
|
||||||
|
|
||||||
|
return fakeBinary;
|
||||||
|
}
|
||||||
|
|
||||||
|
private GpuDiscoverer creatediscovererWithGpuPathDefined(
|
||||||
|
Configuration conf) throws YarnException {
|
||||||
|
conf.set(YarnConfiguration.NM_GPU_PATH_TO_EXEC, TEST_PARENT_DIR);
|
||||||
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
|
discoverer.initialize(conf);
|
||||||
|
return discoverer;
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testLinuxGpuResourceDiscoverPluginConfig() throws Exception {
|
public void testLinuxGpuResourceDiscoverPluginConfig() throws Exception {
|
||||||
// Only run this on demand.
|
// Only run this on demand.
|
||||||
|
@ -89,31 +158,151 @@ public class TestGpuDiscoverer {
|
||||||
|
|
||||||
// test case 1, check default setting.
|
// test case 1, check default setting.
|
||||||
Configuration conf = new Configuration(false);
|
Configuration conf = new Configuration(false);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
assertEquals(GpuDiscoverer.DEFAULT_BINARY_NAME,
|
assertEquals(DEFAULT_BINARY_NAME, discoverer.getPathOfGpuBinary());
|
||||||
plugin.getPathOfGpuBinary());
|
assertNvidiaIsOnPath(discoverer);
|
||||||
assertNotNull(plugin.getEnvironmentToRunCommand().get("PATH"));
|
|
||||||
assertTrue(
|
|
||||||
plugin.getEnvironmentToRunCommand().get("PATH").contains("nvidia"));
|
|
||||||
|
|
||||||
// test case 2, check mandatory set path.
|
// test case 2, check mandatory set path.
|
||||||
File fakeBinary = setupFakeBinary(conf);
|
File fakeBinary = setupFakeBinary(conf);
|
||||||
plugin = new GpuDiscoverer();
|
discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
assertEquals(fakeBinary.getAbsolutePath(),
|
assertEquals(fakeBinary.getAbsolutePath(),
|
||||||
plugin.getPathOfGpuBinary());
|
discoverer.getPathOfGpuBinary());
|
||||||
assertNull(plugin.getEnvironmentToRunCommand().get("PATH"));
|
assertNull(discoverer.getEnvironmentToRunCommand().get(PATH));
|
||||||
|
|
||||||
// test case 3, check mandatory set path, but binary doesn't exist so default
|
// test case 3, check mandatory set path,
|
||||||
// path will be used.
|
// but binary doesn't exist so default path will be used.
|
||||||
fakeBinary.delete();
|
fakeBinary.delete();
|
||||||
plugin = new GpuDiscoverer();
|
discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
assertEquals(GpuDiscoverer.DEFAULT_BINARY_NAME,
|
assertEquals(DEFAULT_BINARY_NAME,
|
||||||
plugin.getPathOfGpuBinary());
|
discoverer.getPathOfGpuBinary());
|
||||||
assertTrue(
|
assertNvidiaIsOnPath(discoverer);
|
||||||
plugin.getEnvironmentToRunCommand().get("PATH").contains("nvidia"));
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetGpuDeviceInformationValidNvidiaSmiScript()
|
||||||
|
throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration(false);
|
||||||
|
|
||||||
|
File fakeBinary = createFakeNvidiaSmiScriptAsRunnableFile(
|
||||||
|
this::createNvidiaSmiScript);
|
||||||
|
|
||||||
|
GpuDiscoverer discoverer = creatediscovererWithGpuPathDefined(conf);
|
||||||
|
assertEquals(fakeBinary.getAbsolutePath(),
|
||||||
|
discoverer.getPathOfGpuBinary());
|
||||||
|
assertNull(discoverer.getEnvironmentToRunCommand().get(PATH));
|
||||||
|
|
||||||
|
GpuDeviceInformation result =
|
||||||
|
discoverer.getGpuDeviceInformation();
|
||||||
|
assertNotNull(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetGpuDeviceInformationFakeNvidiaSmiScriptConsecutiveRun()
|
||||||
|
throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration(false);
|
||||||
|
|
||||||
|
File fakeBinary = createFakeNvidiaSmiScriptAsRunnableFile(
|
||||||
|
this::createNvidiaSmiScript);
|
||||||
|
|
||||||
|
GpuDiscoverer discoverer = creatediscovererWithGpuPathDefined(conf);
|
||||||
|
assertEquals(fakeBinary.getAbsolutePath(),
|
||||||
|
discoverer.getPathOfGpuBinary());
|
||||||
|
assertNull(discoverer.getEnvironmentToRunCommand().get(PATH));
|
||||||
|
|
||||||
|
for (int i = 0; i < 5; i++) {
|
||||||
|
GpuDeviceInformation result = discoverer.getGpuDeviceInformation();
|
||||||
|
assertNotNull(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetGpuDeviceInformationFaultyNvidiaSmiScript()
|
||||||
|
throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration(false);
|
||||||
|
|
||||||
|
File fakeBinary = createFakeNvidiaSmiScriptAsRunnableFile(
|
||||||
|
this::createFaultyNvidiaSmiScript);
|
||||||
|
|
||||||
|
GpuDiscoverer discoverer = creatediscovererWithGpuPathDefined(conf);
|
||||||
|
assertEquals(fakeBinary.getAbsolutePath(),
|
||||||
|
discoverer.getPathOfGpuBinary());
|
||||||
|
assertNull(discoverer.getEnvironmentToRunCommand().get(PATH));
|
||||||
|
|
||||||
|
exception.expect(YarnException.class);
|
||||||
|
exception.expectMessage("Failed to execute GPU device detection script");
|
||||||
|
discoverer.getGpuDeviceInformation();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetGpuDeviceInformationFaultyNvidiaSmiScriptConsecutiveRun()
|
||||||
|
throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration(false);
|
||||||
|
|
||||||
|
File fakeBinary = createFakeNvidiaSmiScriptAsRunnableFile(
|
||||||
|
this::createNvidiaSmiScript);
|
||||||
|
|
||||||
|
GpuDiscoverer discoverer = creatediscovererWithGpuPathDefined(conf);
|
||||||
|
assertEquals(fakeBinary.getAbsolutePath(),
|
||||||
|
discoverer.getPathOfGpuBinary());
|
||||||
|
assertNull(discoverer.getEnvironmentToRunCommand().get(PATH));
|
||||||
|
|
||||||
|
LOG.debug("Querying nvidia-smi correctly, once...");
|
||||||
|
discoverer.getGpuDeviceInformation();
|
||||||
|
|
||||||
|
LOG.debug("Replacing script with faulty version!");
|
||||||
|
createFaultyNvidiaSmiScript(fakeBinary);
|
||||||
|
|
||||||
|
final String terminateMsg = "Failed to execute GPU device " +
|
||||||
|
"detection script (" + fakeBinary.getAbsolutePath() + ") for 10 times";
|
||||||
|
final String msg = "Failed to execute GPU device detection script";
|
||||||
|
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
try {
|
||||||
|
LOG.debug("Executing faulty nvidia-smi script...");
|
||||||
|
discoverer.getGpuDeviceInformation();
|
||||||
|
fail("Query of GPU device info via nvidia-smi should fail as " +
|
||||||
|
"script should be faulty: " + fakeBinary);
|
||||||
|
} catch (YarnException e) {
|
||||||
|
assertThat(e.getMessage(), containsString(msg));
|
||||||
|
assertThat(e.getMessage(), not(containsString(terminateMsg)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
LOG.debug("Executing faulty nvidia-smi script again..." +
|
||||||
|
"We should reach the error threshold now!");
|
||||||
|
discoverer.getGpuDeviceInformation();
|
||||||
|
fail("Query of GPU device info via nvidia-smi should fail as " +
|
||||||
|
"script should be faulty: " + fakeBinary);
|
||||||
|
} catch (YarnException e) {
|
||||||
|
assertThat(e.getMessage(), containsString(terminateMsg));
|
||||||
|
}
|
||||||
|
|
||||||
|
LOG.debug("Verifying if GPUs are still hold the value of " +
|
||||||
|
"first successful query");
|
||||||
|
assertNotNull(discoverer.getGpusUsableByYarn());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetGpuDeviceInformationNvidiaSmiScriptWithInvalidXml()
|
||||||
|
throws YarnException, IOException {
|
||||||
|
Configuration conf = new Configuration(false);
|
||||||
|
|
||||||
|
File fakeBinary = createFakeNvidiaSmiScriptAsRunnableFile(
|
||||||
|
this::createNvidiaSmiScriptWithInvalidXml);
|
||||||
|
|
||||||
|
GpuDiscoverer discoverer = creatediscovererWithGpuPathDefined(conf);
|
||||||
|
assertEquals(fakeBinary.getAbsolutePath(),
|
||||||
|
discoverer.getPathOfGpuBinary());
|
||||||
|
assertNull(discoverer.getEnvironmentToRunCommand().get(PATH));
|
||||||
|
|
||||||
|
exception.expect(YarnException.class);
|
||||||
|
exception.expectMessage("Failed to parse XML output of " +
|
||||||
|
"GPU device detection script");
|
||||||
|
discoverer.getGpuDeviceInformation();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -123,12 +312,12 @@ public class TestGpuDiscoverer {
|
||||||
Assume.assumeTrue(
|
Assume.assumeTrue(
|
||||||
Boolean.valueOf(System.getProperty("runGpuDiscoverUnitTest")));
|
Boolean.valueOf(System.getProperty("runGpuDiscoverUnitTest")));
|
||||||
Configuration conf = new Configuration(false);
|
Configuration conf = new Configuration(false);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
GpuDeviceInformation info = plugin.getGpuDeviceInformation();
|
GpuDeviceInformation info = discoverer.getGpuDeviceInformation();
|
||||||
|
|
||||||
assertTrue(info.getGpus().size() > 0);
|
assertTrue(info.getGpus().size() > 0);
|
||||||
assertEquals(plugin.getGpusUsableByYarn().size(),
|
assertEquals(discoverer.getGpusUsableByYarn().size(),
|
||||||
info.getGpus().size());
|
info.getGpus().size());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -137,9 +326,9 @@ public class TestGpuDiscoverer {
|
||||||
throws YarnException {
|
throws YarnException {
|
||||||
Configuration conf = createConfigWithAllowedDevices("1:2");
|
Configuration conf = createConfigWithAllowedDevices("1:2");
|
||||||
|
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
List<GpuDevice> usableGpuDevices = plugin.getGpusUsableByYarn();
|
List<GpuDevice> usableGpuDevices = discoverer.getGpusUsableByYarn();
|
||||||
assertEquals(1, usableGpuDevices.size());
|
assertEquals(1, usableGpuDevices.size());
|
||||||
|
|
||||||
assertEquals(1, usableGpuDevices.get(0).getIndex());
|
assertEquals(1, usableGpuDevices.get(0).getIndex());
|
||||||
|
@ -152,18 +341,18 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0:0,1:1,2:2,3");
|
Configuration conf = createConfigWithAllowedDevices("0:0,1:1,2:2,3");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGetNumberOfUsableGpusFromConfig() throws YarnException {
|
public void testGetNumberOfUsableGpusFromConfig() throws YarnException {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0:0,1:1,2:2,3:4");
|
Configuration conf = createConfigWithAllowedDevices("0:0,1:1,2:2,3:4");
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
|
|
||||||
List<GpuDevice> usableGpuDevices = plugin.getGpusUsableByYarn();
|
List<GpuDevice> usableGpuDevices = discoverer.getGpusUsableByYarn();
|
||||||
assertEquals(4, usableGpuDevices.size());
|
assertEquals(4, usableGpuDevices.size());
|
||||||
|
|
||||||
assertEquals(0, usableGpuDevices.get(0).getIndex());
|
assertEquals(0, usableGpuDevices.get(0).getIndex());
|
||||||
|
@ -185,9 +374,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0:0,1:1,2:2,1:1");
|
Configuration conf = createConfigWithAllowedDevices("0:0,1:1,2:2,1:1");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -196,9 +385,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0:0,1:1,2:2,1:1,2:2");
|
Configuration conf = createConfigWithAllowedDevices("0:0,1:1,2:2,1:1,2:2");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -207,9 +396,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0 : 0,1 : 1");
|
Configuration conf = createConfigWithAllowedDevices("0 : 0,1 : 1");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -218,9 +407,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0:@$1,1:1");
|
Configuration conf = createConfigWithAllowedDevices("0:@$1,1:1");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -229,9 +418,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("x:0, 1:y");
|
Configuration conf = createConfigWithAllowedDevices("x:0, 1:y");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -240,9 +429,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices(":0, :1");
|
Configuration conf = createConfigWithAllowedDevices(":0, :1");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -251,9 +440,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("");
|
Configuration conf = createConfigWithAllowedDevices("");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -262,9 +451,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0:0 0:1");
|
Configuration conf = createConfigWithAllowedDevices("0:0 0:1");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -273,9 +462,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0.1 0.2");
|
Configuration conf = createConfigWithAllowedDevices("0.1 0.2");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -284,9 +473,9 @@ public class TestGpuDiscoverer {
|
||||||
Configuration conf = createConfigWithAllowedDevices("0.1,0.2");
|
Configuration conf = createConfigWithAllowedDevices("0.1,0.2");
|
||||||
|
|
||||||
exception.expect(GpuDeviceSpecificationException.class);
|
exception.expect(GpuDeviceSpecificationException.class);
|
||||||
GpuDiscoverer plugin = new GpuDiscoverer();
|
GpuDiscoverer discoverer = new GpuDiscoverer();
|
||||||
plugin.initialize(conf);
|
discoverer.initialize(conf);
|
||||||
plugin.getGpusUsableByYarn();
|
discoverer.getGpusUsableByYarn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
Loading…
Reference in New Issue