YARN-9444. YARN API ResourceUtils's getRequestedResourcesFromConfig doesn't recognize yarn.io/gpu as a valid resource. Contributed by Gergely Pollak

(cherry picked from commit 52e9ee39a1)
This commit is contained in:
Szilard Nemeth 2019-11-26 16:42:33 +01:00 committed by Eric E Payne
parent 89696b66e7
commit d345994468
2 changed files with 48 additions and 1 deletions

View File

@ -67,6 +67,8 @@ public class ResourceUtils {
"^(((\\p{Alnum}([\\p{Alnum}-]*\\p{Alnum})?\\.)*"
+ "\\p{Alnum}([\\p{Alnum}-]*\\p{Alnum})?)/)?\\p{Alpha}([\\w.-]*)$");
public static final String YARN_IO_OPTIONAL = "(yarn\\.io/)?";
private static volatile boolean initializedResources = false;
private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
new ConcurrentHashMap<String, Integer>();
@ -664,7 +666,7 @@ public class ResourceUtils {
Configuration configuration, String prefix) {
List<ResourceInformation> result = new ArrayList<>();
Map<String, String> customResourcesMap = configuration
.getValByRegex("^" + Pattern.quote(prefix) + "[^.]+$");
.getValByRegex("^" + Pattern.quote(prefix) + YARN_IO_OPTIONAL + "[^.]+$");
for (Entry<String, String> resource : customResourcesMap.entrySet()) {
String resourceName = resource.getKey().substring(prefix.length());
Matcher matcher =

View File

@ -37,8 +37,12 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Test class to verify all resource utility methods.
@ -203,6 +207,47 @@ public class TestResourceUtils {
}
}
@Test
public void testGetRequestedResourcesFromConfig() {
Configuration conf = new Configuration();
//these resource type configurations should be recognised
String propertyPrefix = "mapreduce.mapper.proper.rt.";
String[] expectedKeys = {
"yarn.io/gpu",
"yarn.io/fpga",
"yarn.io/anything_without_a_dot",
"regular_rt",
"regular_rt/with_slash"};
String[] invalidKeys = {
propertyPrefix + "too.many_parts",
propertyPrefix + "yarn.notio/gpu",
"incorrect.prefix.yarn.io/gpu",
propertyPrefix + "yarn.io/",
propertyPrefix};
for (String s : expectedKeys) {
//setting the properties which are expected to be in the resource list
conf.set(propertyPrefix + s, "42");
}
for (String s : invalidKeys) {
//setting the properties which are expected to be in the resource list
conf.set(s, "24");
}
List<ResourceInformation> properList =
ResourceUtils.getRequestedResourcesFromConfig(conf, propertyPrefix);
Set<String> expectedSet =
new HashSet<>(Arrays.asList(expectedKeys));
Assert.assertEquals(properList.size(), expectedKeys.length);
properList.forEach(
item -> Assert.assertTrue(expectedSet.contains(item.getName())));
}
@Test
public void testGetResourceTypesConfigErrors() throws IOException {
Configuration conf = new YarnConfiguration();