diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 9aaff15f469..a86db15116c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -76,6 +76,7 @@ public class ResourceUtils { + "\\p{Alnum}([\\p{Alnum}-]*\\p{Alnum})?)/)?\\p{Alpha}([\\w.-]*)$"); private final static String RES_PATTERN = "^[^=]+=\\d+\\s?\\w*$"; + public static final String YARN_IO_OPTIONAL = "(yarn\\.io/)?"; private static volatile boolean initializedResources = false; private static final Map RESOURCE_NAME_TO_INDEX = @@ -696,7 +697,7 @@ public class ResourceUtils { Configuration configuration, String prefix) { List result = new ArrayList<>(); Map customResourcesMap = configuration - .getValByRegex("^" + Pattern.quote(prefix) + "[^.]+$"); + .getValByRegex("^" + Pattern.quote(prefix) + YARN_IO_OPTIONAL + "[^.]+$"); for (Entry resource : customResourcesMap.entrySet()) { String resourceName = resource.getKey().substring(prefix.length()); Matcher matcher = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 9cc96b62e4d..8f742fa902f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -41,9 +41,12 @@ import java.io.File; import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** * Test class to verify all resource utility methods. @@ -191,6 +194,47 @@ public class TestResourceUtils { } } + @Test + public void testGetRequestedResourcesFromConfig() { + Configuration conf = new Configuration(); + + //these resource type configurations should be recognised + String propertyPrefix = "mapreduce.mapper.proper.rt."; + String[] expectedKeys = { + "yarn.io/gpu", + "yarn.io/fpga", + "yarn.io/anything_without_a_dot", + "regular_rt", + "regular_rt/with_slash"}; + + String[] invalidKeys = { + propertyPrefix + "too.many_parts", + propertyPrefix + "yarn.notio/gpu", + "incorrect.prefix.yarn.io/gpu", + propertyPrefix + "yarn.io/", + propertyPrefix}; + + for (String s : expectedKeys) { + //setting the properties which are expected to be in the resource list + conf.set(propertyPrefix + s, "42"); + } + + for (String s : invalidKeys) { + //setting the properties which are expected to be in the resource list + conf.set(s, "24"); + } + + List properList = + ResourceUtils.getRequestedResourcesFromConfig(conf, propertyPrefix); + Set expectedSet = + new HashSet<>(Arrays.asList(expectedKeys)); + + Assert.assertEquals(properList.size(), expectedKeys.length); + properList.forEach( + item -> Assert.assertTrue(expectedSet.contains(item.getName()))); + + } + @Test public void testGetResourceTypesConfigErrors() throws IOException { Configuration conf = new YarnConfiguration();