Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7383. Node resource is not parsed correctly for resource names containing dot. Contributed by Gergely Novák.

This commit is contained in:
Sunil G 2017-12-13 22:00:07 +05:30
parent 880cd754a5
commit 10fc8d2a7d
4 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -464,21 +464,18 @@ public class ResourceUtils {
for (Map.Entry<String, String> entry : conf) { for (Map.Entry<String, String> entry : conf) {
String key = entry.getKey(); String key = entry.getKey();
String value = entry.getValue(); String value = entry.getValue();
addResourceTypeInformation(key, value, nodeResources);
if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
addResourceInformation(key, value, nodeResources);
}
} }
return nodeResources; return nodeResources;
} }
private static void addResourceInformation(String prop, String value, private static void addResourceTypeInformation(String prop, String value,
Map<String, ResourceInformation> nodeResources) { Map<String, ResourceInformation> nodeResources) {
String[] parts = prop.split("\\."); if (prop.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
LOG.info("Found resource entry " + prop); LOG.info("Found resource entry " + prop);
if (parts.length == 4) { String resourceType = prop.substring(
String resourceType = parts[3]; YarnConfiguration.NM_RESOURCES_PREFIX.length());
if (!nodeResources.containsKey(resourceType)) { if (!nodeResources.containsKey(resourceType)) {
nodeResources nodeResources
.put(resourceType, ResourceInformation.newInstance(resourceType)); .put(resourceType, ResourceInformation.newInstance(resourceType));

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
View File

@ -123,9 +123,10 @@ public class TestResourceUtils {
new ResourceFileInformation("resource-types-3.xml", 3); new ResourceFileInformation("resource-types-3.xml", 3);
testFile3.resourceNameUnitsMap.put("resource2", ""); testFile3.resourceNameUnitsMap.put("resource2", "");
ResourceFileInformation testFile4 = ResourceFileInformation testFile4 =
new ResourceFileInformation("resource-types-4.xml", 4); new ResourceFileInformation("resource-types-4.xml", 5);
testFile4.resourceNameUnitsMap.put("resource1", "G"); testFile4.resourceNameUnitsMap.put("resource1", "G");
testFile4.resourceNameUnitsMap.put("resource2", "m"); testFile4.resourceNameUnitsMap.put("resource2", "m");
testFile4.resourceNameUnitsMap.put("yarn.io/gpu", "");
ResourceFileInformation[] tests = {testFile1, testFile2, testFile3, ResourceFileInformation[] tests = {testFile1, testFile2, testFile3,
testFile4}; testFile4};
@ -291,6 +292,8 @@ public class TestResourceUtils {
ResourceInformation.newInstance("resource1", "Gi", 5L)); ResourceInformation.newInstance("resource1", "Gi", 5L));
test3Resources.setResourceInformation("resource2", test3Resources.setResourceInformation("resource2",
ResourceInformation.newInstance("resource2", "m", 2L)); ResourceInformation.newInstance("resource2", "m", 2L));
test3Resources.setResourceInformation("yarn.io/gpu",
ResourceInformation.newInstance("yarn.io/gpu", "", 1));
testRun.put("node-resources-2.xml", test3Resources); testRun.put("node-resources-2.xml", test3Resources);
for (Map.Entry<String, Resource> entry : testRun.entrySet()) { for (Map.Entry<String, Resource> entry : testRun.entrySet()) {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml
View File

@ -36,4 +36,9 @@ limitations under the License. See accompanying LICENSE file.
<value>2m</value> <value>2m</value>
</property> </property>
<property>
<name>yarn.nodemanager.resource-type.yarn.io/gpu</name>
<value>1</value>
</property>
</configuration> </configuration>

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml
View File

@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file.
<property> <property>
<name>yarn.resource-types</name> <name>yarn.resource-types</name>
<value>resource1,resource2</value> <value>resource1,resource2,yarn.io/gpu</value>
</property> </property>
<property> <property>
@ -31,4 +31,9 @@ limitations under the License. See accompanying LICENSE file.
<value>m</value> <value>m</value>
</property> </property>
<property>
<name>yarn.resource-types.yarn.io/gpu.units</name>
<value></value>
</property>
</configuration> </configuration>