From 42f3a7082a90bc71f0e86dc1e50b0c77b05489cf Mon Sep 17 00:00:00 2001 From: Sunil G Date: Mon, 12 Nov 2018 21:09:30 +0530 Subject: [PATCH] YARN-8877. [CSI] Extend service spec to allow setting resource attributes. Contributed by Weiwei Yang. --- .../yarn/api/records/ResourceInformation.java | 7 +++ .../api/records/ResourceInformation.java | 18 +++++++ .../yarn/service/component/Component.java | 3 +- .../hadoop/yarn/service/TestServiceAM.java | 47 +++++++++++++++++++ .../yarn/service/conf/ExampleAppJson.java | 1 + .../yarn/service/conf/TestAppJsonResolve.java | 18 +++++++ .../yarn/service/conf/examples/external3.json | 26 ++++++++++ .../markdown/yarn-service/YarnServiceAPI.md | 2 +- 8 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 057e94ebd78..047c09ac8b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -275,6 +275,13 @@ public class ResourceInformation implements Comparable { Long.MAX_VALUE); } + public static ResourceInformation newInstance(String name, String units, + long value, Map attributes) { + return ResourceInformation + .newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L, + Long.MAX_VALUE, null, attributes); + } + public static ResourceInformation newInstance(String name, String units, ResourceTypes resourceType) { return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java index 103fffb6f75..e466ce7ea57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.service.api.records; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import java.util.Map; import java.util.Objects; /** @@ -35,11 +37,25 @@ public class ResourceInformation { @SerializedName("unit") private String unit = null; + @SerializedName("attributes") + private Map attributes = null; + public ResourceInformation value(Long value) { this.value = value; return this; } + @ApiModelProperty(value = "") + @JsonProperty("attributes") + public Map getAttributes() { + return attributes == null ? ImmutableMap.of() : attributes; + } + + public ResourceInformation attributes(Map attributes) { + this.attributes = attributes; + return this; + } + /** * Integer value of the resource. * @@ -98,6 +114,8 @@ public class ResourceInformation { sb.append("class ResourceInformation {\n"); sb.append(" value: ").append(toIndentedString(value)).append("\n"); sb.append(" unit: ").append(toIndentedString(unit)).append("\n"); + sb.append(" attributes: ").append(toIndentedString(attributes)) + .append("\n"); sb.append("}"); return sb.toString(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java index 526bde0ff13..7833b0618ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -701,7 +701,8 @@ public class Component implements EventHandler { org.apache.hadoop.yarn.api.records.ResourceInformation.newInstance( entry.getKey(), specInfo.getUnit(), - specInfo.getValue()); + specInfo.getValue(), + specInfo.getAttributes()); resource.setResourceInformation(resourceName, ri); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java index 80f491090d7..51c27e892ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java @@ -456,4 +456,51 @@ public class TestServiceAM extends ServiceTestUtils{ Assert.fail("Fail to sync sysfs."); } } + + @Test + public void testScheduleWithResourceAttributes() throws Exception { + ApplicationId applicationId = ApplicationId.newInstance(123456, 1); + Service exampleApp = new Service(); + exampleApp.setId(applicationId.toString()); + exampleApp.setName("testScheduleWithResourceAttributes"); + exampleApp.setVersion("v1"); + + List resourceTypeInfos = new ArrayList<>( + ResourceUtils.getResourcesTypeInfo()); + // Add 3rd resource type. + resourceTypeInfos.add(ResourceTypeInfo + .newInstance("test-resource", "", ResourceTypes.COUNTABLE)); + // Reinitialize resource types + ResourceUtils.reinitializeResources(resourceTypeInfos); + + Component serviceCompoent = createComponent("compa", 1, "pwd"); + serviceCompoent.getResource().setResourceInformations( + ImmutableMap.of("test-resource", + new ResourceInformation() + .value(1234L) + .unit("Gi") + .attributes(ImmutableMap.of("k1", "v1", "k2", "v2")))); + exampleApp.addComponent(serviceCompoent); + + MockServiceAM am = new MockServiceAM(exampleApp); + am.init(conf); + am.start(); + + ServiceScheduler serviceScheduler = am.context.scheduler; + AMRMClientAsync amrmClientAsync = + serviceScheduler.getAmRMClient(); + + Collection rr = + amrmClientAsync.getMatchingRequests(0); + Assert.assertEquals(1, rr.size()); + + org.apache.hadoop.yarn.api.records.Resource capability = + rr.iterator().next().getCapability(); + Assert.assertEquals(1234L, capability.getResourceValue("test-resource")); + Assert.assertEquals("Gi", + capability.getResourceInformation("test-resource").getUnits()); + Assert.assertEquals(2, capability.getResourceInformation("test-resource") + .getAttributes().size()); + am.stop(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java index 5fdd2ab0c7c..754b589dce0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java @@ -38,6 +38,7 @@ public final class ExampleAppJson { public static final String EXTERNAL_JSON_0 = "external0.json"; public static final String EXTERNAL_JSON_1 = "external1.json"; public static final String EXTERNAL_JSON_2 = "external2.json"; + public static final String EXTERNAL_JSON_3 = "external3.json"; public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java index 73f7fa163af..25c502f6494 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java @@ -20,7 +20,9 @@ package org.apache.hadoop.yarn.service.conf; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.ServiceTestUtils; +import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.api.records.ResourceInformation; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Configuration; @@ -214,4 +216,20 @@ public class TestAppJsonResolve extends Assert { other = orig.getComponent("other").getConfiguration(); assertEquals(0, other.getProperties().size()); } + + @Test + public void testSetResourceAttributes() throws IOException { + Service orig = ExampleAppJson.loadResource(EXTERNAL_JSON_3); + Component component = orig.getComponent("volume-service"); + Assert.assertNotNull(component); + Map adResource = component + .getResource().getAdditional(); + Assert.assertNotNull(adResource); + Assert.assertEquals(1, adResource.size()); + Map.Entry volume = adResource + .entrySet().iterator().next(); + Assert.assertEquals("yarn.io/csi-volume", volume.getKey()); + Assert.assertEquals(100L, volume.getValue().getValue().longValue()); + Assert.assertEquals(2, volume.getValue().getAttributes().size()); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json new file mode 100644 index 00000000000..74569bd0ba1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json @@ -0,0 +1,26 @@ +{ + "name": "external-3", + "version": "1.0.0", + "lifetime": "3600", + "components": [ + { + "name": "volume-service", + "launch_command": "sleep 3600", + "number_of_containers": 1, + "resource": { + "": 1, + "memory": "512", + "additional": { + "yarn.io/csi-volume": { + "value": 100, + "unit": "Gi", + "attributes" : { + "driver" : "hostpath", + "mountPath" : "/mnt/data" + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index fe4915838e6..4ee15e16481 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -371,7 +371,7 @@ Resource determines the amount of resources (vcores, memory, network, etc.) usab |profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string|| |cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)|| |memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string|| -|additional|A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below.|false|object|| +|additional|A map of resource type name to resource type information. Including value (integer), unit (string) and optional attributes (map). This will be used to specify resource other than cpu and memory. Please refer to example below.|false|object|| ### ResourceInformation