From c915a658c8a5481ca88d8b2f6ce9a72f2e1f5b6a Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Mon, 14 Dec 2015 11:24:30 -0800 Subject: [PATCH] YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well. (Sunil G via wangda) (cherry picked from commit 07b0fb996a32020678bd2ce482b672f0434651f0) --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/ResourceUsage.java | 62 +++++++++++++++++-- .../scheduler/capacity/LeafQueue.java | 1 + .../scheduler/capacity/QueueCapacities.java | 9 +++ .../scheduler/TestResourceUsage.java | 22 +++++-- .../capacity/TestQueueCapacities.java | 3 +- 6 files changed, 88 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index bde35e4e9ea..ee06f9748f1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -564,6 +564,9 @@ Release 2.8.0 - UNRELEASED YARN-4309. Add container launch related debug information to container logs when a container fails. (Varun Vasudev via wangda) + YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well. + (Sunil G via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java index 88e93c1689e..28573792ae5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java @@ -61,7 +61,7 @@ private enum ResourceType { //CACHED_USED and CACHED_PENDING may be read by anyone, but must only //be written by ordering policies USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), - CACHED_PENDING(5); + CACHED_PENDING(5), AMLIMIT(6); private int idx; @@ -92,6 +92,7 @@ public String toString() { sb.append("pending=" + resArr[1] + "%, "); sb.append("am_used=" + resArr[2] + "%, "); sb.append("reserved=" + resArr[3] + "%}"); + sb.append("am_limit=" + resArr[6] + "%, "); return sb.toString(); } } @@ -106,11 +107,19 @@ public Resource getUsed() { public Resource getUsed(String label) { return _get(label, ResourceType.USED); } - + + public Resource getCachedUsed() { + return _get(NL, ResourceType.CACHED_USED); + } + public Resource getCachedUsed(String label) { return _get(label, ResourceType.CACHED_USED); } - + + public Resource getCachedPending() { + return _get(NL, ResourceType.CACHED_PENDING); + } + public Resource getCachedPending(String label) { return _get(label, ResourceType.CACHED_PENDING); } @@ -149,15 +158,23 @@ public void copyAllUsed(ResourceUsage other) { public void setUsed(String label, Resource res) { _set(label, ResourceType.USED, res); } - + public void setCachedUsed(String label, Resource res) { _set(label, ResourceType.CACHED_USED, res); } - + + public void setCachedUsed(Resource res) { + _set(NL, ResourceType.CACHED_USED, res); + } + public void setCachedPending(String label, Resource res) { _set(label, ResourceType.CACHED_PENDING, res); } + public void setCachedPending(Resource res) { + _set(NL, ResourceType.CACHED_PENDING, res); + } + /* * Pending */ @@ -263,6 +280,41 @@ public void setAMUsed(String label, Resource res) { _set(label, ResourceType.AMUSED, res); } + /* + * AM-Resource Limit + */ + public Resource getAMLimit() { + return getAMLimit(NL); + } + + public Resource getAMLimit(String label) { + return _get(label, ResourceType.AMLIMIT); + } + + public void incAMLimit(String label, Resource res) { + _inc(label, ResourceType.AMLIMIT, res); + } + + public void incAMLimit(Resource res) { + incAMLimit(NL, res); + } + + public void decAMLimit(Resource res) { + decAMLimit(NL, res); + } + + public void decAMLimit(String label, Resource res) { + _dec(label, ResourceType.AMLIMIT, res); + } + + public void setAMLimit(Resource res) { + setAMLimit(NL, res); + } + + public void setAMLimit(String label, Resource res) { + _set(label, ResourceType.AMLIMIT, res); + } + private static Resource normalize(Resource res) { if (res == null) { return Resources.none(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 02a786f392e..6c273868736 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -606,6 +606,7 @@ public synchronized Resource getAMResourceLimitPerPartition( minimumAllocation); metrics.setAMResouceLimit(amResouceLimit); + queueUsage.setAMLimit(nodePartition, amResouceLimit); return amResouceLimit; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java index 65100f614b7..f2c26327ae0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java @@ -217,6 +217,11 @@ public void setAbsoluteMaximumCapacity(String label, float value) { } /* Absolute Maximum AM resource percentage Getter and Setter */ + + public float getMaxAMResourcePercentage() { + return _get(NL, CapacityType.MAX_AM_PERC); + } + public float getMaxAMResourcePercentage(String label) { return _get(label, CapacityType.MAX_AM_PERC); } @@ -225,6 +230,10 @@ public void setMaxAMResourcePercentage(String label, float value) { _set(label, CapacityType.MAX_AM_PERC, value); } + public void setMaxAMResourcePercentage(float value) { + _set(NL, CapacityType.MAX_AM_PERC, value); + } + /** * Clear configurable fields, like * (absolute)capacity/(absolute)maximum-capacity, this will be used by queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java index f0bf8925049..1ddeb0b33b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java @@ -37,8 +37,8 @@ public class TestResourceUsage { @Parameterized.Parameters public static Collection getParameters() { - return Arrays.asList(new String[][] { { "Pending" }, { "Used" }, - { "Reserved" }, { "AMUsed" } }); + return Arrays.asList(new String[][]{{"Pending"}, {"Used"}, {"Reserved"}, + {"AMUsed"}, {"AMLimit"}, {"CachedUsed"}, {"CachedPending"}}); } public TestResourceUsage(String suffix) { @@ -112,16 +112,26 @@ private void internalTestModifyAndRead(String label) throws Exception { check(0, 0, res); // Add 1,1 should returns 1,1 - inc(usage, suffix, Resource.newInstance(1, 1), label); - check(1, 1, get(usage, suffix, label)); + try { + inc(usage, suffix, Resource.newInstance(1, 1), label); + check(1, 1, get(usage, suffix, label)); + } catch (NoSuchMethodException e) { + // Few operations need not have to be verified as some resources doesn't + // inc/dec apis exposed (For Eg: CachedUsed and CachedPending). + } // Set 2,2 set(usage, suffix, Resource.newInstance(2, 2), label); check(2, 2, get(usage, suffix, label)); // dec 2,2 - dec(usage, suffix, Resource.newInstance(2, 2), label); - check(0, 0, get(usage, suffix, label)); + try { + dec(usage, suffix, Resource.newInstance(2, 2), label); + check(0, 0, get(usage, suffix, label)); + } catch (NoSuchMethodException e) { + // Few operations need not have to be verified, as some resources doesn't + // inc/dec apis exposed (For Eg: CachedUsed and CachedPending). + } } void check(int mem, int cpu, Resource res) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java index d543f801162..9d2fa15d4e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java @@ -43,7 +43,8 @@ public static Collection getParameters() { { "UsedCapacity" }, { "AbsoluteUsedCapacity" }, { "MaximumCapacity" }, - { "AbsoluteMaximumCapacity" } }); + { "AbsoluteMaximumCapacity" }, + { "MaxAMResourcePercentage" } }); } public TestQueueCapacities(String suffix) {