YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well. (Sunil G via wangda)
This commit is contained in:
parent
dfcbbddb09
commit
07b0fb996a
|
@ -616,6 +616,9 @@ Release 2.8.0 - UNRELEASED
|
|||
YARN-4309. Add container launch related debug information to container logs
|
||||
when a container fails. (Varun Vasudev via wangda)
|
||||
|
||||
YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well.
|
||||
(Sunil G via wangda)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
YARN-3339. TestDockerContainerExecutor should pull a single image and not
|
||||
|
|
|
@ -61,7 +61,7 @@ public class ResourceUsage {
|
|||
//CACHED_USED and CACHED_PENDING may be read by anyone, but must only
|
||||
//be written by ordering policies
|
||||
USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
|
||||
CACHED_PENDING(5);
|
||||
CACHED_PENDING(5), AMLIMIT(6);
|
||||
|
||||
private int idx;
|
||||
|
||||
|
@ -92,6 +92,7 @@ public class ResourceUsage {
|
|||
sb.append("pending=" + resArr[1] + "%, ");
|
||||
sb.append("am_used=" + resArr[2] + "%, ");
|
||||
sb.append("reserved=" + resArr[3] + "%}");
|
||||
sb.append("am_limit=" + resArr[6] + "%, ");
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
@ -106,11 +107,19 @@ public class ResourceUsage {
|
|||
public Resource getUsed(String label) {
|
||||
return _get(label, ResourceType.USED);
|
||||
}
|
||||
|
||||
|
||||
public Resource getCachedUsed() {
|
||||
return _get(NL, ResourceType.CACHED_USED);
|
||||
}
|
||||
|
||||
public Resource getCachedUsed(String label) {
|
||||
return _get(label, ResourceType.CACHED_USED);
|
||||
}
|
||||
|
||||
|
||||
public Resource getCachedPending() {
|
||||
return _get(NL, ResourceType.CACHED_PENDING);
|
||||
}
|
||||
|
||||
public Resource getCachedPending(String label) {
|
||||
return _get(label, ResourceType.CACHED_PENDING);
|
||||
}
|
||||
|
@ -149,15 +158,23 @@ public class ResourceUsage {
|
|||
public void setUsed(String label, Resource res) {
|
||||
_set(label, ResourceType.USED, res);
|
||||
}
|
||||
|
||||
|
||||
public void setCachedUsed(String label, Resource res) {
|
||||
_set(label, ResourceType.CACHED_USED, res);
|
||||
}
|
||||
|
||||
|
||||
public void setCachedUsed(Resource res) {
|
||||
_set(NL, ResourceType.CACHED_USED, res);
|
||||
}
|
||||
|
||||
public void setCachedPending(String label, Resource res) {
|
||||
_set(label, ResourceType.CACHED_PENDING, res);
|
||||
}
|
||||
|
||||
public void setCachedPending(Resource res) {
|
||||
_set(NL, ResourceType.CACHED_PENDING, res);
|
||||
}
|
||||
|
||||
/*
|
||||
* Pending
|
||||
*/
|
||||
|
@ -263,6 +280,41 @@ public class ResourceUsage {
|
|||
_set(label, ResourceType.AMUSED, res);
|
||||
}
|
||||
|
||||
/*
|
||||
* AM-Resource Limit
|
||||
*/
|
||||
public Resource getAMLimit() {
|
||||
return getAMLimit(NL);
|
||||
}
|
||||
|
||||
public Resource getAMLimit(String label) {
|
||||
return _get(label, ResourceType.AMLIMIT);
|
||||
}
|
||||
|
||||
public void incAMLimit(String label, Resource res) {
|
||||
_inc(label, ResourceType.AMLIMIT, res);
|
||||
}
|
||||
|
||||
public void incAMLimit(Resource res) {
|
||||
incAMLimit(NL, res);
|
||||
}
|
||||
|
||||
public void decAMLimit(Resource res) {
|
||||
decAMLimit(NL, res);
|
||||
}
|
||||
|
||||
public void decAMLimit(String label, Resource res) {
|
||||
_dec(label, ResourceType.AMLIMIT, res);
|
||||
}
|
||||
|
||||
public void setAMLimit(Resource res) {
|
||||
setAMLimit(NL, res);
|
||||
}
|
||||
|
||||
public void setAMLimit(String label, Resource res) {
|
||||
_set(label, ResourceType.AMLIMIT, res);
|
||||
}
|
||||
|
||||
private static Resource normalize(Resource res) {
|
||||
if (res == null) {
|
||||
return Resources.none();
|
||||
|
|
|
@ -606,6 +606,7 @@ public class LeafQueue extends AbstractCSQueue {
|
|||
minimumAllocation);
|
||||
|
||||
metrics.setAMResouceLimit(amResouceLimit);
|
||||
queueUsage.setAMLimit(nodePartition, amResouceLimit);
|
||||
return amResouceLimit;
|
||||
}
|
||||
|
||||
|
|
|
@ -217,6 +217,11 @@ public class QueueCapacities {
|
|||
}
|
||||
|
||||
/* Absolute Maximum AM resource percentage Getter and Setter */
|
||||
|
||||
public float getMaxAMResourcePercentage() {
|
||||
return _get(NL, CapacityType.MAX_AM_PERC);
|
||||
}
|
||||
|
||||
public float getMaxAMResourcePercentage(String label) {
|
||||
return _get(label, CapacityType.MAX_AM_PERC);
|
||||
}
|
||||
|
@ -225,6 +230,10 @@ public class QueueCapacities {
|
|||
_set(label, CapacityType.MAX_AM_PERC, value);
|
||||
}
|
||||
|
||||
public void setMaxAMResourcePercentage(float value) {
|
||||
_set(NL, CapacityType.MAX_AM_PERC, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear configurable fields, like
|
||||
* (absolute)capacity/(absolute)maximum-capacity, this will be used by queue
|
||||
|
|
|
@ -37,8 +37,8 @@ public class TestResourceUsage {
|
|||
|
||||
@Parameterized.Parameters
|
||||
public static Collection<String[]> getParameters() {
|
||||
return Arrays.asList(new String[][] { { "Pending" }, { "Used" },
|
||||
{ "Reserved" }, { "AMUsed" } });
|
||||
return Arrays.asList(new String[][]{{"Pending"}, {"Used"}, {"Reserved"},
|
||||
{"AMUsed"}, {"AMLimit"}, {"CachedUsed"}, {"CachedPending"}});
|
||||
}
|
||||
|
||||
public TestResourceUsage(String suffix) {
|
||||
|
@ -112,16 +112,26 @@ public class TestResourceUsage {
|
|||
check(0, 0, res);
|
||||
|
||||
// Add 1,1 should returns 1,1
|
||||
inc(usage, suffix, Resource.newInstance(1, 1), label);
|
||||
check(1, 1, get(usage, suffix, label));
|
||||
try {
|
||||
inc(usage, suffix, Resource.newInstance(1, 1), label);
|
||||
check(1, 1, get(usage, suffix, label));
|
||||
} catch (NoSuchMethodException e) {
|
||||
// Few operations need not have to be verified as some resources doesn't
|
||||
// inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
|
||||
}
|
||||
|
||||
// Set 2,2
|
||||
set(usage, suffix, Resource.newInstance(2, 2), label);
|
||||
check(2, 2, get(usage, suffix, label));
|
||||
|
||||
// dec 2,2
|
||||
dec(usage, suffix, Resource.newInstance(2, 2), label);
|
||||
check(0, 0, get(usage, suffix, label));
|
||||
try {
|
||||
dec(usage, suffix, Resource.newInstance(2, 2), label);
|
||||
check(0, 0, get(usage, suffix, label));
|
||||
} catch (NoSuchMethodException e) {
|
||||
// Few operations need not have to be verified, as some resources doesn't
|
||||
// inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
|
||||
}
|
||||
}
|
||||
|
||||
void check(int mem, int cpu, Resource res) {
|
||||
|
|
|
@ -43,7 +43,8 @@ public class TestQueueCapacities {
|
|||
{ "UsedCapacity" },
|
||||
{ "AbsoluteUsedCapacity" },
|
||||
{ "MaximumCapacity" },
|
||||
{ "AbsoluteMaximumCapacity" } });
|
||||
{ "AbsoluteMaximumCapacity" },
|
||||
{ "MaxAMResourcePercentage" } });
|
||||
}
|
||||
|
||||
public TestQueueCapacities(String suffix) {
|
||||
|
|
Loading…
Reference in New Issue