Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4718. Rename variables in SchedulerNode to reduce ambiguity post YARN-1011. (Inigo Goiri via kasha)

Browse Source
This commit is contained in:
Karthik Kambatla 2016-02-28 09:35:59 -08:00
parent 7545ce6636
commit f9692770a5
25 changed files with 273 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -146,6 +146,9 @@ Release 2.9.0 - UNRELEASED
YARN-4697. NM aggregation thread pool is not bound by
limits (haibochen via rkanter)
YARN-4718. Rename variables in SchedulerNode to reduce ambiguity post
YARN-1011. (Inigo Goiri via kasha)
OPTIMIZATIONS
BUG FIXES

172
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
View File

@ -54,9 +54,9 @@ public abstract class SchedulerNode {
private static final Log LOG = LogFactory.getLog(SchedulerNode.class);
private Resource availableResource = Resource.newInstance(0, 0);
private Resource usedResource = Resource.newInstance(0, 0);
private Resource totalResourceCapability;
private Resource unallocatedResource = Resource.newInstance(0, 0);
private Resource allocatedResource = Resource.newInstance(0, 0);
private Resource totalResource;
private RMContainer reservedContainer;
private volatile int numContainers;
private volatile ResourceUtilization containersUtilization =
@ -65,20 +65,20 @@ public abstract class SchedulerNode {
ResourceUtilization.newInstance(0, 0, 0f);
/* set of containers that are allocated containers */
/** Set of containers that are allocated containers. */
private final Map<ContainerId, RMContainer> launchedContainers =
new HashMap<ContainerId, RMContainer>();
new HashMap<>();
private final RMNode rmNode;
private final String nodeName;
private volatile Set<String> labels = null;
public SchedulerNode(RMNode node, boolean usePortForNodeName,
Set<String> labels) {
this.rmNode = node;
this.availableResource = Resources.clone(node.getTotalCapability());
this.totalResourceCapability = Resources.clone(node.getTotalCapability());
this.unallocatedResource = Resources.clone(node.getTotalCapability());
this.totalResource = Resources.clone(node.getTotalCapability());
if (usePortForNodeName) {
nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort();
} else {
@ -97,23 +97,26 @@ public RMNode getRMNode() {
/**
* Set total resources on the node.
* @param resource total resources on the node.
* @param resource Total resources on the node.
*/
public synchronized void setTotalResource(Resource resource){
this.totalResourceCapability = resource;
this.availableResource = Resources.subtract(totalResourceCapability,
this.usedResource);
this.totalResource = resource;
this.unallocatedResource = Resources.subtract(totalResource,
this.allocatedResource);
}
/**
* Get the ID of the node which contains both its hostname and port.
*
* @return the ID of the node
* @return The ID of the node.
*/
public NodeId getNodeID() {
return this.rmNode.getNodeID();
}
/**
* Get HTTP address for the node.
* @return HTTP address for the node.
*/
public String getHttpAddress() {
return this.rmNode.getHttpAddress();
}
@ -126,8 +129,7 @@ public String getHttpAddress() {
* {@link YarnConfiguration#RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME} constant.
* The main usecase of this is Yarn minicluster to be able to differentiate
* node manager instances by their port number.
*
* @return name of the node for scheduling matching decisions.
* @return Name of the node for scheduling matching decisions.
*/
public String getNodeName() {
return nodeName;
@ -135,7 +137,6 @@ public String getNodeName() {
/**
* Get rackname.
*
* @return rackname
*/
public String getRackName() {
@ -145,13 +146,11 @@ public String getRackName() {
/**
* The Scheduler has allocated containers on this node to the given
* application.
*
* @param rmContainer
* allocated container
* @param rmContainer Allocated container
*/
public synchronized void allocateContainer(RMContainer rmContainer) {
Container container = rmContainer.getContainer();
deductAvailableResource(container.getResource());
deductUnallocatedResource(container.getResource());
++numContainers;
launchedContainers.put(container.getId(), rmContainer);
@ -159,27 +158,35 @@ public synchronized void allocateContainer(RMContainer rmContainer) {
LOG.info("Assigned container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + rmNode.getNodeAddress()
+ ", which has " + numContainers + " containers, "
+ getUsedResource() + " used and " + getAvailableResource()
+ getAllocatedResource() + " used and " + getUnallocatedResource()
+ " available after allocation");
}
/**
* Change the resources allocated for a container.
* @param containerId Identifier of the container to change.
* @param deltaResource Change in the resource allocation.
* @param increase True if the change is an increase of allocation.
*/
private synchronized void changeContainerResource(ContainerId containerId,
Resource deltaResource, boolean increase) {
if (increase) {
deductAvailableResource(deltaResource);
deductUnallocatedResource(deltaResource);
} else {
addAvailableResource(deltaResource);
addUnallocatedResource(deltaResource);
}
LOG.info((increase ? "Increased" : "Decreased") + " container "
+ containerId + " of capacity " + deltaResource + " on host "
+ rmNode.getNodeAddress() + ", which has " + numContainers
+ " containers, " + getUsedResource() + " used and "
+ getAvailableResource() + " available after allocation");
+ " containers, " + getAllocatedResource() + " used and "
+ getUnallocatedResource() + " available after allocation");
}
/**
* The Scheduler increased container
* Increase the resources allocated to a container.
* @param containerId Identifier of the container to change.
* @param deltaResource Increase of resource allocation.
*/
public synchronized void increaseContainer(ContainerId containerId,
Resource deltaResource) {
@ -187,7 +194,9 @@ public synchronized void increaseContainer(ContainerId containerId,
}
/**
* The Scheduler decreased container
* Decrease the resources allocated to a container.
* @param containerId Identifier of the container to change.
* @param deltaResource Decrease of resource allocation.
*/
public synchronized void decreaseContainer(ContainerId containerId,
Resource deltaResource) {
@ -195,32 +204,33 @@ public synchronized void decreaseContainer(ContainerId containerId,
}
/**
* Get available resources on the node.
*
* @return available resources on the node
* Get unallocated resources on the node.
* @return Unallocated resources on the node
*/
public synchronized Resource getAvailableResource() {
return this.availableResource;
public synchronized Resource getUnallocatedResource() {
return this.unallocatedResource;
}
/**
* Get used resources on the node.
*
* @return used resources on the node
* Get allocated resources on the node.
* @return Allocated resources on the node
*/
public synchronized Resource getUsedResource() {
return this.usedResource;
public synchronized Resource getAllocatedResource() {
return this.allocatedResource;
}
/**
* Get total resources on the node.
*
* @return total resources on the node.
* @return Total resources on the node.
*/
public synchronized Resource getTotalResource() {
return this.totalResourceCapability;
return this.totalResource;
}
/**
* Check if a container is launched by this node.
* @return If the container is launched by the node.
*/
public synchronized boolean isValidContainer(ContainerId containerId) {
if (launchedContainers.containsKey(containerId)) {
return true;
@ -228,16 +238,18 @@ public synchronized boolean isValidContainer(ContainerId containerId) {
return false;
}
/**
* Update the resources of the node when allocating a new container.
* @param container Container to allocate.
*/
private synchronized void updateResource(Container container) {
addAvailableResource(container.getResource());
addUnallocatedResource(container.getResource());
--numContainers;
}
/**
* Release an allocated container on this node.
*
* @param container
* container to be released
* @param container Container to be released.
*/
public synchronized void releaseContainer(Container container) {
if (!isValidContainer(container.getId())) {
@ -245,7 +257,7 @@ public synchronized void releaseContainer(Container container) {
return;
}
/* remove the containers from the nodemanger */
// Remove the containers from the nodemanger
if (null != launchedContainers.remove(container.getId())) {
updateResource(container);
}
@ -253,81 +265,118 @@ public synchronized void releaseContainer(Container container) {
LOG.info("Released container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + rmNode.getNodeAddress()
+ ", which currently has " + numContainers + " containers, "
+ getUsedResource() + " used and " + getAvailableResource()
+ getAllocatedResource() + " used and " + getUnallocatedResource()
+ " available" + ", release resources=" + true);
}
private synchronized void addAvailableResource(Resource resource) {
/**
* Add unallocated resources to the node. This is used when unallocating a
* container.
* @param resource Resources to add.
*/
private synchronized void addUnallocatedResource(Resource resource) {
if (resource == null) {
LOG.error("Invalid resource addition of null resource for "
+ rmNode.getNodeAddress());
return;
}
Resources.addTo(availableResource, resource);
Resources.subtractFrom(usedResource, resource);
Resources.addTo(unallocatedResource, resource);
Resources.subtractFrom(allocatedResource, resource);
}
private synchronized void deductAvailableResource(Resource resource) {
/**
* Deduct unallocated resources from the node. This is used when allocating a
* container.
* @param resource Resources to deduct.
*/
private synchronized void deductUnallocatedResource(Resource resource) {
if (resource == null) {
LOG.error("Invalid deduction of null resource for "
+ rmNode.getNodeAddress());
return;
}
Resources.subtractFrom(availableResource, resource);
Resources.addTo(usedResource, resource);
Resources.subtractFrom(unallocatedResource, resource);
Resources.addTo(allocatedResource, resource);
}
/**
* Reserve container for the attempt on this node.
* @param attempt Application attempt asking for the reservation.
* @param priority Priority of the reservation.
* @param container Container reserving resources for.
*/
public abstract void reserveResource(SchedulerApplicationAttempt attempt,
Priority priority, RMContainer container);
/**
* Unreserve resources on this node.
* @param attempt Application attempt that had done the reservation.
*/
public abstract void unreserveResource(SchedulerApplicationAttempt attempt);
@Override
public String toString() {
return "host: " + rmNode.getNodeAddress() + " #containers="
+ getNumContainers() + " available=" + getAvailableResource()
+ " used=" + getUsedResource();
+ getNumContainers() + " available=" + getUnallocatedResource()
+ " used=" + getAllocatedResource();
}
/**
* Get number of active containers on the node.
*
* @return number of active containers on the node
* @return Number of active containers on the node.
*/
public int getNumContainers() {
return numContainers;
}
/**
* Get the running containers in the node.
* @return List of running containers in the node.
*/
public synchronized List<RMContainer> getRunningContainers() {
return new ArrayList<RMContainer>(launchedContainers.values());
}
/**
* Get the reserved container in the node.
* @return Reserved container in the node.
*/
public synchronized RMContainer getReservedContainer() {
return reservedContainer;
}
/**
* Set the reserved container in the node.
* @param reservedContainer Reserved container in the node.
*/
protected synchronized void
setReservedContainer(RMContainer reservedContainer) {
this.reservedContainer = reservedContainer;
}
/**
* Recover a container.
* @param rmContainer Container to recover.
*/
public synchronized void recoverContainer(RMContainer rmContainer) {
if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
return;
}
allocateContainer(rmContainer);
}
/**
* Get the labels for the node.
* @return Set of labels for the node.
*/
public Set<String> getLabels() {
return labels;
}
/**
* Update the labels for the node.
* @param labels Set of labels for the node.
*/
public void updateLabels(Set<String> labels) {
this.labels = labels;
}
@ -336,6 +385,7 @@ public void updateLabels(Set<String> labels) {
* Get partition of which the node belongs to, if node-labels of this node is
* empty or null, it belongs to NO_LABEL partition. And since we only support
* one partition for each node (YARN-2694), first label will be its partition.
* @return Partition for the node.
*/
public String getPartition() {
if (this.labels == null || this.labels.isEmpty()) {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
View File

@ -33,8 +33,8 @@ public class SchedulerNodeReport {
private final int num;
public SchedulerNodeReport(SchedulerNode node) {
this.used = node.getUsedResource();
this.avail = node.getAvailableResource();
this.used = node.getAllocatedResource();
this.avail = node.getUnallocatedResource();
this.num = node.getNumContainers();
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
View File

@ -1096,7 +1096,7 @@ private synchronized void nodeUpdate(RMNode nm) {
.handle(
new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption
.newInstance(getSchedulerNode(nm.getNodeID())
.getUsedResource(), 0)));
.getAllocatedResource(), 0)));
}
schedulerHealth.updateSchedulerReleaseDetails(lastNodeUpdateTime,
releaseResources);
@ -1109,8 +1109,8 @@ private synchronized void nodeUpdate(RMNode nm) {
// Now node data structures are upto date and ready for scheduling.
if(LOG.isDebugEnabled()) {
LOG.debug("Node being looked for scheduling " + nm
+ " availableResource: " + node.getAvailableResource());
LOG.debug("Node being looked for scheduling " + nm +
" availableResource: " + node.getUnallocatedResource());
}
}
@ -1254,11 +1254,11 @@ protected synchronized void allocateContainersToNode(FiCaSchedulerNode node) {
// Try to schedule more if there are no reservations to fulfill
if (node.getReservedContainer() == null) {
if (calculator.computeAvailableContainers(node.getAvailableResource(),
if (calculator.computeAvailableContainers(node.getUnallocatedResource(),
minimumAllocation) > 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("Trying to schedule on node: " + node.getNodeName() +
", available: " + node.getAvailableResource());
", available: " + node.getUnallocatedResource());
}
assignment = root.assignContainers(

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
View File

@ -496,7 +496,7 @@ public synchronized CSAssignment assignContainers(Resource clusterResource,
private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) {
return (node.getReservedContainer() == null) &&
Resources.greaterThanOrEqual(resourceCalculator, clusterResource,
node.getAvailableResource(), minimumAllocation);
node.getUnallocatedResource(), minimumAllocation);
}
private ResourceLimits getResourceLimitsOfChild(CSQueue child,

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
View File

@ -112,7 +112,7 @@ private CSAssignment allocateIncreaseRequestFromReservedContainer(
SchedulerNode node, Resource cluster,
SchedContainerChangeRequest increaseRequest) {
if (Resources.fitsIn(rc, cluster, increaseRequest.getDeltaCapacity(),
node.getAvailableResource())) {
node.getUnallocatedResource())) {
// OK, we can allocate this increase request
// Unreserve it first
application.unreserve(increaseRequest.getPriority(),
@ -141,7 +141,7 @@ private CSAssignment allocateIncreaseRequestFromReservedContainer(
private CSAssignment allocateIncreaseRequest(FiCaSchedulerNode node,
Resource cluster, SchedContainerChangeRequest increaseRequest) {
if (Resources.fitsIn(rc, cluster, increaseRequest.getDeltaCapacity(),
node.getAvailableResource())) {
node.getUnallocatedResource())) {
// Notify node
node.increaseContainer(increaseRequest.getContainerId(),
increaseRequest.getDeltaCapacity());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
View File

@ -423,7 +423,7 @@ private ContainerAllocation assignContainer(Resource clusterResource,
}
Resource capability = request.getCapability();
Resource available = node.getAvailableResource();
Resource available = node.getUnallocatedResource();
Resource totalResource = node.getTotalResource();
if (!Resources.lessThanOrEqual(rc, clusterResource,

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
View File

@ -589,7 +589,7 @@ public void updateNodeInfoForAMDiagnostics(FiCaSchedulerNode node) {
diagnosticMessageBldr.append(", Total resource : ");
diagnosticMessageBldr.append(node.getTotalResource());
diagnosticMessageBldr.append(", Available resource : ");
diagnosticMessageBldr.append(node.getAvailableResource());
diagnosticMessageBldr.append(node.getUnallocatedResource());
diagnosticMessageBldr.append(" ).");
updateAMContainerDiagnostics(AMState.ACTIVATED, diagnosticMessageBldr.toString());
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
View File

@ -191,7 +191,7 @@ private void subtractResourcesOnBlacklistedNodes(
SchedulerNode node = scheduler.getSchedulerNode(nodeId);
if (node != null) {
Resources.subtractFrom(availableResources,
node.getAvailableResource());
node.getUnallocatedResource());
}
}
if (availableResources.getMemory() < 0) {
@ -613,7 +613,7 @@ private Resource assignContainer(
Resource capability = request.getCapability();
// How much does the node have?
Resource available = node.getAvailableResource();
Resource available = node.getUnallocatedResource();
Container container = null;
if (reserved) {
@ -840,7 +840,7 @@ public boolean assignReservedContainer(FSSchedulerNode node) {
// Note that we have an assumption here that
// there's only one container size per priority.
if (Resources.fitsIn(node.getReservedContainer().getReservedResource(),
node.getAvailableResource())) {
node.getUnallocatedResource())) {
assignContainer(node, true);
}
return true;

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
View File

@ -1069,7 +1069,7 @@ private synchronized void nodeUpdate(RMNode nm) {
.handle(
new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption
.newInstance(getSchedulerNode(nm.getNodeID())
.getUsedResource(), 0)));
.getAllocatedResource(), 0)));
}
if (continuousSchedulingEnabled) {
@ -1105,7 +1105,7 @@ void continuousSchedulingAttempt() throws InterruptedException {
FSSchedulerNode node = getFSSchedulerNode(nodeId);
try {
if (node != null && Resources.fitsIn(minimumAllocation,
node.getAvailableResource())) {
node.getUnallocatedResource())) {
attemptScheduling(node);
}
} catch (Throwable ex) {
@ -1137,8 +1137,8 @@ public int compare(NodeId n1, NodeId n2) {
return -1;
}
return RESOURCE_CALCULATOR.compare(clusterResource,
nodes.get(n2).getAvailableResource(),
nodes.get(n1).getAvailableResource());
nodes.get(n2).getUnallocatedResource(),
nodes.get(n1).getUnallocatedResource());
}
}

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
View File

@ -527,7 +527,7 @@ private void assignContainers(FiCaSchedulerNode node) {
// Done
if (Resources.lessThan(resourceCalculator, clusterResource,
node.getAvailableResource(), minimumAllocation)) {
node.getUnallocatedResource(), minimumAllocation)) {
break;
}
}
@ -682,13 +682,9 @@ private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application
" request=" + request + " type=" + type);
Resource capability = request.getCapability();
int availableContainers =
node.getAvailableResource().getMemory() / capability.getMemory(); // TODO: A buggy
// application
// with this
// zero would
// crash the
// scheduler.
// TODO: A buggy application with this zero would crash the scheduler.
int availableContainers = node.getUnallocatedResource().getMemory() /
capability.getMemory();
int assignedContainers =
Math.min(assignableContainers, availableContainers);
@ -760,7 +756,7 @@ private synchronized void nodeUpdate(RMNode rmNode) {
.handle(
new RMNodeResourceUpdateEvent(rmNode.getNodeID(), ResourceOption
.newInstance(getSchedulerNode(rmNode.getNodeID())
.getUsedResource(), 0)));
.getAllocatedResource(), 0)));
}
if (rmContext.isWorkPreservingRecoveryEnabled()
@ -769,14 +765,14 @@ private synchronized void nodeUpdate(RMNode rmNode) {
}
if (Resources.greaterThanOrEqual(resourceCalculator, clusterResource,
node.getAvailableResource(),minimumAllocation)) {
node.getUnallocatedResource(), minimumAllocation)) {
LOG.debug("Node heartbeat " + rmNode.getNodeID() +
" available resource = " + node.getAvailableResource());
" available resource = " + node.getUnallocatedResource());
assignContainers(node);
LOG.debug("Node after allocation " + rmNode.getNodeID() + " resource = "
+ node.getAvailableResource());
+ node.getUnallocatedResource());
}
updateAvailableResourcesMetrics();

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
View File

@ -213,11 +213,11 @@ public void testSchedulerRecovery() throws Exception {
assertTrue(
"SchedulerNode#toString is not in expected format",
schedulerNode1
.toString().contains(schedulerNode1.getAvailableResource().toString()));
.toString().contains(schedulerNode1.getUnallocatedResource().toString()));
assertTrue(
"SchedulerNode#toString is not in expected format",
schedulerNode1
.toString().contains(schedulerNode1.getUsedResource().toString()));
.toString().contains(schedulerNode1.getAllocatedResource().toString()));
// ********* check scheduler node state.*******
// 2 running containers.
@ -234,8 +234,8 @@ public void testSchedulerRecovery() throws Exception {
assertEquals(2, schedulerNode1.getNumContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
schedulerNode1.getAvailableResource());
assertEquals(usedResources, schedulerNode1.getUsedResource());
schedulerNode1.getUnallocatedResource());
assertEquals(usedResources, schedulerNode1.getAllocatedResource());
Resource availableResources = Resources.subtract(nmResource, usedResources);
// ***** check queue state based on the underlying scheduler ********
@ -379,8 +379,8 @@ public void testDynamicQueueRecovery() throws Exception {
assertEquals(2, schedulerNode1.getNumContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
schedulerNode1.getAvailableResource());
assertEquals(usedResources, schedulerNode1.getUsedResource());
schedulerNode1.getUnallocatedResource());
assertEquals(usedResources, schedulerNode1.getAllocatedResource());
Resource availableResources = Resources.subtract(nmResource, usedResources);
// 6. Verify the scheduler state like attempt info.

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
View File

@ -268,12 +268,12 @@ public void testUpdateMaxAllocationUsesTotal() throws IOException {
SchedulerNode mockNode1 = mock(SchedulerNode.class);
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080));
when(mockNode1.getAvailableResource()).thenReturn(emptyResource);
when(mockNode1.getUnallocatedResource()).thenReturn(emptyResource);
when(mockNode1.getTotalResource()).thenReturn(fullResource1);
SchedulerNode mockNode2 = mock(SchedulerNode.class);
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081));
when(mockNode2.getAvailableResource()).thenReturn(emptyResource);
when(mockNode2.getUnallocatedResource()).thenReturn(emptyResource);
when(mockNode2.getTotalResource()).thenReturn(fullResource2);
verifyMaximumResourceCapability(configuredMaximumResource, scheduler);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
View File

@ -3431,7 +3431,7 @@ public void handle(Event event) {
Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
Resource usedResource =
resourceManager.getResourceScheduler()
.getSchedulerNode(nm_0.getNodeId()).getUsedResource();
.getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
Assert.assertEquals(usedResource.getMemory(), 1 * GB);
Assert.assertEquals(usedResource.getVirtualCores(), 1);
// Check total resource of scheduler node is also changed to 1 GB 1 core
@ -3443,7 +3443,7 @@ public void handle(Event event) {
// Check the available resource is 0/0
Resource availableResource =
resourceManager.getResourceScheduler()
.getSchedulerNode(nm_0.getNodeId()).getAvailableResource();
.getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
Assert.assertEquals(availableResource.getMemory(), 0);
Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
View File

@ -147,9 +147,9 @@ public CSAssignment answer(InvocationOnMock invocation) throws Throwable {
any(ResourceLimits.class), any(SchedulingMode.class));
// Mock the node's resource availability
Resource available = node.getAvailableResource();
Resource available = node.getUnallocatedResource();
doReturn(Resources.subtractFrom(available, allocatedResource)).
when(node).getAvailableResource();
when(node).getUnallocatedResource();
}
return new CSAssignment(allocatedResource, type);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
View File

@ -388,7 +388,7 @@ public void testExcessReservationWillBeUnreserved() throws Exception {
// NM1 has available resource = 2G (8G - 2 * 1G - 4G)
Assert.assertEquals(2 * GB, cs.getNode(nm1.getNodeId())
.getAvailableResource().getMemory());
.getUnallocatedResource().getMemory());
Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Usage of queue = 4G + 2 * 1G + 4G (reserved)
Assert.assertEquals(10 * GB, cs.getRootQueue().getQueueResourceUsage()
@ -401,7 +401,7 @@ public void testExcessReservationWillBeUnreserved() throws Exception {
// App2's reservation will be cancelled
Assert.assertTrue(schedulerApp2.getReservedContainers().size() == 0);
Assert.assertEquals(2 * GB, cs.getNode(nm1.getNodeId())
.getAvailableResource().getMemory());
.getUnallocatedResource().getMemory());
Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
Assert.assertEquals(6 * GB, cs.getRootQueue().getQueueResourceUsage()
.getUsed().getMemory());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
View File

@ -1121,6 +1121,6 @@ private void verifyAvailableResourceOfSchedulerNode(MockRM rm, NodeId nodeId,
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
SchedulerNode node = cs.getNode(nodeId);
Assert
.assertEquals(expectedMemory, node.getAvailableResource().getMemory());
.assertEquals(expectedMemory, node.getUnallocatedResource().getMemory());
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
View File

@ -429,7 +429,7 @@ private void verifyAvailableResourceOfSchedulerNode(MockRM rm, NodeId nodeId,
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
SchedulerNode node = cs.getNode(nodeId);
Assert
.assertEquals(expectedMemory, node.getAvailableResource().getMemory());
.assertEquals(expectedMemory, node.getUnallocatedResource().getMemory());
}
private Container getContainer(

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -1311,7 +1311,7 @@ public void testReservation() throws Exception {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(2*GB, node_0.getUsedResource().getMemory());
assertEquals(2*GB, node_0.getAllocatedResource().getMemory());
assertEquals(4*GB, a.getMetrics().getReservedMB());
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
@ -1328,7 +1328,7 @@ public void testReservation() throws Exception {
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(1*GB, node_0.getUsedResource().getMemory());
assertEquals(1*GB, node_0.getAllocatedResource().getMemory());
assertEquals(4*GB, a.getMetrics().getReservedMB());
assertEquals(1*GB, a.getMetrics().getAllocatedMB());
@ -1345,7 +1345,7 @@ public void testReservation() throws Exception {
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
assertEquals(4*GB, node_0.getUsedResource().getMemory());
assertEquals(4*GB, node_0.getAllocatedResource().getMemory());
assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(4*GB, a.getMetrics().getAllocatedMB());
}
@ -1434,7 +1434,7 @@ public void testReservationExchange() throws Exception {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(2*GB, node_0.getUsedResource().getMemory());
assertEquals(2*GB, node_0.getAllocatedResource().getMemory());
// Now free 1 container from app_0 i.e. 1G, and re-reserve it
RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
@ -1449,7 +1449,7 @@ public void testReservationExchange() throws Exception {
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(1*GB, node_0.getUsedResource().getMemory());
assertEquals(1*GB, node_0.getAllocatedResource().getMemory());
assertEquals(1, app_1.getReReservations(priority));
// Re-reserve
@ -1459,7 +1459,7 @@ public void testReservationExchange() throws Exception {
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(1*GB, node_0.getUsedResource().getMemory());
assertEquals(1*GB, node_0.getAllocatedResource().getMemory());
assertEquals(2, app_1.getReReservations(priority));
// Try to schedule on node_1 now, should *move* the reservation
@ -1469,7 +1469,7 @@ public void testReservationExchange() throws Exception {
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(4*GB, node_1.getUsedResource().getMemory());
assertEquals(4*GB, node_1.getAllocatedResource().getMemory());
// Doesn't change yet... only when reservation is cancelled or a different
// container is reserved
assertEquals(2, app_1.getReReservations(priority));
@ -1487,7 +1487,7 @@ public void testReservationExchange() throws Exception {
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
assertEquals(0*GB, node_0.getUsedResource().getMemory());
assertEquals(0*GB, node_0.getAllocatedResource().getMemory());
}
private void verifyContainerAllocated(CSAssignment assignment, NodeType nodeType) {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
View File

@ -158,9 +158,9 @@ public CSAssignment answer(InvocationOnMock invocation) throws Throwable {
any(ResourceLimits.class), any(SchedulingMode.class));
// Mock the node's resource availability
Resource available = node.getAvailableResource();
Resource available = node.getUnallocatedResource();
doReturn(Resources.subtractFrom(available, allocatedResource)).
when(node).getAvailableResource();
when(node).getUnallocatedResource();
}
return new CSAssignment(allocatedResource, type);

196
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
View File

@ -263,9 +263,9 @@ public void testReservation() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(22 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@ -275,9 +275,9 @@ public void testReservation() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(19 * GB, a.getMetrics().getAvailableMB());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@ -289,9 +289,9 @@ public void testReservation() throws Exception {
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
assertEquals(16 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// try to assign reducer (5G on node 0 and should reserve)
@ -305,9 +305,9 @@ public void testReservation() throws Exception {
assertEquals(11 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// assign reducer to node 2
@ -321,9 +321,9 @@ public void testReservation() throws Exception {
assertEquals(6 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(1, app_0.getTotalRequiredResources(priorityReduce));
// node_1 heartbeat and unreserves from node_0 in order to allocate
@ -337,9 +337,9 @@ public void testReservation() throws Exception {
assertEquals(6 * GB, a.getMetrics().getAvailableMB());
assertEquals(6 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(8 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(8 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(0, app_0.getTotalRequiredResources(priorityReduce));
}
@ -425,9 +425,9 @@ public void testReservationLimitOtherUsers() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(22 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
a.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
@ -437,9 +437,9 @@ public void testReservationLimitOtherUsers() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(4 * GB, a.getMetrics().getAllocatedMB());
assertEquals(20 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(2 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(2 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Add a few requests to each app
app_0.updateResourceRequests(Collections.singletonList(TestUtils
@ -458,9 +458,9 @@ public void testReservationLimitOtherUsers() throws Exception {
assertEquals(8 * GB, a.getMetrics().getReservedMB());
assertEquals(4 * GB, a.getMetrics().getAllocatedMB());
assertEquals(12 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(2 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(2 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// next assignment is beyond user limit for user_0 but it should assign to
// app_1 for user_1
@ -472,9 +472,9 @@ public void testReservationLimitOtherUsers() throws Exception {
assertEquals(8 * GB, a.getMetrics().getReservedMB());
assertEquals(6 * GB, a.getMetrics().getAllocatedMB());
assertEquals(10 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(4 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(4 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
}
@Test
@ -559,9 +559,9 @@ public void testReservationNoContinueLook() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(22 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@ -571,9 +571,9 @@ public void testReservationNoContinueLook() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(19 * GB, a.getMetrics().getAvailableMB());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@ -585,9 +585,9 @@ public void testReservationNoContinueLook() throws Exception {
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
assertEquals(16 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// try to assign reducer (5G on node 0 and should reserve)
@ -601,9 +601,9 @@ public void testReservationNoContinueLook() throws Exception {
assertEquals(11 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// assign reducer to node 2
@ -617,9 +617,9 @@ public void testReservationNoContinueLook() throws Exception {
assertEquals(6 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(1, app_0.getTotalRequiredResources(priorityReduce));
// node_1 heartbeat and won't unreserve from node_0, potentially stuck
@ -634,9 +634,9 @@ public void testReservationNoContinueLook() throws Exception {
assertEquals(6 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(1, app_0.getTotalRequiredResources(priorityReduce));
}
@ -718,8 +718,8 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(14 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@ -729,8 +729,8 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(11 * GB, a.getMetrics().getAvailableMB());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@ -742,8 +742,8 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
assertEquals(8 * GB, a.getMetrics().getAvailableMB());
assertEquals(8 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// try to assign reducer (5G on node 0 and should reserve)
@ -757,8 +757,8 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// could allocate but told need to unreserve first
@ -771,8 +771,8 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(8 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(8 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(1, app_0.getTotalRequiredResources(priorityReduce));
}
@ -981,8 +981,8 @@ public void testAssignToQueue() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(14 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@ -992,8 +992,8 @@ public void testAssignToQueue() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(11 * GB, a.getMetrics().getAvailableMB());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@ -1004,8 +1004,8 @@ public void testAssignToQueue() throws Exception {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(8 * GB, a.getMetrics().getAvailableMB());
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
// now add in reservations and make sure it continues if config set
// allocate to queue so that the potential new capacity is greater then
@ -1018,8 +1018,8 @@ public void testAssignToQueue() throws Exception {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
ResourceLimits limits =
new ResourceLimits(Resources.createResource(13 * GB));
@ -1155,8 +1155,8 @@ public void testAssignToUser() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(14 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@ -1166,8 +1166,8 @@ public void testAssignToUser() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(11 * GB, a.getMetrics().getAvailableMB());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@ -1178,8 +1178,8 @@ public void testAssignToUser() throws Exception {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(8 * GB, a.getMetrics().getAvailableMB());
assertEquals(null, node_0.getReservedContainer());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
// now add in reservations and make sure it continues if config set
// allocate to queue so that the potential new capacity is greater then
@ -1194,8 +1194,8 @@ public void testAssignToUser() throws Exception {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
// not over the limit
Resource limit = Resources.createResource(14 * GB, 0);
@ -1307,9 +1307,9 @@ public void testReservationsNoneAvailable() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(22 * GB, a.getMetrics().getAvailableMB());
assertEquals(2 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@ -1319,9 +1319,9 @@ public void testReservationsNoneAvailable() throws Exception {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(19 * GB, a.getMetrics().getAvailableMB());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(0 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@ -1332,9 +1332,9 @@ public void testReservationsNoneAvailable() throws Exception {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
assertEquals(16 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// try to assign reducer (5G on node 0), but tell it's resource limits <
// used (8G) + required (5G). It will not reserved since it has to unreserve
@ -1349,9 +1349,9 @@ public void testReservationsNoneAvailable() throws Exception {
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
// app_0's headroom = limit (10G) - used (8G) = 2G
assertEquals(2 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// try to assign reducer (5G on node 0), but tell it's resource limits <
// used (8G) + required (5G). It will not reserved since it has to unreserve
@ -1365,9 +1365,9 @@ public void testReservationsNoneAvailable() throws Exception {
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
// app_0's headroom = limit (10G) - used (8G) = 2G
assertEquals(2 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(0 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// let it assign 5G to node_2
a.assignContainers(clusterResource, node_2,
@ -1378,9 +1378,9 @@ public void testReservationsNoneAvailable() throws Exception {
assertEquals(13 * GB, a.getMetrics().getAllocatedMB());
assertEquals(11 * GB, a.getMetrics().getAvailableMB());
assertEquals(11 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
// reserve 8G node_0
a.assignContainers(clusterResource, node_0,
@ -1391,9 +1391,9 @@ public void testReservationsNoneAvailable() throws Exception {
assertEquals(13 * GB, a.getMetrics().getAllocatedMB());
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
// try to assign (8G on node 2). No room to allocate,
// continued to try due to having reservation above,
@ -1406,8 +1406,8 @@ public void testReservationsNoneAvailable() throws Exception {
assertEquals(13 * GB, a.getMetrics().getAllocatedMB());
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getUsedResource().getMemory());
assertEquals(3 * GB, node_1.getUsedResource().getMemory());
assertEquals(5 * GB, node_2.getUsedResource().getMemory());
assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
View File

@ -195,7 +195,7 @@ public static FiCaSchedulerNode getMockNode(
when(rmNode.getRackName()).thenReturn(rack);
FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false));
LOG.info("node = " + host + " avail=" + node.getAvailableResource());
LOG.info("node = " + host + " avail=" + node.getUnallocatedResource());
when(node.getNodeID()).thenReturn(nodeId);
return node;

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
View File

@ -318,7 +318,7 @@ public void testHeadroomWithBlackListedNodes() {
.when(spyApp).isWaitingForAMContainer();
assertTrue(spyApp.isBlacklisted(n1.getNodeName()));
assertFalse(spyApp.isBlacklisted(n2.getNodeName()));
assertEquals(n2.getAvailableResource(), spyApp.getHeadroom());
assertEquals(n2.getUnallocatedResource(), spyApp.getHeadroom());
blacklistAdditions.clear();
blacklistAdditions.add(n2.getNodeName());
@ -326,7 +326,7 @@ public void testHeadroomWithBlackListedNodes() {
app.updateBlacklist(blacklistAdditions, blacklistRemovals);
assertFalse(spyApp.isBlacklisted(n1.getNodeName()));
assertTrue(spyApp.isBlacklisted(n2.getNodeName()));
assertEquals(n1.getAvailableResource(), spyApp.getHeadroom());
assertEquals(n1.getUnallocatedResource(), spyApp.getHeadroom());
blacklistAdditions.clear();
blacklistRemovals.clear();

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
View File

@ -4433,7 +4433,7 @@ public void handle(Event event) {
// Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
Resource usedResource =
resourceManager.getResourceScheduler()
.getSchedulerNode(nm_0.getNodeId()).getUsedResource();
.getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
Assert.assertEquals(usedResource.getMemory(), 0);
Assert.assertEquals(usedResource.getVirtualCores(), 0);
// Check total resource of scheduler node is also changed to 0 GB 0 core
@ -4445,7 +4445,7 @@ public void handle(Event event) {
// Check the available resource is 0/0
Resource availableResource =
resourceManager.getResourceScheduler()
.getSchedulerNode(nm_0.getNodeId()).getAvailableResource();
.getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
Assert.assertEquals(availableResource.getMemory(), 0);
Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
View File

@ -348,7 +348,7 @@ public Map<NodeId, FiCaSchedulerNode> getNodes(){
assertEquals(schedulerNodes.get(node0.getNodeID()).getTotalResource()
.getMemory(), 1024);
assertEquals(schedulerNodes.get(node0.getNodeID()).
getAvailableResource().getMemory(), 1024);
getUnallocatedResource().getMemory(), 1024);
QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
@ -1247,7 +1247,7 @@ public void handle(Event event) {
// Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
Resource usedResource =
resourceManager.getResourceScheduler()
.getSchedulerNode(nm_0.getNodeId()).getUsedResource();
.getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
Assert.assertEquals(usedResource.getMemory(), 1 * GB);
Assert.assertEquals(usedResource.getVirtualCores(), 1);
// Check total resource of scheduler node is also changed to 1 GB 1 core
@ -1259,7 +1259,7 @@ public void handle(Event event) {
// Check the available resource is 0/0
Resource availableResource =
resourceManager.getResourceScheduler()
.getSchedulerNode(nm_0.getNodeId()).getAvailableResource();
.getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
Assert.assertEquals(availableResource.getMemory(), 0);
Assert.assertEquals(availableResource.getVirtualCores(), 0);
}