diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 425a0b12d7c..fd50f203d9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; @@ -797,7 +796,7 @@ public abstract class AbstractYarnScheduler writeLock.unlock(); } } - + /** * Process resource update on a node. */ @@ -900,12 +899,12 @@ public abstract class AbstractYarnScheduler LOG.info("Updated the cluste max priority to maxClusterLevelAppPriority = " + maxClusterLevelAppPriority); } - + /** * Sanity check increase/decrease request, and return * SchedulerContainerResourceChangeRequest according to given * UpdateContainerRequest. - * + * *
* - Returns non-null value means validation succeeded * - Throw exception when any other error happens @@ -1334,9 +1333,7 @@ public abstract class AbstractYarnScheduler } /* - * Get a Resource object with for the minimum allocation possible. If resource - * profiles are enabled then the 'minimum' resource profile will be used. If - * they are not enabled, use the minimums specified in the config files. + * Get a Resource object with for the minimum allocation possible. * * @return a Resource object with the minimum allocation for the scheduler */ @@ -1347,9 +1344,7 @@ public abstract class AbstractYarnScheduler } /** - * Get a Resource object with for the maximum allocation possible. If resource - * profiles are enabled then the 'maximum' resource profile will be used. If - * they are not enabled, use the maximums specified in the config files. + * Get a Resource object with for the maximum allocation possible. * * @return a Resource object with the maximum allocation for the scheduler */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index db92d7c98dc..a02ac8429de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -47,14 +47,14 @@ import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.YarnVersionInfo; +import org.apache.hadoop.yarn.util.resource.Resources; import org.mortbay.log.Log; public class MockNM { private int responseId; private NodeId nodeId; - private long memory; - private int vCores; + private Resource capatibility; private ResourceTrackerService resourceTracker; private int httpPort = 2; private MasterKey currentContainerTokenMasterKey; @@ -75,13 +75,25 @@ public class MockNM { public MockNM(String nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker) { - this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.getVersion()); + this(nodeIdStr, memory, vcores, resourceTracker, + YarnVersionInfo.getVersion()); } public MockNM(String nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker, String version) { - this.memory = memory; - this.vCores = vcores; + this(nodeIdStr, Resource.newInstance(memory, vcores), resourceTracker, + version); + } + + public MockNM(String nodeIdStr, Resource capatibility, + ResourceTrackerService resourceTracker) { + this(nodeIdStr, capatibility, resourceTracker, + YarnVersionInfo.getVersion()); + } + + public MockNM(String nodeIdStr, Resource capatibility, + ResourceTrackerService resourceTracker, String version) { + this.capatibility = capatibility; this.resourceTracker = resourceTracker; this.version = version; String[] splits = nodeIdStr.split(":"); @@ -146,8 +158,7 @@ public class MockNM { RegisterNodeManagerRequest.class); req.setNodeId(nodeId); req.setHttpPort(httpPort); - Resource resource = BuilderUtils.newResource(memory, vCores); - req.setResource(resource); + req.setResource(capatibility); req.setContainerStatuses(containerReports); req.setNMVersion(version); req.setRunningApplications(runningApplications); @@ -158,8 +169,7 @@ public class MockNM { this.currentNMTokenMasterKey = registrationResponse.getNMTokenMasterKey(); Resource newResource = registrationResponse.getResource(); if (newResource != null) { - memory = (int) newResource.getMemorySize(); - vCores = newResource.getVirtualCores(); + capatibility = Resources.clone(newResource); } containerStats.clear(); if (containerReports != null) { @@ -185,7 +195,7 @@ public class MockNM { long containerId, ContainerState containerState) throws Exception { ContainerStatus containerStatus = BuilderUtils.newContainerStatus( BuilderUtils.newContainerId(attemptId, containerId), containerState, - "Success", 0, BuilderUtils.newResource(memory, vCores)); + "Success", 0, capatibility); ArrayListcontainerStatusList = new ArrayList (1); containerStatusList.add(containerStatus); @@ -266,19 +276,22 @@ public class MockNM { Resource newResource = heartbeatResponse.getResource(); if (newResource != null) { - memory = newResource.getMemorySize(); - vCores = newResource.getVirtualCores(); + capatibility = Resources.clone(newResource); } return heartbeatResponse; } public long getMemory() { - return memory; + return capatibility.getMemorySize(); } public int getvCores() { - return vCores; + return capatibility.getVirtualCores(); + } + + public Resource getCapatibility() { + return capatibility; } public String getVersion() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 07545845f5c..fd327f1df2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -848,6 +848,15 @@ public class MockRM extends ResourceManager { return nm; } + public MockNM registerNode(String nodeIdStr, Resource nodeCapatibility) + throws Exception { + MockNM nm = new MockNM(nodeIdStr, nodeCapatibility, + getResourceTrackerService()); + nm.registerNode(); + drainEventsImplicitly(); + return nm; + } + public void sendNodeStarted(MockNM nm) throws Exception { RMNodeImpl node = (RMNodeImpl) getRMContext().getRMNodes().get( nm.getNodeId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index e81ffbd5354..cb150e0363b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -18,16 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; - +import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -38,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; @@ -53,20 +45,28 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; public class TestUtils { private static final Log LOG = LogFactory.getLog(TestUtils.class); @@ -456,4 +456,21 @@ public class TestUtils { cs.submitResourceCommitRequest(clusterResource, csAssignment); } + + /** + * An easy way to create resources other than memory and vcores for tests. + * @param memory memory + * @param vcores vcores + * @param nameToValues resource types other than memory and vcores. + * @return created resource + */ + public static Resource createResource(long memory, int vcores, + Map nameToValues) { + Resource res = Resource.newInstance(memory, vcores); + for (Map.Entry entry : nameToValues.entrySet()) { + res.setResourceInformation(entry.getKey(), ResourceInformation + .newInstance(entry.getKey(), "", entry.getValue())); + } + return res; + } }