From 520033b1cd81c76b38fcdcfcfeed16158db4bbba Mon Sep 17 00:00:00 2001 From: Bikas Saha Date: Thu, 28 Mar 2013 19:44:28 +0000 Subject: [PATCH] YARN-450. Define value for * in the scheduling protocol (Zhijie Shen via bikas) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1462271 13f79535-47bb-0310-9956-ffa450edef68 --- .../v2/app/rm/RMContainerRequestor.java | 5 +- .../mapreduce/v2/app/MRAppBenchmark.java | 2 +- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/api/records/ResourceRequest.java | 20 +++++++ .../apache/hadoop/yarn/client/AMRMClient.java | 5 -- .../hadoop/yarn/client/AMRMClientImpl.java | 6 +- .../hadoop/yarn/client/TestAMRMClient.java | 18 +++--- .../rmapp/attempt/RMAppAttemptImpl.java | 6 +- .../server/resourcemanager/rmnode/RMNode.java | 2 - .../scheduler/AppSchedulingInfo.java | 14 ++--- .../scheduler/capacity/LeafQueue.java | 8 +-- .../common/fica/FiCaSchedulerApp.java | 3 +- .../common/fica/FiCaSchedulerNode.java | 2 - .../scheduler/fair/AppSchedulable.java | 3 +- .../scheduler/fair/FSSchedulerApp.java | 5 +- .../scheduler/fair/FSSchedulerNode.java | 2 - .../scheduler/fifo/FifoScheduler.java | 8 +-- .../server/resourcemanager/Application.java | 11 +--- .../yarn/server/resourcemanager/MockAM.java | 4 +- .../yarn/server/resourcemanager/Task.java | 4 +- .../resourcemanager/TestFifoScheduler.java | 8 +-- .../resourcemanager/TestResourceManager.java | 3 +- .../capacity/TestApplicationLimits.java | 9 ++- .../capacity/TestCapacityScheduler.java | 3 +- .../scheduler/capacity/TestLeafQueue.java | 55 +++++++++---------- .../scheduler/fair/TestFairScheduler.java | 30 ++++++---- .../scheduler/fifo/TestFifoScheduler.java | 10 ++-- .../server/TestContainerManagerSecurity.java | 9 +-- 28 files changed, 132 insertions(+), 126 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index d9a2c1f4ecc..79c97689c1f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -54,7 +54,6 @@ public abstract class RMContainerRequestor extends RMCommunicator { private static final Log LOG = LogFactory.getLog(RMContainerRequestor.class); - static final String ANY = "*"; private int lastResponseID; private Resource availableResources; @@ -278,7 +277,7 @@ protected void addContainerReq(ContainerRequest req) { } // Off-switch - addResourceRequest(req.priority, ANY, req.capability); + addResourceRequest(req.priority, ResourceRequest.ANY, req.capability); } protected void decContainerReq(ContainerRequest req) { @@ -291,7 +290,7 @@ protected void decContainerReq(ContainerRequest req) { decResourceRequest(req.priority, rack, req.capability); } - decResourceRequest(req.priority, ANY, req.capability); + decResourceRequest(req.priority, ResourceRequest.ANY, req.capability); } private void addResourceRequest(Priority priority, String resourceName, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index e50d91c0cf8..efb8b7a134b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -230,7 +230,7 @@ public AllocateResponse allocate(AllocateRequest request) List askList = request.getAskList(); List containers = new ArrayList(); for (ResourceRequest req : askList) { - if (req.getHostName() != "*") { + if (!ResourceRequest.isAnyLocation(req.getHostName())) { continue; } int numContainers = req.getNumContainers(); diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 56528af93a3..7114465aea3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -100,6 +100,9 @@ Release 2.0.5-beta - UNRELEASED YARN-469. Make scheduling mode in FS pluggable. (kkambatl via tucu) + YARN-450. Define value for * in the scheduling protocol (Zhijie Shen via + bikas) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index c1b7541265d..0406a57525d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -49,6 +49,26 @@ @Public @Stable public abstract class ResourceRequest implements Comparable { + + /** + * The constant string representing no locality. + * It should be used by all references that want to pass an arbitrary host + * name in. + */ + public static final String ANY = "*"; + + /** + * Check whether the given host/rack string represents an arbitrary + * host name. + * + * @param hostName host/rack on which the allocation is desired + * @return whether the given host/rack string represents an arbitrary + * host name + */ + public static boolean isAnyLocation(String hostName) { + return ANY.equals(hostName); + } + /** * Get the Priority of the request. * @return Priority of the request diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java index 1fa86d2cc24..996d4668b54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java @@ -34,11 +34,6 @@ @InterfaceStability.Unstable public interface AMRMClient extends Service { - /** - * Value used to define no locality - */ - static final String ANY = "*"; - /** * Object to represent container request for resources. * Resources may be localized to nodes and racks. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java index 9f4358717bd..139a46c1988 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java @@ -258,7 +258,8 @@ public synchronized void addContainerRequest(ContainerRequest req) { } // Off-switch - addResourceRequest(req.priority, ANY, req.capability, req.containerCount); + addResourceRequest(req.priority, ResourceRequest.ANY, req.capability, + req.containerCount); } @Override @@ -276,7 +277,8 @@ public synchronized void removeContainerRequest(ContainerRequest req) { } } - decResourceRequest(req.priority, ANY, req.capability, req.containerCount); + decResourceRequest(req.priority, ResourceRequest.ANY, req.capability, + req.containerCount); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java index fd298ce96a7..a4d65db032c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java @@ -18,18 +18,15 @@ package org.apache.hadoop.yarn.client; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.util.List; import java.util.Set; import java.util.TreeSet; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; @@ -57,6 +54,11 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.service.Service.STATE; import org.apache.hadoop.yarn.util.Records; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; public class TestAMRMClient { Configuration conf = null; @@ -182,7 +184,7 @@ private void testAllocation(final AMRMClientImpl amClient) int containersRequestedRack = amClient.remoteRequestsTable.get(priority) .get(rack).get(capability).getNumContainers(); int containersRequestedAny = amClient.remoteRequestsTable.get(priority) - .get(AMRMClient.ANY).get(capability).getNumContainers(); + .get(ResourceRequest.ANY).get(capability).getNumContainers(); assertTrue(containersRequestedNode == 2); assertTrue(containersRequestedRack == 2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index bb3e7338491..c58613e70c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -38,9 +38,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -63,10 +63,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -737,7 +737,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, // Request a container for the AM. ResourceRequest request = BuilderUtils.newResourceRequest( - AM_CONTAINER_PRIORITY, "*", appAttempt.submissionContext + AM_CONTAINER_PRIORITY, ResourceRequest.ANY, appAttempt.submissionContext .getAMContainerSpec().getResource(), 1); Allocation amContainerAllocation = appAttempt.scheduler.allocate( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index 9d1103d4fc0..82986702601 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -36,8 +36,6 @@ */ public interface RMNode { - public static final String ANY = "*"; - /** * the node id of of this node. * @return the node id of this node. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 452cb9b6138..645d34bba3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -38,8 +38,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; /** * This class keeps track of all the consumption of an application. This also @@ -129,7 +127,7 @@ synchronized public void updateResourceRequests( boolean updatePendingResources = false; ResourceRequest lastRequest = null; - if (hostName.equals(RMNode.ANY)) { + if (hostName.equals(ResourceRequest.ANY)) { if (LOG.isDebugEnabled()) { LOG.debug("update:" + " application=" + applicationId + " request=" + request); @@ -195,7 +193,7 @@ synchronized public ResourceRequest getResourceRequest(Priority priority, } public synchronized Resource getResource(Priority priority) { - ResourceRequest request = getResourceRequest(priority, RMNode.ANY); + ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY); return request.getCapability(); } @@ -261,7 +259,7 @@ synchronized private void allocateNodeLocal( this.requests.get(priority).remove(node.getRackName()); } - decrementOutstanding(requests.get(priority).get(RMNode.ANY)); + decrementOutstanding(requests.get(priority).get(ResourceRequest.ANY)); } /** @@ -284,7 +282,7 @@ synchronized private void allocateRackLocal( this.requests.get(priority).remove(node.getRackName()); } - decrementOutstanding(requests.get(priority).get(RMNode.ANY)); + decrementOutstanding(requests.get(priority).get(ResourceRequest.ANY)); } /** @@ -322,7 +320,7 @@ synchronized private void decrementOutstanding( synchronized private void checkForDeactivation() { boolean deactivate = true; for (Priority priority : getPriorities()) { - ResourceRequest request = getResourceRequest(priority, RMNodeImpl.ANY); + ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY); if (request.getNumContainers() > 0) { deactivate = false; break; @@ -351,7 +349,7 @@ synchronized public void stop(RMAppAttemptState rmAppAttemptFinalState) { // clear pending resources metrics for the application QueueMetrics metrics = queue.getMetrics(); for (Map asks : requests.values()) { - ResourceRequest request = asks.get(RMNode.ANY); + ResourceRequest request = asks.get(ResourceRequest.ANY); if (request != null) { metrics.decrPendingResources(user, request.getNumContainers(), Resources.multiply(request.getCapability(), request diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index efc6e3efb25..d0e03a73ee4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -816,7 +815,8 @@ private synchronized FiCaSchedulerApp getApplication( for (Priority priority : application.getPriorities()) { // Required resource Resource required = - application.getResourceRequest(priority, RMNode.ANY).getCapability(); + application.getResourceRequest( + priority, ResourceRequest.ANY).getCapability(); // Do we need containers at this 'priority'? if (!needContainers(application, priority, required)) { @@ -1161,7 +1161,7 @@ private Resource assignOffSwitchContainers(Resource clusterResource, FiCaSchedul FiCaSchedulerApp application, Priority priority, RMContainer reservedContainer) { ResourceRequest request = - application.getResourceRequest(priority, RMNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { if (canAssign(application, priority, node, NodeType.OFF_SWITCH, reservedContainer)) { @@ -1183,7 +1183,7 @@ boolean canAssign(FiCaSchedulerApp application, Priority priority, // 'Delay' off-switch ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, RMNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); long missedOpportunities = application.getSchedulingOpportunities(priority); long requiredContainers = offSwitchRequest.getNumContainers(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 317f2a211bd..1ff6e024f6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; @@ -153,7 +152,7 @@ public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) } public synchronized int getTotalRequiredResources(Priority priority) { - return getResourceRequest(priority, RMNode.ANY).getNumContainers(); + return getResourceRequest(priority, ResourceRequest.ANY).getNumContainers(); } public Resource getResource(Priority priority) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java index b84607560ec..f6bac00218d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java @@ -60,8 +60,6 @@ public class FiCaSchedulerNode extends SchedulerNode { private final RMNode rmNode; - public static final String ANY = "*"; - public FiCaSchedulerNode(RMNode node) { this.rmNode = node; this.availableResource.setMemory(node.getTotalCapability().getMemory()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index 46a6fe9bb7a..32328be5a68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -336,7 +335,7 @@ public Resource assignContainer(FSSchedulerNode node, boolean reserved) { } ResourceRequest offSwitchRequest = app.getResourceRequest(priority, - RMNode.ANY); + ResourceRequest.ANY); if (offSwitchRequest != null && offSwitchRequest.getNumContainers() != 0 && allowedLocality.equals(NodeType.OFF_SWITCH)) { return assignContainer(node, app, priority, offSwitchRequest, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 49a52bbb8dc..0734a4aead5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -40,8 +40,8 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -50,7 +50,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; @@ -153,7 +152,7 @@ public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) } public synchronized int getTotalRequiredResources(Priority priority) { - return getResourceRequest(priority, RMNode.ANY).getNumContainers(); + return getResourceRequest(priority, ResourceRequest.ANY).getNumContainers(); } public Resource getResource(Priority priority) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java index 2833ca67b95..efc1c9f6842 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java @@ -59,8 +59,6 @@ public class FSSchedulerNode extends SchedulerNode { private final RMNode rmNode; - public static final String ANY = "*"; - public FSSchedulerNode(RMNode node) { this.rmNode = node; this.availableResource.setMemory(node.getTotalCapability().getMemory()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 3f255378231..df143566a7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -57,8 +57,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -393,7 +393,7 @@ private void assignContainers(FiCaSchedulerNode node) { private int getMaxAllocatableContainers(FiCaSchedulerApp application, Priority priority, FiCaSchedulerNode node, NodeType type) { ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); int maxContainers = offSwitchRequest.getNumContainers(); if (type == NodeType.OFF_SWITCH) { @@ -483,7 +483,7 @@ private int assignRackLocalContainers(FiCaSchedulerNode node, if (request != null) { // Don't allocate on this rack if the application doens't need containers ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (offSwitchRequest.getNumContainers() <= 0) { return 0; } @@ -504,7 +504,7 @@ private int assignOffSwitchContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) { int assignedContainers = 0; ResourceRequest request = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { assignedContainers = assignContainer(node, application, priority, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index ee6cc99f0bf..20ffeabfeba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -43,11 +43,9 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.Task.State; @@ -200,9 +198,7 @@ public synchronized void addTask(Task task) { } // Off-switch - addResourceRequest(priority, requests, - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY, - capability); + addResourceRequest(priority, requests, ResourceRequest.ANY, capability); } public synchronized void finishTask(Task task) throws IOException { @@ -377,10 +373,7 @@ private void updateResourceRequests(Map requests, } } - updateResourceRequest( - requests.get( - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY) - ); + updateResourceRequest(requests.get(ResourceRequest.ANY)); if(LOG.isDebugEnabled()) { LOG.debug("updateResourceRequests:" + " application=" + applicationId diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index 9eb5bfb9b6a..1ed4652a96e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -120,8 +120,8 @@ public List createReq(String[] hosts, int memory, int priority, reqs.add(rackReq); } - ResourceRequest offRackReq = createResourceReq("*", memory, priority, - containers); + ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, + priority, containers); reqs.add(offRackReq); return reqs; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java index 0fac0cd02ab..eebfa1da9e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; public class Task { @@ -57,8 +58,7 @@ public Task(Application application, Priority priority, String[] hosts) { // Special case: Don't care about locality if (!(hosts.length == 1 && - hosts[0].equals( - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY))) { + hosts[0].equals(ResourceRequest.ANY))) { for (String host : hosts) { this.hosts.add(host); this.racks.add(Application.resolve(host)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 8965f40f1b1..4780ec38821 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -264,14 +264,14 @@ public void testHeadroom() throws Exception { // Ask for a 1 GB container for app 1 List ask1 = new ArrayList(); - ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(GB, 1), 1)); + ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1)); fs.allocate(appAttemptId1, ask1, emptyId); // Ask for a 2 GB container for app 2 List ask2 = new ArrayList(); - ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(2 * GB, 1), 1)); + ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1)); fs.allocate(appAttemptId2, ask2, emptyId); // Trigger container assignment diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index 1596bfddd6e..4b2323d241d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -124,7 +125,7 @@ public void testResourceAllocation() throws IOException { Task t2 = new Task(application, priority1, new String[] {host1, host2}); application.addTask(t2); - Task t3 = new Task(application, priority0, new String[] {RMNode.ANY}); + Task t3 = new Task(application, priority0, new String[] {ResourceRequest.ANY}); application.addTask(t3); // Send resource requests to the scheduler diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 28894358fd2..5953f843df3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -47,10 +47,9 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.junit.After; @@ -506,7 +505,7 @@ public void testHeadroom() throws Exception { List app_0_0_requests = new ArrayList(); app_0_0_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_0_0.updateResourceRequests(app_0_0_requests); @@ -525,7 +524,7 @@ public void testHeadroom() throws Exception { List app_0_1_requests = new ArrayList(); app_0_1_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_0_1.updateResourceRequests(app_0_1_requests); @@ -544,7 +543,7 @@ public void testHeadroom() throws Exception { List app_1_0_requests = new ArrayList(); app_1_0_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_1_0.updateResourceRequests(app_1_0_requests); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index d3a7b0c35f7..48717ed8d02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.Application; @@ -183,7 +184,7 @@ public void testCapacityScheduler() throws Exception { LOG.info("Adding new tasks..."); Task task_1_1 = new Task(application_1, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_1); application_1.schedule(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 828c796d011..65b9d0cc1f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -24,8 +24,8 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; @@ -55,13 +55,12 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -287,7 +286,7 @@ public void testSingleQueueOneUserMetrics() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, priority, recordFactory))); // Start testing... @@ -409,11 +408,11 @@ public void testSingleQueueWithOneUser() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); // Start testing... @@ -542,11 +541,11 @@ public void testUserLimits() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); /** @@ -635,11 +634,11 @@ public void testHeadroomWithMaxCap() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); /** @@ -674,7 +673,7 @@ public void testHeadroomWithMaxCap() throws Exception { // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f); app_2.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, priority, recordFactory))); assertEquals(2, a.getActiveUsersManager().getNumActiveUsers()); @@ -691,7 +690,7 @@ public void testHeadroomWithMaxCap() throws Exception { // Check headroom for app_2 LOG.info("here"); app_1.updateResourceRequests(Collections.singletonList( // unset - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 0, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, priority, recordFactory))); assertEquals(1, a.getActiveUsersManager().getNumActiveUsers()); a.assignContainers(clusterResource, node_1); @@ -752,11 +751,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 10, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 10, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, priority, recordFactory))); /** @@ -786,11 +785,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception { // Submit resource requests for other apps now to 'activate' them app_2.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 3*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, priority, recordFactory))); app_3.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); // Now allocations should goto app_2 since @@ -914,11 +913,11 @@ public void testReservation() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, priority, recordFactory))); // Start testing... @@ -1016,7 +1015,7 @@ public void testStolenReservedContainer() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); // Setup app_1 to request a 4GB container on host_0 and @@ -1027,7 +1026,7 @@ public void testStolenReservedContainer() throws Exception { priority, recordFactory)); appRequests_1.add(TestUtils.createResourceRequest(DEFAULT_RACK, 4*GB, 1, priority, recordFactory)); - appRequests_1.add(TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 2, + appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2, priority, recordFactory)); app_1.updateResourceRequests(appRequests_1); @@ -1122,11 +1121,11 @@ public void testReservationExchange() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, priority, recordFactory))); // Start testing... @@ -1248,7 +1247,7 @@ public void testLocalityScheduling() throws Exception { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, // one extra + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1313,7 +1312,7 @@ public void testLocalityScheduling() throws Exception { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, // one extra + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); assertEquals(2, app_0.getTotalRequiredResources(priority)); @@ -1391,7 +1390,7 @@ public void testApplicationPriorityScheduling() throws Exception { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority_1, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); // P2 @@ -1403,7 +1402,7 @@ public void testApplicationPriorityScheduling() throws Exception { TestUtils.createResourceRequest(rack_2, 2*GB, 1, priority_2, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority_2, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1528,7 +1527,7 @@ public void testSchedulingConstraints() throws Exception { // Add one request app_0_requests_0.clear(); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // only one + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1551,7 +1550,7 @@ public void testSchedulingConstraints() throws Exception { // Add one request app_0_requests_0.clear(); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // only one + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 709005c417d..caab5ed0725 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -142,7 +141,8 @@ private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { } - private ResourceRequest createResourceRequest(int memory, String host, int priority, int numContainers) { + private ResourceRequest createResourceRequest(int memory, String host, + int priority, int numContainers) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); request.setCapability(Resources.createResource(memory)); request.setHostName(host); @@ -157,27 +157,33 @@ private ResourceRequest createResourceRequest(int memory, String host, int prior * Creates a single container priority-1 request and submits to * scheduler. */ - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId) { return createSchedulingRequest(memory, queueId, userId, 1); } - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId, int numContainers) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers) { return createSchedulingRequest(memory, queueId, userId, numContainers, 1); } - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId, int numContainers, int priority) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); scheduler.addApplication(id, queueId, userId); List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, "*", priority, numContainers); + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, numContainers); ask.add(request); scheduler.allocate(id, ask, new ArrayList()); return id; } - private void createSchedulingRequestExistingApplication(int memory, int priority, ApplicationAttemptId attId) { + private void createSchedulingRequestExistingApplication(int memory, int priority, + ApplicationAttemptId attId) { List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, "*", priority, 1); + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, 1); ask.add(request); scheduler.allocate(attId, ask, new ArrayList()); } @@ -459,7 +465,8 @@ public void testQueueDemandCalculation() throws Exception { // First ask, queue1 requests 1 large (minReqSize * 2). List ask1 = new ArrayList(); - ResourceRequest request1 = createResourceRequest(minReqSize * 2, "*", 1, 1); + ResourceRequest request1 = + createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1); ask1.add(request1); scheduler.allocate(id11, ask1, new ArrayList()); @@ -473,7 +480,8 @@ public void testQueueDemandCalculation() throws Exception { // Third ask, queue2 requests 1 large List ask3 = new ArrayList(); - ResourceRequest request4 = createResourceRequest(2 * minReqSize, "*", 1, 1); + ResourceRequest request4 = + createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1); ask3.add(request4); scheduler.allocate(id22, ask3, new ArrayList()); @@ -1311,7 +1319,7 @@ public void testMultipleNodesSingleRackRequest() throws Exception { asks.add(createResourceRequest(1024, node3.getHostName(), 1, 1)); asks.add(createResourceRequest(1024, node1.getRackName(), 1, 1)); asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1)); - asks.add(createResourceRequest(1024, RMNode.ANY, 1, 2)); + asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2)); scheduler.allocate(appId, asks, new ArrayList()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index c4dbe876c78..8f45f4535cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -40,7 +41,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -200,15 +200,15 @@ public void testFifoScheduler() throws Exception { LOG.info("Adding new tasks..."); Task task_1_1 = new Task(application_1, priority_1, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_1); Task task_1_2 = new Task(application_1, priority_1, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_2); Task task_1_3 = new Task(application_1, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_3); application_1.schedule(); @@ -222,7 +222,7 @@ public void testFifoScheduler() throws Exception { application_0.addTask(task_0_2); Task task_0_3 = new Task(application_0, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_0.addTask(task_0_3); application_0.schedule(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 2f810fdb483..27986cc1a96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FileWriter; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -70,11 +69,8 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -89,7 +85,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -486,8 +481,8 @@ private Container requestAndGetContainer(AMRMProtocol scheduler, // Request a container allocation. List ask = new ArrayList(); - ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(1024, 1), 1)); + ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(1024, 1), 1)); AllocateRequest allocateRequest = BuilderUtils.newAllocateRequest( BuilderUtils.newApplicationAttemptId(appID, 1), 0, 0F, ask,