From f27e484636f9903bb80afac9c9e51a04a7dbce76 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Sun, 2 Jun 2013 22:53:58 +0000 Subject: [PATCH] YARN-749. Rename ResourceRequest.(get,set)HostName to ResourceRequest.(get,set)ResourceName. Contributed by Arun C. Murthy. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1488806 13f79535-47bb-0310-9956-ffa450edef68 --- .../v2/app/rm/RMContainerRequestor.java | 4 +-- .../mapreduce/v2/app/MRAppBenchmark.java | 2 +- .../v2/app/TestRMContainerAllocator.java | 2 +- hadoop-yarn-project/CHANGES.txt | 3 ++ .../yarn/api/records/ResourceRequest.java | 36 ++++++++++--------- .../impl/pb/ResourceRequestPBImpl.java | 16 ++++----- .../src/main/proto/yarn_protos.proto | 2 +- .../apache/hadoop/yarn/util/BuilderUtils.java | 8 ++--- .../scheduler/AppSchedulingInfo.java | 12 +++---- .../common/fica/FiCaSchedulerApp.java | 4 +-- .../yarn/server/resourcemanager/MockAM.java | 2 +- .../scheduler/capacity/TestUtils.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 2 +- .../scheduler/fifo/TestFifoScheduler.java | 2 +- 14 files changed, 52 insertions(+), 45 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index 164789c2749..d99a9ddd608 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -241,7 +241,7 @@ public abstract class RMContainerRequestor extends RMCommunicator { // specified for the blacklisted host to be 0. ResourceRequest zeroedRequest = ResourceRequest.newInstance(req.getPriority(), - req.getHostName(), req.getCapability(), + req.getResourceName(), req.getCapability(), req.getNumContainers()); zeroedRequest.setNumContainers(0); @@ -323,7 +323,7 @@ public abstract class RMContainerRequestor extends RMCommunicator { if (remoteRequest == null) { remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class); remoteRequest.setPriority(priority); - remoteRequest.setHostName(resourceName); + remoteRequest.setResourceName(resourceName); remoteRequest.setCapability(capability); remoteRequest.setNumContainers(0); reqMap.put(capability, remoteRequest); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index d52d8750391..a6dc5667e3f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -227,7 +227,7 @@ public class MRAppBenchmark { List askList = request.getAskList(); List containers = new ArrayList(); for (ResourceRequest req : askList) { - if (!ResourceRequest.isAnyLocation(req.getHostName())) { + if (!ResourceRequest.isAnyLocation(req.getResourceName())) { continue; } int numContainers = req.getNumContainers(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 67b24e37c73..41e88a87bf8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -1219,7 +1219,7 @@ public class TestRMContainerAllocator { List askCopy = new ArrayList(); for (ResourceRequest req : ask) { ResourceRequest reqCopy = ResourceRequest.newInstance(req - .getPriority(), req.getHostName(), req.getCapability(), req + .getPriority(), req.getResourceName(), req.getCapability(), req .getNumContainers()); askCopy.add(reqCopy); } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ddbaa4e062d..8da28a4e397 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -81,6 +81,9 @@ Release 2.1.0-beta - UNRELEASED YARN-735. Make ApplicationAttemptId, ContaienrId and NodeId immutable. (Jian He via sseth) + YARN-749. Rename ResourceRequest.(get,set)HostName to + ResourceRequest.(get,set)ResourceName. (acmurthy) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index 4085ca1fdc7..32fdce744cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -57,7 +57,7 @@ public abstract class ResourceRequest implements Comparable { String hostName, Resource capability, int numContainers) { ResourceRequest request = Records.newRecord(ResourceRequest.class); request.setPriority(priority); - request.setHostName(hostName); + request.setResourceName(hostName); request.setCapability(capability); request.setNumContainers(numContainers); return request; @@ -99,28 +99,32 @@ public abstract class ResourceRequest implements Comparable { public abstract void setPriority(Priority priority); /** - * Get the host/rack on which the allocation is desired. + * Get the resource (e.g. host/rack) on which the allocation + * is desired. * - * A special value of * signifies that any host/rack is - * acceptable. + * A special value of * signifies that any resource + * (host/rack) is acceptable. * - * @return host/rack on which the allocation is desired + * @return resource (e.g. host/rack) on which the allocation + * is desired */ @Public @Stable - public abstract String getHostName(); + public abstract String getResourceName(); /** - * Set host/rack on which the allocation is desired. + * Set the resource (e.g. host/rack) on which the allocation + * is desired. * - * A special value of * signifies that any host/rack is - * acceptable. + * A special value of * signifies that any resource + * (e.g. host/rack) is acceptable. * - * @param hostName host/rack on which the allocation is desired + * @param resourceName (e.g. host/rack) on which the + * allocation is desired */ @Public @Stable - public abstract void setHostName(String hostName); + public abstract void setResourceName(String resourceName); /** * Get the Resource capability of the request. @@ -198,7 +202,7 @@ public abstract class ResourceRequest implements Comparable { final int prime = 2153; int result = 2459; Resource capability = getCapability(); - String hostName = getHostName(); + String hostName = getResourceName(); Priority priority = getPriority(); result = prime * result + ((capability == null) ? 0 : capability.hashCode()); @@ -223,11 +227,11 @@ public abstract class ResourceRequest implements Comparable { return false; } else if (!capability.equals(other.getCapability())) return false; - String hostName = getHostName(); + String hostName = getResourceName(); if (hostName == null) { - if (other.getHostName() != null) + if (other.getResourceName() != null) return false; - } else if (!hostName.equals(other.getHostName())) + } else if (!hostName.equals(other.getResourceName())) return false; if (getNumContainers() != other.getNumContainers()) return false; @@ -245,7 +249,7 @@ public abstract class ResourceRequest implements Comparable { int priorityComparison = this.getPriority().compareTo(other.getPriority()); if (priorityComparison == 0) { int hostNameComparison = - this.getHostName().compareTo(other.getHostName()); + this.getResourceName().compareTo(other.getResourceName()); if (hostNameComparison == 0) { int capabilityComparison = this.getCapability().compareTo(other.getCapability()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index 3726daaf2a4..2b85acb6775 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -98,22 +98,22 @@ public class ResourceRequestPBImpl extends ResourceRequest { this.priority = priority; } @Override - public String getHostName() { + public String getResourceName() { ResourceRequestProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasHostName()) { + if (!p.hasResourceName()) { return null; } - return (p.getHostName()); + return (p.getResourceName()); } @Override - public void setHostName(String hostName) { + public void setResourceName(String resourceName) { maybeInitBuilder(); - if (hostName == null) { - builder.clearHostName(); + if (resourceName == null) { + builder.clearResourceName(); return; } - builder.setHostName((hostName)); + builder.setResourceName((resourceName)); } @Override public Resource getCapability() { @@ -179,7 +179,7 @@ public class ResourceRequestPBImpl extends ResourceRequest { public String toString() { return "{Priority: " + getPriority() + ", Capability: " + getCapability() + ", # Containers: " + getNumContainers() - + ", Location: " + getHostName() + + ", Location: " + getResourceName() + ", Relax Locality: " + getRelaxLocality() + "}"; } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 84eccfb3534..3bf3b2e84f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -201,7 +201,7 @@ message NodeReportProto { //////////////////////////////////////////////////////////////////////// message ResourceRequestProto { optional PriorityProto priority = 1; - optional string host_name = 2; + optional string resource_name = 2; optional ResourceProto capability = 3; optional int32 num_containers = 4; optional bool relax_locality = 5 [default = true]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index 4793288cdcf..576ca2457f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -97,8 +97,8 @@ public class BuilderUtils { // Compare priority, host and capability int ret = r1.getPriority().compareTo(r2.getPriority()); if (ret == 0) { - String h1 = r1.getHostName(); - String h2 = r2.getHostName(); + String h1 = r1.getResourceName(); + String h2 = r2.getResourceName(); ret = h1.compareTo(h2); } if (ret == 0) { @@ -296,7 +296,7 @@ public class BuilderUtils { ResourceRequest request = recordFactory .newRecordInstance(ResourceRequest.class); request.setPriority(priority); - request.setHostName(hostName); + request.setResourceName(hostName); request.setCapability(capability); request.setNumContainers(numContainers); return request; @@ -306,7 +306,7 @@ public class BuilderUtils { ResourceRequest request = recordFactory .newRecordInstance(ResourceRequest.class); request.setPriority(r.getPriority()); - request.setHostName(r.getHostName()); + request.setResourceName(r.getResourceName()); request.setCapability(r.getCapability()); request.setNumContainers(r.getNumContainers()); return request; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 645d34bba3d..6962a2c7316 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -123,11 +123,11 @@ public class AppSchedulingInfo { // Update resource requests for (ResourceRequest request : requests) { Priority priority = request.getPriority(); - String hostName = request.getHostName(); + String resourceName = request.getResourceName(); boolean updatePendingResources = false; ResourceRequest lastRequest = null; - if (hostName.equals(ResourceRequest.ANY)) { + if (resourceName.equals(ResourceRequest.ANY)) { if (LOG.isDebugEnabled()) { LOG.debug("update:" + " application=" + applicationId + " request=" + request); @@ -152,10 +152,10 @@ public class AppSchedulingInfo { this.requests.put(priority, asks); this.priorities.add(priority); } else if (updatePendingResources) { - lastRequest = asks.get(hostName); + lastRequest = asks.get(resourceName); } - asks.put(hostName, request); + asks.put(resourceName, request); if (updatePendingResources) { // Similarly, deactivate application? @@ -187,9 +187,9 @@ public class AppSchedulingInfo { } synchronized public ResourceRequest getResourceRequest(Priority priority, - String nodeAddress) { + String resourceName) { Map nodeRequests = requests.get(priority); - return (nodeRequests == null) ? null : nodeRequests.get(nodeAddress); + return (nodeRequests == null) ? null : nodeRequests.get(resourceName); } public synchronized Resource getResource(Priority priority) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index f7a8f7cb1e1..57e0634e296 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -152,8 +152,8 @@ public class FiCaSchedulerApp extends SchedulerApplication { return this.appSchedulingInfo.getPriorities(); } - public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) { - return this.appSchedulingInfo.getResourceRequest(priority, nodeAddress); + public ResourceRequest getResourceRequest(Priority priority, String resourceName) { + return this.appSchedulingInfo.getResourceRequest(priority, resourceName); } public synchronized int getTotalRequiredResources(Priority priority) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index 1ed4652a96e..59cabb3ad22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -130,7 +130,7 @@ public class MockAM { public ResourceRequest createResourceReq(String resource, int memory, int priority, int containers) throws Exception { ResourceRequest req = Records.newRecord(ResourceRequest.class); - req.setHostName(resource); + req.setResourceName(resource); req.setNumContainers(containers); Priority pri = Records.newRecord(Priority.class); pri.setPriority(priority); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 113e16ed7f4..56dda8a0d21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -122,7 +122,7 @@ public class TestUtils { Resource capability = Resources.createResource(memory, 1); request.setNumContainers(numContainers); - request.setHostName(hostName); + request.setResourceName(hostName); request.setCapability(capability); request.setPriority(priority); return request; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 6a75197a947..b0c0a42fe55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -153,7 +153,7 @@ public class TestFairScheduler { int priority, int numContainers, boolean relaxLocality) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); request.setCapability(Resources.createResource(memory)); - request.setHostName(host); + request.setResourceName(host); request.setNumContainers(numContainers); Priority prio = recordFactory.newRecordInstance(Priority.class); prio.setPriority(priority); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 886c0733719..d252421c6d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -106,7 +106,7 @@ public class TestFifoScheduler { ResourceRequest request = recordFactory .newRecordInstance(ResourceRequest.class); request.setCapability(Resources.createResource(memory)); - request.setHostName(host); + request.setResourceName(host); request.setNumContainers(numContainers); Priority prio = recordFactory.newRecordInstance(Priority.class); prio.setPriority(priority);