From bb8350388bc8933183d657f5cf3b78afefb94f68 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Thu, 28 May 2015 17:04:20 -0700 Subject: [PATCH] YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue info once. (Varun Saxena via wangda) --- hadoop-yarn-project/CHANGES.txt | 5 +-- .../server/resourcemanager/RMServerUtils.java | 10 +++++- .../scheduler/SchedulerUtils.java | 34 ++++++++++++++----- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 409178e9f51..e1af21d9333 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -17,6 +17,9 @@ Release 2.7.1 - UNRELEASED YARN-3723. Need to clearly document primaryFilter and otherInfo value type. (Zhijie Shen via xgong) + YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue + info once. (Varun Saxena via wangda) + OPTIMIZATIONS YARN-3006. Improve the error message when attempting manual failover with @@ -86,7 +89,6 @@ Release 2.7.1 - UNRELEASED YARN-3493. RM fails to come up with error "Failed to load/recover state" when mem settings are changed. (Jian He via wangda) -<<<<<<< HEAD YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. (Craig Welch via xgong) @@ -1131,7 +1133,6 @@ Release 2.6.0 - 2014-11-18 YARN-2406. Move RM recovery related proto to yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe) -<<<<<<< HEAD YARN-1506. Changed RMNode/SchedulerNode to update resource with event notification. (Junping Du via jianhe) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 4669a28e38d..fd17153d920 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -94,9 +95,16 @@ public class RMServerUtils { Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException { + + QueueInfo queueInfo = null; + try { + queueInfo = scheduler.getQueueInfo(queueName, false, false); + } catch (IOException e) { + } + for (ResourceRequest resReq : ask) { SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, - queueName, scheduler, rmContext); + queueName, scheduler, rmContext, queueInfo); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 45730be879c..071f2938108 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -212,28 +212,46 @@ public class SchedulerUtils { public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, - boolean isRecovery, RMContext rmContext) + boolean isRecovery, RMContext rmContext, QueueInfo queueInfo) throws InvalidResourceRequestException { - QueueInfo queueInfo = null; - try { - queueInfo = scheduler.getQueueInfo(queueName, false, false); - } catch (IOException e) { - // it is possible queue cannot get when queue mapping is set, just ignore - // the queueInfo here, and move forward + if (queueInfo == null) { + try { + queueInfo = scheduler.getQueueInfo(queueName, false, false); + } catch (IOException e) { + // it is possible queue cannot get when queue mapping is set, just ignore + // the queueInfo here, and move forward + } } SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo); if (!isRecovery) { validateResourceRequest(resReq, maximumResource, queueInfo, rmContext); } } + + public static void normalizeAndValidateRequest(ResourceRequest resReq, + Resource maximumResource, String queueName, YarnScheduler scheduler, + boolean isRecovery, RMContext rmContext) + throws InvalidResourceRequestException { + normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, + isRecovery, rmContext, null); + } + public static void normalizeAndvalidateRequest(ResourceRequest resReq, + Resource maximumResource, String queueName, YarnScheduler scheduler, + RMContext rmContext, QueueInfo queueInfo) + throws InvalidResourceRequestException { + normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, + false, rmContext, queueInfo); + } + + public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException { normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, - false, rmContext); + false, rmContext, null); } /**