From 425a6c89aedb4d5ad9a524c9f1e631eea2816f9c Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Tue, 1 Oct 2019 16:02:26 +0530 Subject: [PATCH] YARN-9858. Optimize RMContext getExclusiveEnforcedPartitions. Contributed by Jonathan Hung. --- .../hadoop/yarn/conf/YarnConfiguration.java | 16 ++++++++++++++++ .../resourcemanager/DefaultAMSProcessor.java | 6 ++++-- .../server/resourcemanager/RMAppManager.java | 3 ++- .../server/resourcemanager/RMContext.java | 4 ---- .../server/resourcemanager/RMContextImpl.java | 19 ------------------- 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 1314bf9e67b..83871a5bf52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -23,7 +23,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -3795,6 +3797,20 @@ public class YarnConfiguration extends Configuration { public static final String EXCLUSIVE_ENFORCED_PARTITIONS = NODE_LABELS_PREFIX + EXCLUSIVE_ENFORCED_PARTITIONS_SUFFIX; + @Private + public static Set getExclusiveEnforcedPartitions( + Configuration conf) { + Set exclusiveEnforcedPartitions = new HashSet<>(); + String[] configuredPartitions = conf.getStrings( + EXCLUSIVE_ENFORCED_PARTITIONS); + if (configuredPartitions != null) { + for (String partition : configuredPartitions) { + exclusiveEnforcedPartitions.add(partition); + } + } + return exclusiveEnforcedPartitions; + } + public static final String MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY = YARN_PREFIX + "cluster.max-application-priority"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 4d5cb13cf6e..2f28ac75f5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -119,6 +119,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { private ResourceProfilesManager resourceProfilesManager; private boolean timelineServiceV2Enabled; private boolean nodelabelsEnabled; + private Set exclusiveEnforcedPartitions; @Override public void init(ApplicationMasterServiceContext amsContext, @@ -129,6 +130,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { timelineServiceV2Enabled(rmContext.getYarnConfiguration()); this.nodelabelsEnabled = YarnConfiguration .areNodeLabelsEnabled(rmContext.getYarnConfiguration()); + this.exclusiveEnforcedPartitions = YarnConfiguration + .getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration()); } @Override @@ -239,8 +242,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { } if (ResourceRequest.ANY.equals(req.getResourceName())) { SchedulerUtils.enforcePartitionExclusivity(req, - getRmContext().getExclusiveEnforcedPartitions(), - asc.getNodeLabelExpression()); + exclusiveEnforcedPartitions, asc.getNodeLabelExpression()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index cd070a05f50..f4f97936c55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -128,7 +128,8 @@ public class RMAppManager implements EventHandler, timelineServiceV2Enabled(conf); this.nodeLabelsEnabled = YarnConfiguration .areNodeLabelsEnabled(rmContext.getYarnConfiguration()); - this.exclusiveEnforcedPartitions = context.getExclusiveEnforcedPartitions(); + this.exclusiveEnforcedPartitions = YarnConfiguration + .getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration()); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index d4949566cf2..55420bd9270 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; -import java.util.Set; import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; @@ -204,7 +203,4 @@ public interface RMContext extends ApplicationMasterServiceContext { long getTokenSequenceNo(); void incrTokenSequenceNo(); - - Set getExclusiveEnforcedPartitions(); - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index a10f44898b0..54e0281f7e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -20,8 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.net.URI; import java.net.URISyntaxException; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.ConcurrentMap; import org.slf4j.Logger; @@ -34,7 +32,6 @@ import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.ConfigurationProvider; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.SystemCredentialsForAppsProto; @@ -681,20 +678,4 @@ public class RMContextImpl implements RMContext { public void incrTokenSequenceNo() { this.activeServiceContext.incrTokenSequenceNo(); } - - public Set getExclusiveEnforcedPartitions() { - Set exclusiveEnforcedPartitions = new HashSet<>(); - Configuration conf = getYarnConfiguration(); - if (conf == null) { - return new HashSet<>(); - } - String[] configuredPartitions = conf.getStrings( - YarnConfiguration.EXCLUSIVE_ENFORCED_PARTITIONS); - if (configuredPartitions != null) { - for (String partition : configuredPartitions) { - exclusiveEnforcedPartitions.add(partition); - } - } - return exclusiveEnforcedPartitions; - } }