From c57e6bc385a0a8efef1767056aadc992b4af2cee Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Tue, 1 Oct 2019 16:13:53 +0530 Subject: [PATCH] YARN-9858. Optimize RMContext getExclusiveEnforcedPartitions. Contributed by Jonathan Hung. --- .../hadoop/yarn/conf/YarnConfiguration.java | 16 ++++++++++++++++ .../resourcemanager/DefaultAMSProcessor.java | 6 ++++-- .../server/resourcemanager/RMAppManager.java | 3 ++- .../server/resourcemanager/RMContext.java | 4 ---- .../server/resourcemanager/RMContextImpl.java | 19 ------------------- .../resourcemanager/TestAppManager.java | 1 + 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 7139818f779..b8aa1ab839f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -23,7 +23,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -3213,6 +3215,20 @@ public class YarnConfiguration extends Configuration { public static final String EXCLUSIVE_ENFORCED_PARTITIONS = NODE_LABELS_PREFIX + EXCLUSIVE_ENFORCED_PARTITIONS_SUFFIX; + @Private + public static Set getExclusiveEnforcedPartitions( + Configuration conf) { + Set exclusiveEnforcedPartitions = new HashSet<>(); + String[] configuredPartitions = conf.getStrings( + EXCLUSIVE_ENFORCED_PARTITIONS); + if (configuredPartitions != null) { + for (String partition : configuredPartitions) { + exclusiveEnforcedPartitions.add(partition); + } + } + return exclusiveEnforcedPartitions; + } + public static final String MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY = YARN_PREFIX + "cluster.max-application-priority"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 65bbaca19cc..a706f8671c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -101,11 +101,14 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { RecordFactoryProvider.getRecordFactory(null); private RMContext rmContext; + private Set exclusiveEnforcedPartitions; @Override public void init(ApplicationMasterServiceContext amsContext, ApplicationMasterServiceProcessor nextProcessor) { this.rmContext = (RMContext)amsContext; + this.exclusiveEnforcedPartitions = YarnConfiguration + .getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration()); } @Override @@ -209,8 +212,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { } if (ResourceRequest.ANY.equals(req.getResourceName())) { SchedulerUtils.enforcePartitionExclusivity(req, - getRmContext().getExclusiveEnforcedPartitions(), - asc.getNodeLabelExpression()); + exclusiveEnforcedPartitions, asc.getNodeLabelExpression()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 5b50623c9bb..4be5e2ab6d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -112,7 +112,8 @@ public class RMAppManager implements EventHandler, this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } this.authorizer = YarnAuthorizationProvider.getInstance(conf); - this.exclusiveEnforcedPartitions = context.getExclusiveEnforcedPartitions(); + this.exclusiveEnforcedPartitions = YarnConfiguration + .getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration()); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index b5cc5a7ad77..b255a304eae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.nio.ByteBuffer; -import java.util.Set; import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; @@ -162,7 +161,4 @@ public interface RMContext extends ApplicationMasterServiceContext { ResourceManager getResourceManager(); String getAppProxyUrl(Configuration conf, ApplicationId applicationId); - - Set getExclusiveEnforcedPartitions(); - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 6717d1b43cb..d7c624d4863 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.ConcurrentMap; import org.apache.commons.logging.Log; @@ -35,7 +33,6 @@ import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.ConfigurationProvider; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; @@ -585,20 +582,4 @@ public class RMContextImpl implements RMContext { return UNAVAILABLE; } } - - public Set getExclusiveEnforcedPartitions() { - Set exclusiveEnforcedPartitions = new HashSet<>(); - Configuration conf = getYarnConfiguration(); - if (conf == null) { - return new HashSet<>(); - } - String[] configuredPartitions = conf.getStrings( - YarnConfiguration.EXCLUSIVE_ENFORCED_PARTITIONS); - if (configuredPartitions != null) { - for (String partition : configuredPartitions) { - exclusiveEnforcedPartitions.add(partition); - } - } - return exclusiveEnforcedPartitions; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index e7bb2d30a9f..1a04e6c56b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -154,6 +154,7 @@ public class TestAppManager{ metricsPublisher = mock(SystemMetricsPublisher.class); context.setSystemMetricsPublisher(metricsPublisher); context.setRMApplicationHistoryWriter(writer); + ((RMContextImpl) context).setYarnConfiguration(new YarnConfiguration()); return context; }