Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9858. Optimize RMContext getExclusiveEnforcedPartitions. Contributed by Jonathan Hung.

This commit is contained in:
bibinchundatt 2019-10-01 16:06:38 +05:30
parent 5e56914650
commit 6529a30d9e
6 changed files with 23 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -23,7 +23,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -3487,6 +3489,20 @@ public class YarnConfiguration extends Configuration {
public static final String EXCLUSIVE_ENFORCED_PARTITIONS = NODE_LABELS_PREFIX
+ EXCLUSIVE_ENFORCED_PARTITIONS_SUFFIX;
@Private
public static Set<String> getExclusiveEnforcedPartitions(
Configuration conf) {
Set<String> exclusiveEnforcedPartitions = new HashSet<>();
String[] configuredPartitions = conf.getStrings(
EXCLUSIVE_ENFORCED_PARTITIONS);
if (configuredPartitions != null) {
for (String partition : configuredPartitions) {
exclusiveEnforcedPartitions.add(partition);
}
}
return exclusiveEnforcedPartitions;
}
public static final String MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY =
YARN_PREFIX + "cluster.max-application-priority";

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
View File

@ -109,6 +109,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
private RMContext rmContext;
private ResourceProfilesManager resourceProfilesManager;
private boolean timelineServiceV2Enabled;
private Set<String> exclusiveEnforcedPartitions;
@Override
public void init(ApplicationMasterServiceContext amsContext,
@ -117,6 +118,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
this.resourceProfilesManager = rmContext.getResourceProfilesManager();
this.timelineServiceV2Enabled = YarnConfiguration.
timelineServiceV2Enabled(rmContext.getYarnConfiguration());
this.exclusiveEnforcedPartitions = YarnConfiguration
.getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration());
}
@Override
@ -227,8 +230,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
}
if (ResourceRequest.ANY.equals(req.getResourceName())) {
SchedulerUtils.enforcePartitionExclusivity(req,
getRmContext().getExclusiveEnforcedPartitions(),
asc.getNodeLabelExpression());
exclusiveEnforcedPartitions, asc.getNodeLabelExpression());
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -121,7 +121,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
this.authorizer = YarnAuthorizationProvider.getInstance(conf);
this.timelineServiceV2Enabled = YarnConfiguration.
timelineServiceV2Enabled(conf);
this.exclusiveEnforcedPartitions = context.getExclusiveEnforcedPartitions();
this.exclusiveEnforcedPartitions = YarnConfiguration
.getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration());
}
/**

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.nio.ByteBuffer;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
@ -178,7 +177,4 @@ public interface RMContext extends ApplicationMasterServiceContext {
void setPlacementConstraintManager(
PlacementConstraintManager placementConstraintManager);
Set<String> getExclusiveEnforcedPartitions();
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
View File

@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
@ -35,7 +33,6 @@ import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
@ -623,20 +620,4 @@ public class RMContextImpl implements RMContext {
this.activeServiceContext.setResourceProfilesManager(mgr);
}
// Note: Read java doc before adding any services over here.
public Set<String> getExclusiveEnforcedPartitions() {
Set<String> exclusiveEnforcedPartitions = new HashSet<>();
Configuration conf = getYarnConfiguration();
if (conf == null) {
return new HashSet<>();
}
String[] configuredPartitions = conf.getStrings(
YarnConfiguration.EXCLUSIVE_ENFORCED_PARTITIONS);
if (configuredPartitions != null) {
for (String partition : configuredPartitions) {
exclusiveEnforcedPartitions.add(partition);
}
}
return exclusiveEnforcedPartitions;
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -166,6 +166,7 @@ public class TestAppManager{
metricsPublisher = mock(SystemMetricsPublisher.class);
context.setSystemMetricsPublisher(metricsPublisher);
context.setRMApplicationHistoryWriter(writer);
((RMContextImpl) context).setYarnConfiguration(new YarnConfiguration());
return context;
}