YARN-2800. Remove MemoryNodeLabelsStore and add a way to enable/disable node labels feature. Contributed by Wangda Tan.
This commit is contained in:
parent
3aab354e66
commit
24aa462673
|
@ -197,6 +197,9 @@ Release 2.7.0 - UNRELEASED
|
||||||
|
|
||||||
YARN-2984. Metrics for container's actual memory usage. (kasha)
|
YARN-2984. Metrics for container's actual memory usage. (kasha)
|
||||||
|
|
||||||
|
YARN-2800. Remove MemoryNodeLabelsStore and add a way to enable/disable
|
||||||
|
node labels feature. (Wangda Tan via ozawa)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
BUG FIXES
|
BUG FIXES
|
||||||
|
|
|
@ -1648,14 +1648,10 @@ public class YarnConfiguration extends Configuration {
|
||||||
public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY
|
public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY
|
||||||
.name();
|
.name();
|
||||||
|
|
||||||
public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels.";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Class for RMNodeLabelsManager Please note this value should be consistent
|
* Node-labels configurations
|
||||||
* in client nodes and RM node(s)
|
|
||||||
*/
|
*/
|
||||||
public static final String RM_NODE_LABELS_MANAGER_CLASS = NODE_LABELS_PREFIX
|
public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels.";
|
||||||
+ "manager-class";
|
|
||||||
|
|
||||||
/** URI for NodeLabelManager */
|
/** URI for NodeLabelManager */
|
||||||
public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX
|
public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX
|
||||||
|
@ -1665,6 +1661,14 @@ public class YarnConfiguration extends Configuration {
|
||||||
public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC =
|
public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC =
|
||||||
"2000, 500";
|
"2000, 500";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flag to indicate if the node labels feature enabled, by default it's
|
||||||
|
* disabled
|
||||||
|
*/
|
||||||
|
public static final String NODE_LABELS_ENABLED = NODE_LABELS_PREFIX
|
||||||
|
+ "enabled";
|
||||||
|
public static final boolean DEFAULT_NODE_LABELS_ENABLED = false;
|
||||||
|
|
||||||
public YarnConfiguration() {
|
public YarnConfiguration() {
|
||||||
super();
|
super();
|
||||||
}
|
}
|
||||||
|
|
|
@ -80,6 +80,7 @@ public class TestDistributedShell {
|
||||||
conf.set("yarn.log.dir", "target");
|
conf.set("yarn.log.dir", "target");
|
||||||
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
||||||
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
|
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
|
||||||
|
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
|
||||||
|
|
||||||
if (yarnCluster == null) {
|
if (yarnCluster == null) {
|
||||||
yarnCluster =
|
yarnCluster =
|
||||||
|
|
|
@ -41,6 +41,7 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.service.AbstractService;
|
import org.apache.hadoop.service.AbstractService;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
||||||
import org.apache.hadoop.yarn.event.Dispatcher;
|
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
|
@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.nodelabels.event.RemoveClusterNodeLabels;
|
||||||
import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent;
|
import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent;
|
||||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
|
|
||||||
public class CommonNodeLabelsManager extends AbstractService {
|
public class CommonNodeLabelsManager extends AbstractService {
|
||||||
|
@ -65,6 +67,14 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
.compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*");
|
.compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*");
|
||||||
public static final int WILDCARD_PORT = 0;
|
public static final int WILDCARD_PORT = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Error messages
|
||||||
|
*/
|
||||||
|
@VisibleForTesting
|
||||||
|
public static final String NODE_LABELS_NOT_ENABLED_ERR =
|
||||||
|
"Node-label-based scheduling is disabled. Please check "
|
||||||
|
+ YarnConfiguration.NODE_LABELS_ENABLED;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* If a user doesn't specify label of a queue or node, it belongs
|
* If a user doesn't specify label of a queue or node, it belongs
|
||||||
* DEFAULT_LABEL
|
* DEFAULT_LABEL
|
||||||
|
@ -82,6 +92,7 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
protected final WriteLock writeLock;
|
protected final WriteLock writeLock;
|
||||||
|
|
||||||
protected NodeLabelsStore store;
|
protected NodeLabelsStore store;
|
||||||
|
private boolean nodeLabelsEnabled = false;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A <code>Host</code> can have multiple <code>Node</code>s
|
* A <code>Host</code> can have multiple <code>Node</code>s
|
||||||
|
@ -193,7 +204,13 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void serviceInit(Configuration conf) throws Exception {
|
protected void serviceInit(Configuration conf) throws Exception {
|
||||||
|
// set if node labels enabled
|
||||||
|
nodeLabelsEnabled =
|
||||||
|
conf.getBoolean(YarnConfiguration.NODE_LABELS_ENABLED,
|
||||||
|
YarnConfiguration.DEFAULT_NODE_LABELS_ENABLED);
|
||||||
|
if (nodeLabelsEnabled) {
|
||||||
initNodeLabelStore(conf);
|
initNodeLabelStore(conf);
|
||||||
|
}
|
||||||
|
|
||||||
labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
|
labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
|
||||||
}
|
}
|
||||||
|
@ -250,6 +267,10 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public void addToCluserNodeLabels(Set<String> labels) throws IOException {
|
public void addToCluserNodeLabels(Set<String> labels) throws IOException {
|
||||||
|
if (!nodeLabelsEnabled) {
|
||||||
|
LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
}
|
||||||
if (null == labels || labels.isEmpty()) {
|
if (null == labels || labels.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -304,6 +325,10 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
*/
|
*/
|
||||||
public void addLabelsToNode(Map<NodeId, Set<String>> addedLabelsToNode)
|
public void addLabelsToNode(Map<NodeId, Set<String>> addedLabelsToNode)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
if (!nodeLabelsEnabled) {
|
||||||
|
LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
}
|
||||||
addedLabelsToNode = normalizeNodeIdToLabels(addedLabelsToNode);
|
addedLabelsToNode = normalizeNodeIdToLabels(addedLabelsToNode);
|
||||||
checkAddLabelsToNode(addedLabelsToNode);
|
checkAddLabelsToNode(addedLabelsToNode);
|
||||||
internalUpdateLabelsOnNodes(addedLabelsToNode, NodeLabelUpdateOperation.ADD);
|
internalUpdateLabelsOnNodes(addedLabelsToNode, NodeLabelUpdateOperation.ADD);
|
||||||
|
@ -370,6 +395,11 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
*/
|
*/
|
||||||
public void removeFromClusterNodeLabels(Collection<String> labelsToRemove)
|
public void removeFromClusterNodeLabels(Collection<String> labelsToRemove)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
if (!nodeLabelsEnabled) {
|
||||||
|
LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
}
|
||||||
|
|
||||||
labelsToRemove = normalizeLabels(labelsToRemove);
|
labelsToRemove = normalizeLabels(labelsToRemove);
|
||||||
|
|
||||||
checkRemoveFromClusterNodeLabels(labelsToRemove);
|
checkRemoveFromClusterNodeLabels(labelsToRemove);
|
||||||
|
@ -532,6 +562,11 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
public void
|
public void
|
||||||
removeLabelsFromNode(Map<NodeId, Set<String>> removeLabelsFromNode)
|
removeLabelsFromNode(Map<NodeId, Set<String>> removeLabelsFromNode)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
if (!nodeLabelsEnabled) {
|
||||||
|
LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
}
|
||||||
|
|
||||||
removeLabelsFromNode = normalizeNodeIdToLabels(removeLabelsFromNode);
|
removeLabelsFromNode = normalizeNodeIdToLabels(removeLabelsFromNode);
|
||||||
|
|
||||||
checkRemoveLabelsFromNode(removeLabelsFromNode);
|
checkRemoveLabelsFromNode(removeLabelsFromNode);
|
||||||
|
@ -567,6 +602,11 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
*/
|
*/
|
||||||
public void replaceLabelsOnNode(Map<NodeId, Set<String>> replaceLabelsToNode)
|
public void replaceLabelsOnNode(Map<NodeId, Set<String>> replaceLabelsToNode)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
if (!nodeLabelsEnabled) {
|
||||||
|
LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
|
||||||
|
}
|
||||||
|
|
||||||
replaceLabelsToNode = normalizeNodeIdToLabels(replaceLabelsToNode);
|
replaceLabelsToNode = normalizeNodeIdToLabels(replaceLabelsToNode);
|
||||||
|
|
||||||
checkReplaceLabelsOnNode(replaceLabelsToNode);
|
checkReplaceLabelsOnNode(replaceLabelsToNode);
|
||||||
|
|
|
@ -26,6 +26,8 @@ import java.util.Set;
|
||||||
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
@ -41,7 +43,9 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
|
||||||
@Before
|
@Before
|
||||||
public void before() {
|
public void before() {
|
||||||
mgr = new DummyCommonNodeLabelsManager();
|
mgr = new DummyCommonNodeLabelsManager();
|
||||||
mgr.init(new Configuration());
|
Configuration conf = new YarnConfiguration();
|
||||||
|
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
|
||||||
|
mgr.init(conf);
|
||||||
mgr.start();
|
mgr.start();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -319,4 +323,58 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
|
||||||
toSet("p1"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"),
|
toSet("p1"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"),
|
||||||
toSet("p1")));
|
toSet("p1")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void assertNodeLabelsDisabledErrorMessage(IOException e) {
|
||||||
|
Assert.assertEquals(CommonNodeLabelsManager.NODE_LABELS_NOT_ENABLED_ERR,
|
||||||
|
e.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 5000)
|
||||||
|
public void testNodeLabelsDisabled() throws IOException {
|
||||||
|
DummyCommonNodeLabelsManager mgr = new DummyCommonNodeLabelsManager();
|
||||||
|
Configuration conf = new YarnConfiguration();
|
||||||
|
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
|
||||||
|
mgr.init(conf);
|
||||||
|
mgr.start();
|
||||||
|
|
||||||
|
// add labels
|
||||||
|
try {
|
||||||
|
mgr.addToCluserNodeLabels(ImmutableSet.of("x"));
|
||||||
|
} catch (IOException e) {
|
||||||
|
assertNodeLabelsDisabledErrorMessage(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove labels
|
||||||
|
try {
|
||||||
|
mgr.removeFromClusterNodeLabels(ImmutableSet.of("x"));
|
||||||
|
} catch (IOException e) {
|
||||||
|
assertNodeLabelsDisabledErrorMessage(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// add labels to node
|
||||||
|
try {
|
||||||
|
mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host", 0),
|
||||||
|
CommonNodeLabelsManager.EMPTY_STRING_SET));
|
||||||
|
} catch (IOException e) {
|
||||||
|
assertNodeLabelsDisabledErrorMessage(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove labels from node
|
||||||
|
try {
|
||||||
|
mgr.removeLabelsFromNode(ImmutableMap.of(NodeId.newInstance("host", 0),
|
||||||
|
CommonNodeLabelsManager.EMPTY_STRING_SET));
|
||||||
|
} catch (IOException e) {
|
||||||
|
assertNodeLabelsDisabledErrorMessage(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// replace labels on node
|
||||||
|
try {
|
||||||
|
mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("host", 0),
|
||||||
|
CommonNodeLabelsManager.EMPTY_STRING_SET));
|
||||||
|
} catch (IOException e) {
|
||||||
|
assertNodeLabelsDisabledErrorMessage(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
mgr.close();
|
||||||
|
}
|
||||||
}
|
}
|
|
@ -63,6 +63,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
|
||||||
public void before() throws IOException {
|
public void before() throws IOException {
|
||||||
mgr = new MockNodeLabelManager();
|
mgr = new MockNodeLabelManager();
|
||||||
conf = new Configuration();
|
conf = new Configuration();
|
||||||
|
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
|
||||||
File tempDir = File.createTempFile("nlb", ".tmp");
|
File tempDir = File.createTempFile("nlb", ".tmp");
|
||||||
tempDir.delete();
|
tempDir.delete();
|
||||||
tempDir.mkdirs();
|
tempDir.mkdirs();
|
||||||
|
|
|
@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMaste
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
|
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
|
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
|
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
|
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
|
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
|
||||||
|
@ -334,10 +333,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
|
||||||
|
|
||||||
protected RMNodeLabelsManager createNodeLabelManager()
|
protected RMNodeLabelsManager createNodeLabelManager()
|
||||||
throws InstantiationException, IllegalAccessException {
|
throws InstantiationException, IllegalAccessException {
|
||||||
Class<? extends RMNodeLabelsManager> nlmCls =
|
return new RMNodeLabelsManager();
|
||||||
conf.getClass(YarnConfiguration.RM_NODE_LABELS_MANAGER_CLASS,
|
|
||||||
MemoryRMNodeLabelsManager.class, RMNodeLabelsManager.class);
|
|
||||||
return nlmCls.newInstance();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected DelegationTokenRenewer createDelegationTokenRenewer() {
|
protected DelegationTokenRenewer createDelegationTokenRenewer() {
|
||||||
|
|
|
@ -59,7 +59,7 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
|
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
|
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
|
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
|
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
|
@ -115,7 +115,7 @@ public class MockRM extends ResourceManager {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected RMNodeLabelsManager createNodeLabelManager() {
|
protected RMNodeLabelsManager createNodeLabelManager() {
|
||||||
RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
|
RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
|
||||||
mgr.init(getConfig());
|
mgr.init(getConfig());
|
||||||
return mgr;
|
return mgr;
|
||||||
}
|
}
|
||||||
|
|
|
@ -2073,6 +2073,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
|
||||||
|
|
||||||
MemoryRMStateStore memStore = new MemoryRMStateStore();
|
MemoryRMStateStore memStore = new MemoryRMStateStore();
|
||||||
memStore.init(conf);
|
memStore.init(conf);
|
||||||
|
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
|
||||||
MockRM rm1 = new MockRM(conf, memStore) {
|
MockRM rm1 = new MockRM(conf, memStore) {
|
||||||
@Override
|
@Override
|
||||||
protected RMNodeLabelsManager createNodeLabelManager() {
|
protected RMNodeLabelsManager createNodeLabelManager() {
|
||||||
|
|
|
@ -25,9 +25,10 @@ import java.util.Set;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore;
|
import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore;
|
||||||
|
|
||||||
public class MemoryRMNodeLabelsManager extends RMNodeLabelsManager {
|
public class NullRMNodeLabelsManager extends RMNodeLabelsManager {
|
||||||
Map<NodeId, Set<String>> lastNodeToLabels = null;
|
Map<NodeId, Set<String>> lastNodeToLabels = null;
|
||||||
Collection<String> lastAddedlabels = null;
|
Collection<String> lastAddedlabels = null;
|
||||||
Collection<String> lastRemovedlabels = null;
|
Collection<String> lastRemovedlabels = null;
|
||||||
|
@ -79,4 +80,11 @@ public class MemoryRMNodeLabelsManager extends RMNodeLabelsManager {
|
||||||
protected void stopDispatcher() {
|
protected void stopDispatcher() {
|
||||||
// do nothing
|
// do nothing
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void serviceInit(Configuration conf) throws Exception {
|
||||||
|
// always enable node labels while using MemoryRMNodeLabelsManager
|
||||||
|
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
|
||||||
|
super.serviceInit(conf);
|
||||||
|
}
|
||||||
}
|
}
|
|
@ -27,6 +27,7 @@ import java.util.Set;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
|
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
|
||||||
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
|
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
|
||||||
|
@ -44,12 +45,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
|
||||||
private final Resource SMALL_RESOURCE = Resource.newInstance(100, 0);
|
private final Resource SMALL_RESOURCE = Resource.newInstance(100, 0);
|
||||||
private final Resource LARGE_NODE = Resource.newInstance(1000, 0);
|
private final Resource LARGE_NODE = Resource.newInstance(1000, 0);
|
||||||
|
|
||||||
MemoryRMNodeLabelsManager mgr = null;
|
NullRMNodeLabelsManager mgr = null;
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void before() {
|
public void before() {
|
||||||
mgr = new MemoryRMNodeLabelsManager();
|
mgr = new NullRMNodeLabelsManager();
|
||||||
mgr.init(new Configuration());
|
Configuration conf = new Configuration();
|
||||||
|
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
|
||||||
|
mgr.init(conf);
|
||||||
mgr.start();
|
mgr.start();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -84,7 +84,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.Task;
|
import org.apache.hadoop.yarn.server.resourcemanager.Task;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
|
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
|
||||||
|
@ -154,7 +154,7 @@ public class TestCapacityScheduler {
|
||||||
resourceManager = new ResourceManager() {
|
resourceManager = new ResourceManager() {
|
||||||
@Override
|
@Override
|
||||||
protected RMNodeLabelsManager createNodeLabelManager() {
|
protected RMNodeLabelsManager createNodeLabelManager() {
|
||||||
RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
|
RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
|
||||||
mgr.init(getConfig());
|
mgr.init(getConfig());
|
||||||
return mgr;
|
return mgr;
|
||||||
}
|
}
|
||||||
|
@ -1485,7 +1485,7 @@ public class TestCapacityScheduler {
|
||||||
resourceManager = new ResourceManager() {
|
resourceManager = new ResourceManager() {
|
||||||
@Override
|
@Override
|
||||||
protected RMNodeLabelsManager createNodeLabelManager() {
|
protected RMNodeLabelsManager createNodeLabelManager() {
|
||||||
RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
|
RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
|
||||||
mgr.init(getConfig());
|
mgr.init(getConfig());
|
||||||
return mgr;
|
return mgr;
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
|
||||||
|
@ -54,7 +54,7 @@ public class TestCapacitySchedulerNodeLabelUpdate {
|
||||||
conf = new YarnConfiguration();
|
conf = new YarnConfiguration();
|
||||||
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
|
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
|
||||||
ResourceScheduler.class);
|
ResourceScheduler.class);
|
||||||
mgr = new MemoryRMNodeLabelsManager();
|
mgr = new NullRMNodeLabelsManager();
|
||||||
mgr.init(conf);
|
mgr.init(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -45,7 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
||||||
|
@ -81,7 +81,7 @@ public class TestContainerAllocation {
|
||||||
conf = new YarnConfiguration();
|
conf = new YarnConfiguration();
|
||||||
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
|
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
|
||||||
ResourceScheduler.class);
|
ResourceScheduler.class);
|
||||||
mgr = new MemoryRMNodeLabelsManager();
|
mgr = new NullRMNodeLabelsManager();
|
||||||
mgr.init(conf);
|
mgr.init(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -451,7 +451,7 @@ public class TestContainerAllocation {
|
||||||
|
|
||||||
@Test(timeout = 300000)
|
@Test(timeout = 300000)
|
||||||
public void testContainerAllocationWithSingleUserLimits() throws Exception {
|
public void testContainerAllocationWithSingleUserLimits() throws Exception {
|
||||||
final RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
|
final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
|
||||||
mgr.init(conf);
|
mgr.init(conf);
|
||||||
|
|
||||||
// set node -> label
|
// set node -> label
|
||||||
|
|
|
@ -26,7 +26,7 @@ import org.apache.hadoop.service.ServiceOperations;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
|
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
|
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
|
||||||
|
@ -47,7 +47,7 @@ public class TestQueueParsing {
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setup() {
|
public void setup() {
|
||||||
nodeLabelManager = new MemoryRMNodeLabelsManager();
|
nodeLabelManager = new NullRMNodeLabelsManager();
|
||||||
nodeLabelManager.init(new YarnConfiguration());
|
nodeLabelManager.init(new YarnConfiguration());
|
||||||
nodeLabelManager.start();
|
nodeLabelManager.start();
|
||||||
}
|
}
|
||||||
|
@ -566,7 +566,7 @@ public class TestQueueParsing {
|
||||||
new NMTokenSecretManagerInRM(csConf),
|
new NMTokenSecretManagerInRM(csConf),
|
||||||
new ClientToAMTokenSecretManagerInRM(), null);
|
new ClientToAMTokenSecretManagerInRM(), null);
|
||||||
|
|
||||||
RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
|
RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
|
||||||
nodeLabelsManager.init(conf);
|
nodeLabelsManager.init(conf);
|
||||||
nodeLabelsManager.start();
|
nodeLabelsManager.start();
|
||||||
|
|
||||||
|
@ -594,7 +594,7 @@ public class TestQueueParsing {
|
||||||
new NMTokenSecretManagerInRM(csConf),
|
new NMTokenSecretManagerInRM(csConf),
|
||||||
new ClientToAMTokenSecretManagerInRM(), null);
|
new ClientToAMTokenSecretManagerInRM(), null);
|
||||||
|
|
||||||
RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
|
RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
|
||||||
nodeLabelsManager.init(conf);
|
nodeLabelsManager.init(conf);
|
||||||
nodeLabelsManager.start();
|
nodeLabelsManager.start();
|
||||||
|
|
||||||
|
@ -622,7 +622,7 @@ public class TestQueueParsing {
|
||||||
new NMTokenSecretManagerInRM(csConf),
|
new NMTokenSecretManagerInRM(csConf),
|
||||||
new ClientToAMTokenSecretManagerInRM(), null);
|
new ClientToAMTokenSecretManagerInRM(), null);
|
||||||
|
|
||||||
RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
|
RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
|
||||||
nodeLabelsManager.init(conf);
|
nodeLabelsManager.init(conf);
|
||||||
nodeLabelsManager.start();
|
nodeLabelsManager.start();
|
||||||
|
|
||||||
|
@ -649,7 +649,7 @@ public class TestQueueParsing {
|
||||||
new NMTokenSecretManagerInRM(csConf),
|
new NMTokenSecretManagerInRM(csConf),
|
||||||
new ClientToAMTokenSecretManagerInRM(), null);
|
new ClientToAMTokenSecretManagerInRM(), null);
|
||||||
|
|
||||||
RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
|
RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
|
||||||
nodeLabelsManager.init(conf);
|
nodeLabelsManager.init(conf);
|
||||||
nodeLabelsManager.start();
|
nodeLabelsManager.start();
|
||||||
|
|
||||||
|
|
|
@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm;
|
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||||
|
@ -179,7 +179,7 @@ public class TestRMWebApp {
|
||||||
return nodesMap;
|
return nodesMap;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager());
|
rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
|
||||||
return rmContext;
|
return rmContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -211,7 +211,7 @@ public class TestRMWebApp {
|
||||||
null, new RMContainerTokenSecretManager(conf),
|
null, new RMContainerTokenSecretManager(conf),
|
||||||
new NMTokenSecretManagerInRM(conf),
|
new NMTokenSecretManagerInRM(conf),
|
||||||
new ClientToAMTokenSecretManagerInRM(), null);
|
new ClientToAMTokenSecretManagerInRM(), null);
|
||||||
rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager());
|
rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
|
||||||
cs.setRMContext(rmContext);
|
cs.setRMContext(rmContext);
|
||||||
cs.init(conf);
|
cs.init(conf);
|
||||||
return cs;
|
return cs;
|
||||||
|
|
Loading…
Reference in New Issue