Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4201. AMBlacklist does not work for minicluster. Contributed by Jun Gong.

This commit is contained in:
Zhihai Xu 2015-10-12 00:13:30 -07:00
parent 73b86a5046
commit 049c6e8dc0
5 changed files with 35 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -924,6 +924,8 @@ Release 2.8.0 - UNRELEASED
YARN-4140. RM container allocation delayed incase of app submitted to
Nodelabel partition. (Bibin A Chundatt via wangda)
YARN-4201. AMBlacklist does not work for minicluster. (Jun Gong via zxu)
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -94,6 +94,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
@ -1811,7 +1812,13 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
private void addAMNodeToBlackList(NodeId nodeId) {
blacklistedNodesForAM.addNode(nodeId.getHost().toString());
SchedulerNode schedulerNode = scheduler.getSchedulerNode(nodeId);
if (schedulerNode != null) {
blacklistedNodesForAM.addNode(schedulerNode.getNodeName());
} else {
LOG.info(nodeId + " is not added to AM blacklist for "
+ applicationAttemptId + ", because it has been removed");
}
}
@Override

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
View File

@ -604,6 +604,7 @@ public abstract class AbstractYarnScheduler
SchedContainerChangeRequest decreaseRequest,
SchedulerApplicationAttempt attempt);
@Override
public SchedulerNode getSchedulerNode(NodeId nodeId) {
return nodes.get(nodeId);
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
View File

@ -352,4 +352,13 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* @return maximum priority of cluster
*/
Priority getMaxClusterLevelAppPriority();
/**
* Get SchedulerNode corresponds to nodeId.
*
* @param nodeId the node id of RMNode
*
* @return SchedulerNode corresponds to nodeId
*/
SchedulerNode getSchedulerNode(NodeId nodeId);
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
View File

@ -383,6 +383,21 @@ public class TestAMRestart {
public void testAMBlacklistPreventsRestartOnSameNode() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true);
testAMBlacklistPreventRestartOnSameNode(conf);
}
@Test(timeout = 100000)
public void testAMBlacklistPreventsRestartOnSameNodeForMinicluster()
throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true);
conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
true);
testAMBlacklistPreventRestartOnSameNode(conf);
}
private void testAMBlacklistPreventRestartOnSameNode(YarnConfiguration conf)
throws Exception{
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
final DrainDispatcher dispatcher = new DrainDispatcher();