Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5837. NPE when getting node status of a decommissioned node after an RM restart. Contributed by Robert Kanter

This commit is contained in:
Jason Lowe 2016-11-04 22:41:11 +00:00
parent 49f983633a
commit c1a6f4f2ee
4 changed files with 29 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -59,6 +59,9 @@ Release 2.7.4 - UNRELEASED
YARN-5001. Aggregated Logs root directory is created with wrong group if
nonexistent (Haibo Chen via jlowe)
YARN-5837. NPE when getting node status of a decommissioned node after an
RM restart (Robert Kanter via jlowe)
Release 2.7.3 - 2016-08-25
INCOMPATIBLE CHANGES

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
View File

@ -32,6 +32,7 @@ import org.apache.hadoop.net.Node;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.HostsFileReader;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
@ -142,7 +143,8 @@ public class NodesListManager extends AbstractService implements
for (final String host : excludeList) {
NodeId nodeId = createUnknownNodeId(host);
RMNodeImpl rmNode = new RMNodeImpl(nodeId,
rmContext, host, -1, -1, new UnknownNode(host), null, null);
rmContext, host, -1, -1, new UnknownNode(host),
Resource.newInstance(0, 0), "unknown");
rmContext.getInactiveRMNodes().put(nodeId.getHost(), rmNode);
rmNode.handle(new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
View File

@ -231,4 +231,8 @@ public class MockNM {
public int getvCores() {
return vCores;
}
public String getVersion() {
return version;
}
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
View File

@ -38,6 +38,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
@ -98,6 +99,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@ -1856,6 +1858,9 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
rm1.start();
MockNM nm1 = rm1.registerNode("localhost:1234", 8000);
MockNM nm2 = rm1.registerNode("host2:1234", 8000);
Resource expectedCapability =
Resource.newInstance(nm1.getMemory(), nm1.getvCores());
String expectedVersion = nm1.getVersion();
Assert
.assertEquals(0,
ClusterMetrics.getMetrics().getNumDecommisionedNMs());
@ -1877,6 +1882,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
Assert
.assertEquals(2,
ClusterMetrics.getMetrics().getNumDecommisionedNMs());
verifyNodesAfterDecom(rm1, 2, expectedCapability, expectedVersion);
rm1.stop();
rm1 = null;
Assert
@ -1890,6 +1896,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
Assert
.assertEquals(2,
ClusterMetrics.getMetrics().getNumDecommisionedNMs());
verifyNodesAfterDecom(rm2, 2, Resource.newInstance(0, 0), "unknown");
} finally {
if (rm1 != null) {
rm1.stop();
@ -1900,6 +1907,18 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
}
}
private void verifyNodesAfterDecom(MockRM rm, int numNodes,
Resource expectedCapability,
String expectedVersion) {
ConcurrentMap<String, RMNode> inactiveRMNodes =
rm.getRMContext().getInactiveRMNodes();
Assert.assertEquals(numNodes, inactiveRMNodes.size());
for (RMNode rmNode : inactiveRMNodes.values()) {
Assert.assertEquals(expectedCapability, rmNode.getTotalCapability());
Assert.assertEquals(expectedVersion, rmNode.getNodeManagerVersion());
}
}
// Test Delegation token is renewed synchronously so that recover events
// can be processed before any other external incoming events, specifically
// the ContainerFinished event on NM re-registraton.