Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7041. Nodemanager NPE running jobs with security off. Contributed by Varun Saxena. 

(cherry picked from commit e976bf10bb0d42e4d971358554ea736ddcdafd19)
This commit is contained in:
Rohith Sharma K S 2017-08-19 11:37:17 +05:30 committed by Varun Saxena
parent 53ddf1b15f
commit 5d9b4bd3e8
5 changed files with 71 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java
View File

@ -114,9 +114,13 @@ public class CollectorInfoPBImpl extends CollectorInfo {
@Override
public Token getCollectorToken() {
CollectorInfoProtoOrBuilder p = viaProto ? proto : builder;
if (this.collectorToken == null && p.hasCollectorToken()) {
this.collectorToken = convertFromProtoFormat(p.getCollectorToken());
if (this.collectorToken != null) {
return this.collectorToken;
}
if (!p.hasCollectorToken()) {
return null;
}
this.collectorToken = convertFromProtoFormat(p.getCollectorToken());
return this.collectorToken;
}

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
View File

@ -168,12 +168,17 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
for (Map.Entry<ApplicationId, AppCollectorData> entry :
registeringCollectors.entrySet()) {
AppCollectorData data = entry.getValue();
builder.addRegisteringCollectors(AppCollectorDataProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey()))
.setAppCollectorAddr(data.getCollectorAddr())
.setAppCollectorToken(convertToProtoFormat(data.getCollectorToken()))
.setRmIdentifier(data.getRMIdentifier())
.setVersion(data.getVersion()));
AppCollectorDataProto.Builder appCollectorDataBuilder =
AppCollectorDataProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey()))
.setAppCollectorAddr(data.getCollectorAddr())
.setRmIdentifier(data.getRMIdentifier())
.setVersion(data.getVersion());
if (data.getCollectorToken() != null) {
appCollectorDataBuilder.setAppCollectorToken(
convertToProtoFormat(data.getCollectorToken()));
}
builder.addRegisteringCollectors(appCollectorDataBuilder);
}
}
@ -274,7 +279,10 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
this.registeringCollectors = new HashMap<>();
for (AppCollectorDataProto c : list) {
ApplicationId appId = convertFromProtoFormat(c.getAppId());
Token collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
Token collectorToken = null;
if (c.hasAppCollectorToken()){
collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
}
AppCollectorData data = AppCollectorData.newInstance(appId,
c.getAppCollectorAddr(), c.getRmIdentifier(), c.getVersion(),
collectorToken);

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
View File

@ -158,13 +158,17 @@ public class NodeHeartbeatResponsePBImpl extends
for (Map.Entry<ApplicationId, AppCollectorData> entry
: appCollectorsMap.entrySet()) {
AppCollectorData data = entry.getValue();
builder.addAppCollectors(AppCollectorDataProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey()))
.setAppCollectorAddr(data.getCollectorAddr())
.setAppCollectorToken(
convertToProtoFormat(entry.getValue().getCollectorToken()))
.setRmIdentifier(data.getRMIdentifier())
.setVersion(data.getVersion()));
AppCollectorDataProto.Builder appCollectorDataBuilder =
AppCollectorDataProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey()))
.setAppCollectorAddr(data.getCollectorAddr())
.setRmIdentifier(data.getRMIdentifier())
.setVersion(data.getVersion());
if (data.getCollectorToken() != null) {
appCollectorDataBuilder.setAppCollectorToken(
convertToProtoFormat(data.getCollectorToken()));
}
builder.addAppCollectors(appCollectorDataBuilder);
}
}
@ -668,7 +672,10 @@ public class NodeHeartbeatResponsePBImpl extends
this.appCollectorsMap = new HashMap<>();
for (AppCollectorDataProto c : list) {
ApplicationId appId = convertFromProtoFormat(c.getAppId());
Token collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
Token collectorToken = null;
if (c.hasAppCollectorToken()){
collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
}
AppCollectorData data = AppCollectorData.newInstance(appId,
c.getAppCollectorAddr(), c.getRmIdentifier(), c.getVersion(),
collectorToken);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java
View File

@ -163,9 +163,13 @@ public class AppCollectorDataPBImpl extends AppCollectorData {
@Override
public Token getCollectorToken() {
AppCollectorDataProtoOrBuilder p = viaProto ? proto : builder;
if (this.collectorToken == null && p.hasAppCollectorToken()) {
this.collectorToken = new TokenPBImpl(p.getAppCollectorToken());
if (this.collectorToken != null) {
return this.collectorToken;
}
if (!p.hasAppCollectorToken()) {
return null;
}
this.collectorToken = new TokenPBImpl(p.getAppCollectorToken());
return this.collectorToken;
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
View File

@ -111,7 +111,7 @@ public class TestYarnServerApiClasses {
original.setLastKnownNMTokenMasterKey(getMasterKey());
original.setNodeStatus(getNodeStatus());
original.setNodeLabels(getValidNodeLabels());
Map<ApplicationId, AppCollectorData> collectors = getCollectors();
Map<ApplicationId, AppCollectorData> collectors = getCollectors(false);
original.setRegisteringCollectors(collectors);
NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl(
original.getProto());
@ -130,6 +130,16 @@ public class TestYarnServerApiClasses {
Assert.assertEquals(0, copy.getNodeLabels().size());
}
@Test
public void testNodeHBRequestPBImplWithNullCollectorToken() {
NodeHeartbeatRequestPBImpl original = new NodeHeartbeatRequestPBImpl();
Map<ApplicationId, AppCollectorData> collectors = getCollectors(true);
original.setRegisteringCollectors(collectors);
NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl(
original.getProto());
assertEquals(collectors, copy.getRegisteringCollectors());
}
/**
* Test NodeHeartbeatRequestPBImpl.
*/
@ -155,7 +165,7 @@ public class TestYarnServerApiClasses {
original.setNextHeartBeatInterval(1000);
original.setNodeAction(NodeAction.NORMAL);
original.setResponseId(100);
Map<ApplicationId, AppCollectorData> collectors = getCollectors();
Map<ApplicationId, AppCollectorData> collectors = getCollectors(false);
original.setAppCollectors(collectors);
NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl(
@ -179,6 +189,16 @@ public class TestYarnServerApiClasses {
assertTrue(copy.getAreNodeLabelsAcceptedByRM());
}
@Test
public void testNodeHBResponsePBImplWithNullCollectorToken() {
NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl();
Map<ApplicationId, AppCollectorData> collectors = getCollectors(true);
original.setAppCollectors(collectors);
NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl(
original.getProto());
assertEquals(collectors, copy.getAppCollectors());
}
@Test
public void testNodeHeartbeatResponsePBImplWithDecreasedContainers() {
NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl();
@ -349,11 +369,15 @@ public class TestYarnServerApiClasses {
return nodeLabels;
}
private Map<ApplicationId, AppCollectorData> getCollectors() {
private Map<ApplicationId, AppCollectorData> getCollectors(
boolean hasNullCollectorToken) {
ApplicationId appID = ApplicationId.newInstance(1L, 1);
String collectorAddr = "localhost:0";
AppCollectorData data = AppCollectorData.newInstance(appID, collectorAddr,
Token.newInstance(new byte[0], "kind", new byte[0], "s"));
AppCollectorData data = AppCollectorData.newInstance(appID, collectorAddr);
if (!hasNullCollectorToken) {
data.setCollectorToken(
Token.newInstance(new byte[0], "kind", new byte[0], "s"));
}
Map<ApplicationId, AppCollectorData> collectorMap =
new HashMap<>();
collectorMap.put(appID, data);