Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-2743. Fixed a bug in ResourceManager that was causing RMDelegationToken identifiers to be tampered and thus causing app submission failures in secure mode. Contributed by Jian He.

This commit is contained in:
Vinod Kumar Vavilapalli 2014-10-26 11:14:34 -07:00
parent 65d95b1a52
commit 0186645505
13 changed files with 199 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
View File

@ -159,7 +159,7 @@ extends TokenIdentifier {
return masterKeyId;
}
static boolean isEqual(Object a, Object b) {
protected static boolean isEqual(Object a, Object b) {
return a == null ? b == null : a.equals(b);
}

15
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
View File

@ -18,15 +18,8 @@
package org.apache.hadoop.security.token.delegation;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
@ -39,10 +32,13 @@ import javax.crypto.SecretKey;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.HadoopKerberosName;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.Time;
@ -386,7 +382,8 @@ extends AbstractDelegationTokenIdentifier>
identifier.setMaxDate(now + tokenMaxLifetime);
identifier.setMasterKeyId(currentKey.getKeyId());
identifier.setSequenceNumber(sequenceNum);
LOG.info("Creating password for identifier: [" + MD5Hash.digest(identifier.getBytes()) + ", " + currentKey.getKeyId() + "]");
LOG.info("Creating password for identifier: " + identifier
+ ", currentKey: " + currentKey.getKeyId());
byte[] password = createPassword(identifier.getBytes(), currentKey.getKey());
DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now
+ tokenRenewInterval, password, getTrackingIdIfEnabled(identifier));

4
hadoop-yarn-project/CHANGES.txt
View File

@ -640,6 +640,10 @@ Release 2.6.0 - UNRELEASED
YARN-2715. Fixed ResourceManager to respect common configurations for proxy
users/groups beyond just the YARN level config. (Zhijie Shen via vinodkv)
YARN-2743. Fixed a bug in ResourceManager that was causing RMDelegationToken
identifiers to be tampered and thus causing app submission failures in
secure mode. (Jian He via vinodkv)
BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
YARN-1707. Introduce APIs to add/remove/resize queues in the

210
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
View File

@ -22,193 +22,61 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.HadoopKerberosName;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto;
@Private
public abstract class YARNDelegationTokenIdentifier extends
AbstractDelegationTokenIdentifier {
YARNDelegationTokenIdentifierProto.Builder builder =
YARNDelegationTokenIdentifierProto.Builder builder =
YARNDelegationTokenIdentifierProto.newBuilder();
public YARNDelegationTokenIdentifier() {}
public YARNDelegationTokenIdentifier() {
}
public YARNDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
setOwner(owner);
setRenewer(renewer);
setRealUser(realUser);
}
/**
* Get the username encoded in the token identifier
*
* @return the username or owner
*/
@Override
public UserGroupInformation getUser() {
String owner = getOwner() == null ? null : getOwner().toString();
String realUser = getRealUser() == null ? null: getRealUser().toString();
if ( (owner == null) || (owner.toString().isEmpty())) {
return null;
}
final UserGroupInformation realUgi;
final UserGroupInformation ugi;
if ((realUser == null) || (realUser.toString().isEmpty())
|| realUser.equals(owner)) {
ugi = realUgi = UserGroupInformation.createRemoteUser(owner.toString());
} else {
realUgi = UserGroupInformation.createRemoteUser(realUser.toString());
ugi = UserGroupInformation.createProxyUser(owner.toString(), realUgi);
}
realUgi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
return ugi;
super(owner, renewer, realUser);
}
public Text getOwner() {
String owner = builder.getOwner();
if (owner == null) {
return null;
} else {
return new Text(owner);
}
public YARNDelegationTokenIdentifier(
YARNDelegationTokenIdentifierProto.Builder builder) {
this.builder = builder;
}
@Override
public void setOwner(Text owner) {
if (builder != null && owner != null) {
builder.setOwner(owner.toString());
}
}
public Text getRenewer() {
String renewer = builder.getRenewer();
if (renewer == null) {
return null;
} else {
return new Text(renewer);
}
}
@Override
public void setRenewer(Text renewer) {
if (builder != null && renewer != null) {
HadoopKerberosName renewerKrbName = new HadoopKerberosName(renewer.toString());
try {
builder.setRenewer(renewerKrbName.getShortName());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
public Text getRealUser() {
String realUser = builder.getRealUser();
if (realUser == null) {
return null;
} else {
return new Text(realUser);
}
}
@Override
public void setRealUser(Text realUser) {
if (builder != null && realUser != null) {
builder.setRealUser(realUser.toString());
}
}
public void setIssueDate(long issueDate) {
builder.setIssueDate(issueDate);
}
public long getIssueDate() {
return builder.getIssueDate();
}
public void setRenewDate(long renewDate) {
builder.setRenewDate(renewDate);
}
public long getRenewDate() {
return builder.getRenewDate();
}
public void setMaxDate(long maxDate) {
builder.setMaxDate(maxDate);
}
public long getMaxDate() {
return builder.getMaxDate();
}
public void setSequenceNumber(int seqNum) {
builder.setSequenceNumber(seqNum);
}
public int getSequenceNumber() {
return builder.getSequenceNumber();
}
public void setMasterKeyId(int newId) {
builder.setMasterKeyId(newId);
}
public int getMasterKeyId() {
return builder.getMasterKeyId();
}
protected static boolean isEqual(Object a, Object b) {
return a == null ? b == null : a.equals(b);
}
@Override
public boolean equals(Object obj) {
if (obj == this) {
return true;
}
if (obj instanceof YARNDelegationTokenIdentifier) {
YARNDelegationTokenIdentifier that = (YARNDelegationTokenIdentifier) obj;
return this.getSequenceNumber() == that.getSequenceNumber()
&& this.getIssueDate() == that.getIssueDate()
&& this.getMaxDate() == that.getMaxDate()
&& this.getMasterKeyId() == that.getMasterKeyId()
&& isEqual(this.getOwner(), that.getOwner())
&& isEqual(this.getRenewer(), that.getRenewer())
&& isEqual(this.getRealUser(), that.getRealUser());
}
return false;
}
@Override
public int hashCode() {
return this.getSequenceNumber();
}
@Override
public void readFields(DataInput in) throws IOException {
public synchronized void readFields(DataInput in) throws IOException {
builder.mergeFrom((DataInputStream) in);
}
@Override
public void write(DataOutput out) throws IOException {
builder.build().writeTo((DataOutputStream)out);
}
@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
buffer
.append("owner=" + getOwner() + ", renewer=" + getRenewer() + ", realUser="
+ getRealUser() + ", issueDate=" + getIssueDate()
+ ", maxDate=" + getMaxDate() + ", sequenceNumber="
+ getSequenceNumber() + ", masterKeyId="
+ getMasterKeyId());
return buffer.toString();
if (builder.getOwner() != null) {
setOwner(new Text(builder.getOwner()));
}
if (builder.getRenewer() != null) {
setRenewer(new Text(builder.getRenewer()));
}
if (builder.getRealUser() != null) {
setRealUser(new Text(builder.getRealUser()));
}
setIssueDate(builder.getIssueDate());
setMaxDate(builder.getMaxDate());
setSequenceNumber(builder.getSequenceNumber());
setMasterKeyId(builder.getMasterKeyId());
}
@Override
public synchronized void write(DataOutput out) throws IOException {
builder.setOwner(getOwner().toString());
builder.setRenewer(getRenewer().toString());
builder.setRealUser(getRealUser().toString());
builder.setIssueDate(getIssueDate());
builder.setMaxDate(getMaxDate());
builder.setSequenceNumber(getSequenceNumber());
builder.setMasterKeyId(getMasterKeyId());
builder.build().writeTo((DataOutputStream) out);
}
public YARNDelegationTokenIdentifierProto getProto() {
return builder.build();
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
View File

@ -63,7 +63,6 @@ message YARNDelegationTokenIdentifierProto {
optional int64 issueDate = 4;
optional int64 maxDate = 5;
optional int32 sequenceNumber = 6;
optional int32 masterKeyId = 7 [default = -1];
optional int64 renewDate = 8;
optional int32 masterKeyId = 7;
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
View File

@ -257,7 +257,6 @@ public class TestYARNTokenIdentifier {
long maxDate = 2;
int sequenceNumber = 3;
int masterKeyId = 4;
long renewDate = 5;
TimelineDelegationTokenIdentifier token =
new TimelineDelegationTokenIdentifier(owner, renewer, realUser);
@ -265,7 +264,6 @@ public class TestYARNTokenIdentifier {
token.setMaxDate(maxDate);
token.setSequenceNumber(sequenceNumber);
token.setMasterKeyId(masterKeyId);
token.setRenewDate(renewDate);
TimelineDelegationTokenIdentifier anotherToken =
new TimelineDelegationTokenIdentifier();
@ -299,9 +297,6 @@ public class TestYARNTokenIdentifier {
Assert.assertEquals("masterKeyId from proto is not the same with original token",
anotherToken.getMasterKeyId(), masterKeyId);
Assert.assertEquals("renewDate from proto is not the same with original token",
anotherToken.getRenewDate(), renewDate);
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
View File

@ -261,6 +261,7 @@
<imports>
<param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>
<param>${basedir}/../../hadoop-yarn-common/src/main/proto/server/</param>
<param>${basedir}/../hadoop-yarn-server-common/src/main/proto</param>
<param>${basedir}/src/main/proto</param>
</imports>

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
View File

@ -47,9 +47,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.EpochProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.EpochProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenS
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMDelegationTokenIdentifierData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@ -369,12 +370,24 @@ public class FileSystemRMStateStore extends RMStateStore {
DelegationKey key = new DelegationKey();
key.readFields(fsIn);
rmState.rmSecretManagerState.masterKeyState.add(key);
if (LOG.isDebugEnabled()) {
LOG.debug("Loaded delegation key: keyId=" + key.getKeyId()
+ ", expirationDate=" + key.getExpiryDate());
}
} else if (childNodeName.startsWith(DELEGATION_TOKEN_PREFIX)) {
RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier();
identifier.readFields(fsIn);
long renewDate = identifier.getRenewDate();
RMDelegationTokenIdentifierData identifierData =
new RMDelegationTokenIdentifierData();
identifierData.readFields(fsIn);
RMDelegationTokenIdentifier identifier =
identifierData.getTokenIdentifier();
long renewDate = identifierData.getRenewDate();
rmState.rmSecretManagerState.delegationTokenState.put(identifier,
renewDate);
if (LOG.isDebugEnabled()) {
LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier
+ " renewDate=" + renewDate);
}
} else {
LOG.warn("Unknown file for recovering RMDelegationTokenSecretManager");
}
@ -503,18 +516,15 @@ public class FileSystemRMStateStore extends RMStateStore {
Path nodeCreatePath =
getNodePath(rmDTSecretManagerRoot,
DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber());
ByteArrayOutputStream os = new ByteArrayOutputStream();
DataOutputStream fsOut = new DataOutputStream(os);
identifier.setRenewDate(renewDate);
identifier.write(fsOut);
RMDelegationTokenIdentifierData identifierData =
new RMDelegationTokenIdentifierData(identifier, renewDate);
if (isUpdate) {
LOG.info("Updating RMDelegationToken_" + identifier.getSequenceNumber());
updateFile(nodeCreatePath, os.toByteArray());
updateFile(nodeCreatePath, identifierData.toByteArray());
} else {
LOG.info("Storing RMDelegationToken_" + identifier.getSequenceNumber());
writeFile(nodeCreatePath, os.toByteArray());
writeFile(nodeCreatePath, identifierData.toByteArray());
}
fsOut.close();
// store sequence number
Path latestSequenceNumberPath = getNodePath(rmDTSecretManagerRoot,

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
View File

@ -50,13 +50,14 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.Appl
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.EpochProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.RMZKUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMDelegationTokenIdentifierData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@ -488,6 +489,10 @@ public class ZKRMStateStore extends RMStateStore {
DelegationKey key = new DelegationKey();
key.readFields(fsIn);
rmState.rmSecretManagerState.masterKeyState.add(key);
if (LOG.isDebugEnabled()) {
LOG.debug("Loaded delegation key: keyId=" + key.getKeyId()
+ ", expirationDate=" + key.getExpiryDate());
}
}
} finally {
is.close();
@ -527,12 +532,18 @@ public class ZKRMStateStore extends RMStateStore {
try {
if (childNodeName.startsWith(DELEGATION_TOKEN_PREFIX)) {
RMDelegationTokenIdentifierData identifierData =
new RMDelegationTokenIdentifierData();
identifierData.readFields(fsIn);
RMDelegationTokenIdentifier identifier =
new RMDelegationTokenIdentifier();
identifier.readFields(fsIn);
long renewDate = identifier.getRenewDate();
identifierData.getTokenIdentifier();
long renewDate = identifierData.getRenewDate();
rmState.rmSecretManagerState.delegationTokenState.put(identifier,
renewDate);
if (LOG.isDebugEnabled()) {
LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier
+ " renewDate=" + renewDate);
}
}
} finally {
is.close();
@ -770,23 +781,20 @@ public class ZKRMStateStore extends RMStateStore {
String nodeCreatePath =
getNodePath(delegationTokensRootPath, DELEGATION_TOKEN_PREFIX
+ rmDTIdentifier.getSequenceNumber());
ByteArrayOutputStream tokenOs = new ByteArrayOutputStream();
DataOutputStream tokenOut = new DataOutputStream(tokenOs);
ByteArrayOutputStream seqOs = new ByteArrayOutputStream();
DataOutputStream seqOut = new DataOutputStream(seqOs);
RMDelegationTokenIdentifierData identifierData =
new RMDelegationTokenIdentifierData(rmDTIdentifier, renewDate);
try {
rmDTIdentifier.setRenewDate(renewDate);
rmDTIdentifier.write(tokenOut);
if (LOG.isDebugEnabled()) {
LOG.debug((isUpdate ? "Storing " : "Updating ") + "RMDelegationToken_" +
rmDTIdentifier.getSequenceNumber());
}
if (isUpdate) {
opList.add(Op.setData(nodeCreatePath, tokenOs.toByteArray(), -1));
opList.add(Op.setData(nodeCreatePath, identifierData.toByteArray(), -1));
} else {
opList.add(Op.create(nodeCreatePath, tokenOs.toByteArray(), zkAcl,
opList.add(Op.create(nodeCreatePath, identifierData.toByteArray(), zkAcl,
CreateMode.PERSISTENT));
}
@ -799,7 +807,6 @@ public class ZKRMStateStore extends RMStateStore {
opList.add(Op.setData(dtSequenceNumberPath, seqOs.toByteArray(), -1));
} finally {
tokenOs.close();
seqOs.close();
}
}

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/RMDelegationTokenIdentifierData.java Normal file
View File

@ -0,0 +1,61 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.recovery.records;
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.IOException;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.RMDelegationTokenIdentifierDataProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
public class RMDelegationTokenIdentifierData {
RMDelegationTokenIdentifierDataProto.Builder builder =
RMDelegationTokenIdentifierDataProto.newBuilder();
public RMDelegationTokenIdentifierData() {}
public RMDelegationTokenIdentifierData(
YARNDelegationTokenIdentifier identifier, long renewdate) {
builder.setTokenIdentifier(identifier.getProto());
builder.setRenewDate(renewdate);
}
public void readFields(DataInput in) throws IOException {
builder.mergeFrom((DataInputStream) in);
}
public byte[] toByteArray() throws IOException {
return builder.build().toByteArray();
}
public RMDelegationTokenIdentifier getTokenIdentifier() throws IOException {
ByteArrayInputStream in =
new ByteArrayInputStream(builder.getTokenIdentifier().toByteArray());
RMDelegationTokenIdentifier identifer = new RMDelegationTokenIdentifier();
identifer.readFields(new DataInputStream(in));
return identifer;
}
public long getRenewDate() {
return builder.getRenewDate();
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
View File

@ -24,6 +24,7 @@ package hadoop.yarn;
import "yarn_server_common_protos.proto";
import "yarn_protos.proto";
import "yarn_security_token.proto";
////////////////////////////////////////////////////////////////////////
////// RM recovery related records /////////////////////////////////////
@ -91,3 +92,8 @@ message AMRMTokenSecretManagerStateProto {
optional MasterKeyProto current_master_key = 1;
optional MasterKeyProto next_master_key = 2;
}
message RMDelegationTokenIdentifierDataProto {
optional YARNDelegationTokenIdentifierProto token_identifier = 1;
optional int64 renewDate = 2;
}

156
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java
View File

@ -20,155 +20,73 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.proto.YarnSecurityTestClientAMTokenProtos.RMDelegationTokenIdentifierForTestProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
public class RMDelegationTokenIdentifierForTest extends
RMDelegationTokenIdentifier {
private RMDelegationTokenIdentifierForTestProto proto;
private RMDelegationTokenIdentifierForTestProto.Builder builder;
private RMDelegationTokenIdentifierForTestProto.Builder builder =
RMDelegationTokenIdentifierForTestProto.newBuilder();
public RMDelegationTokenIdentifierForTest() {
}
public RMDelegationTokenIdentifierForTest(
RMDelegationTokenIdentifier token, String message) {
builder = RMDelegationTokenIdentifierForTestProto.newBuilder();
public RMDelegationTokenIdentifierForTest(RMDelegationTokenIdentifier token,
String message) {
if (token.getOwner() != null) {
builder.setOwner(token.getOwner().toString());
setOwner(new Text(token.getOwner()));
}
if (token.getRenewer() != null) {
builder.setRenewer(token.getRenewer().toString());
setRenewer(new Text(token.getRenewer()));
}
if (token.getRealUser() != null) {
builder.setRealUser(token.getRealUser().toString());
setRealUser(new Text(token.getRealUser()));
}
builder.setIssueDate(token.getIssueDate());
builder.setMaxDate(token.getMaxDate());
builder.setSequenceNumber(token.getSequenceNumber());
builder.setMasterKeyId(token.getMasterKeyId());
setIssueDate(token.getIssueDate());
setMaxDate(token.getMaxDate());
setSequenceNumber(token.getSequenceNumber());
setMasterKeyId(token.getMasterKeyId());
builder.setMessage(message);
proto = builder.build();
builder = null;
}
@Override
public void write(DataOutput out) throws IOException {
out.write(proto.toByteArray());
builder.setOwner(getOwner().toString());
builder.setRenewer(getRenewer().toString());
builder.setRealUser(getRealUser().toString());
builder.setIssueDate(getIssueDate());
builder.setMaxDate(getMaxDate());
builder.setSequenceNumber(getSequenceNumber());
builder.setMasterKeyId(getMasterKeyId());
builder.setMessage(getMessage());
builder.build().writeTo((DataOutputStream) out);
}
@Override
public void readFields(DataInput in) throws IOException {
DataInputStream dis = (DataInputStream)in;
byte[] buffer = IOUtils.toByteArray(dis);
proto = RMDelegationTokenIdentifierForTestProto.parseFrom(buffer);
}
/**
* Get the username encoded in the token identifier
*
* @return the username or owner
*/
@Override
public UserGroupInformation getUser() {
String owner = getOwner().toString();
String realUser = getRealUser().toString();
if ( (owner == null) || (owner.toString().isEmpty())) {
return null;
builder.mergeFrom((DataInputStream) in);
if (builder.getOwner() != null) {
setOwner(new Text(builder.getOwner()));
}
final UserGroupInformation realUgi;
final UserGroupInformation ugi;
if ((realUser == null) || (realUser.toString().isEmpty())
|| realUser.equals(owner)) {
ugi = realUgi = UserGroupInformation.createRemoteUser(owner.toString());
} else {
realUgi = UserGroupInformation.createRemoteUser(realUser.toString());
ugi = UserGroupInformation.createProxyUser(owner.toString(), realUgi);
if (builder.getRenewer() != null) {
setRenewer(new Text(builder.getRenewer()));
}
realUgi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
return ugi;
}
public Text getOwner() {
String owner = proto.getOwner();
if (owner == null) {
return null;
} else {
return new Text(owner);
if (builder.getRealUser() != null) {
setRealUser(new Text(builder.getRealUser()));
}
}
public Text getRenewer() {
String renewer = proto.getRenewer();
if (renewer == null) {
return null;
} else {
return new Text(renewer);
}
}
public Text getRealUser() {
String realUser = proto.getRealUser();
if (realUser == null) {
return null;
} else {
return new Text(realUser);
}
}
public void setIssueDate(long issueDate) {
RMDelegationTokenIdentifierForTestProto.Builder builder =
RMDelegationTokenIdentifierForTestProto.newBuilder(proto);
builder.setIssueDate(issueDate);
proto = builder.build();
}
public long getIssueDate() {
return proto.getIssueDate();
}
public void setMaxDate(long maxDate) {
RMDelegationTokenIdentifierForTestProto.Builder builder =
RMDelegationTokenIdentifierForTestProto.newBuilder(proto);
builder.setMaxDate(maxDate);
proto = builder.build();
}
public long getMaxDate() {
return proto.getMaxDate();
}
public void setSequenceNumber(int seqNum) {
RMDelegationTokenIdentifierForTestProto.Builder builder =
RMDelegationTokenIdentifierForTestProto.newBuilder(proto);
builder.setSequenceNumber(seqNum);
proto = builder.build();
}
public int getSequenceNumber() {
return proto.getSequenceNumber();
}
public void setMasterKeyId(int newId) {
RMDelegationTokenIdentifierForTestProto.Builder builder =
RMDelegationTokenIdentifierForTestProto.newBuilder(proto);
builder.setMasterKeyId(newId);
proto = builder.build();
}
public int getMasterKeyId() {
return proto.getMasterKeyId();
setIssueDate(builder.getIssueDate());
setMaxDate(builder.getMaxDate());
setSequenceNumber(builder.getSequenceNumber());
setMasterKeyId(builder.getMasterKeyId());
}
public String getMessage() {
return proto.getMessage();
return builder.getMessage();
}
@Override
@ -189,10 +107,4 @@ public class RMDelegationTokenIdentifierForTest extends
}
return false;
}
@Override
public int hashCode() {
return this.getSequenceNumber();
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
View File

@ -28,6 +28,7 @@ import static org.mockito.Mockito.when;
import static org.mockito.Mockito.spy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@ -402,6 +403,7 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
RMDelegationTokenIdentifier dtId1 =
new RMDelegationTokenIdentifier(new Text("owner1"),
new Text("renewer1"), new Text("realuser1"));
byte[] tokenBeforeStore = dtId1.getBytes();
Long renewDate1 = new Long(System.currentTimeMillis());
int sequenceNumber = 1111;
store.storeRMDelegationTokenAndSequenceNumber(dtId1, renewDate1,
@ -423,6 +425,10 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
Assert.assertEquals(keySet, secretManagerState.getMasterKeyState());
Assert.assertEquals(sequenceNumber,
secretManagerState.getDTSequenceNumber());
RMDelegationTokenIdentifier tokenAfterStore =
secretManagerState.getTokenState().keySet().iterator().next();
Assert.assertTrue(Arrays.equals(tokenBeforeStore,
tokenAfterStore.getBytes()));
// update RM delegation token;
renewDate1 = new Long(System.currentTimeMillis());