HDFS-12300. Audit-log delegation token related operations.
This commit is contained in:
parent
36f33a1efb
commit
1b3b9938cf
|
@ -36,6 +36,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_
|
||||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
|
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
|
||||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
|
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.DataInputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintStream;
|
import java.io.PrintStream;
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
|
@ -70,6 +72,7 @@ import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||||
|
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
||||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.NameNode;
|
import org.apache.hadoop.hdfs.server.namenode.NameNode;
|
||||||
import org.apache.hadoop.http.HttpConfig;
|
import org.apache.hadoop.http.HttpConfig;
|
||||||
|
@ -80,6 +83,7 @@ import org.apache.hadoop.net.NetUtils;
|
||||||
import org.apache.hadoop.security.SecurityUtil;
|
import org.apache.hadoop.security.SecurityUtil;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||||
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
@ -1570,4 +1574,22 @@ public class DFSUtil {
|
||||||
.createKeyProviderCryptoExtension(keyProvider);
|
.createKeyProviderCryptoExtension(keyProvider);
|
||||||
return cryptoProvider;
|
return cryptoProvider;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decodes an HDFS delegation token to its identifier.
|
||||||
|
*
|
||||||
|
* @param token the token
|
||||||
|
* @return the decoded identifier.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static DelegationTokenIdentifier decodeDelegationToken(
|
||||||
|
final Token<DelegationTokenIdentifier> token) throws IOException {
|
||||||
|
final DelegationTokenIdentifier id = new DelegationTokenIdentifier();
|
||||||
|
final ByteArrayInputStream buf =
|
||||||
|
new ByteArrayInputStream(token.getIdentifier());
|
||||||
|
try (DataInputStream in = new DataInputStream(buf)) {
|
||||||
|
id.readFields(in);
|
||||||
|
}
|
||||||
|
return id;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -101,9 +101,7 @@ import static org.apache.hadoop.util.Time.monotonicNow;
|
||||||
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
|
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
|
||||||
|
|
||||||
import java.io.BufferedWriter;
|
import java.io.BufferedWriter;
|
||||||
import java.io.ByteArrayInputStream;
|
|
||||||
import java.io.DataInput;
|
import java.io.DataInput;
|
||||||
import java.io.DataInputStream;
|
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
|
@ -5399,6 +5397,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
||||||
*/
|
*/
|
||||||
Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
|
Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
final String operationName = "getDelegationToken";
|
||||||
|
final boolean success;
|
||||||
|
final String tokenId;
|
||||||
Token<DelegationTokenIdentifier> token;
|
Token<DelegationTokenIdentifier> token;
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
writeLock();
|
writeLock();
|
||||||
|
@ -5427,10 +5428,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
||||||
dtId, dtSecretManager);
|
dtId, dtSecretManager);
|
||||||
long expiryTime = dtSecretManager.getTokenExpiryTime(dtId);
|
long expiryTime = dtSecretManager.getTokenExpiryTime(dtId);
|
||||||
getEditLog().logGetDelegationToken(dtId, expiryTime);
|
getEditLog().logGetDelegationToken(dtId, expiryTime);
|
||||||
|
tokenId = dtId.toStringStable();
|
||||||
|
success = true;
|
||||||
} finally {
|
} finally {
|
||||||
writeUnlock("getDelegationToken");
|
writeUnlock("getDelegationToken");
|
||||||
}
|
}
|
||||||
getEditLog().logSync();
|
getEditLog().logSync();
|
||||||
|
logAuditEvent(success, operationName, tokenId);
|
||||||
return token;
|
return token;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5443,6 +5447,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
||||||
*/
|
*/
|
||||||
long renewDelegationToken(Token<DelegationTokenIdentifier> token)
|
long renewDelegationToken(Token<DelegationTokenIdentifier> token)
|
||||||
throws InvalidToken, IOException {
|
throws InvalidToken, IOException {
|
||||||
|
final String operationName = "renewDelegationToken";
|
||||||
|
boolean success = false;
|
||||||
|
String tokenId;
|
||||||
long expiryTime;
|
long expiryTime;
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
writeLock();
|
writeLock();
|
||||||
|
@ -5456,15 +5463,20 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
||||||
}
|
}
|
||||||
String renewer = getRemoteUser().getShortUserName();
|
String renewer = getRemoteUser().getShortUserName();
|
||||||
expiryTime = dtSecretManager.renewToken(token, renewer);
|
expiryTime = dtSecretManager.renewToken(token, renewer);
|
||||||
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
|
final DelegationTokenIdentifier id = DFSUtil.decodeDelegationToken(token);
|
||||||
ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
|
|
||||||
DataInputStream in = new DataInputStream(buf);
|
|
||||||
id.readFields(in);
|
|
||||||
getEditLog().logRenewDelegationToken(id, expiryTime);
|
getEditLog().logRenewDelegationToken(id, expiryTime);
|
||||||
|
tokenId = id.toStringStable();
|
||||||
|
success = true;
|
||||||
|
} catch (AccessControlException ace) {
|
||||||
|
final DelegationTokenIdentifier id = DFSUtil.decodeDelegationToken(token);
|
||||||
|
tokenId = id.toStringStable();
|
||||||
|
logAuditEvent(success, operationName, tokenId);
|
||||||
|
throw ace;
|
||||||
} finally {
|
} finally {
|
||||||
writeUnlock("renewDelegationToken");
|
writeUnlock("renewDelegationToken");
|
||||||
}
|
}
|
||||||
getEditLog().logSync();
|
getEditLog().logSync();
|
||||||
|
logAuditEvent(success, operationName, tokenId);
|
||||||
return expiryTime;
|
return expiryTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5475,6 +5487,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
||||||
*/
|
*/
|
||||||
void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
|
void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
final String operationName = "cancelDelegationToken";
|
||||||
|
boolean success = false;
|
||||||
|
String tokenId;
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
|
@ -5485,10 +5500,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
||||||
DelegationTokenIdentifier id = dtSecretManager
|
DelegationTokenIdentifier id = dtSecretManager
|
||||||
.cancelToken(token, canceller);
|
.cancelToken(token, canceller);
|
||||||
getEditLog().logCancelDelegationToken(id);
|
getEditLog().logCancelDelegationToken(id);
|
||||||
|
tokenId = id.toStringStable();
|
||||||
|
success = true;
|
||||||
|
} catch (AccessControlException ace) {
|
||||||
|
final DelegationTokenIdentifier id = DFSUtil.decodeDelegationToken(token);
|
||||||
|
tokenId = id.toStringStable();
|
||||||
|
logAuditEvent(success, operationName, tokenId);
|
||||||
|
throw ace;
|
||||||
} finally {
|
} finally {
|
||||||
writeUnlock("cancelDelegationToken");
|
writeUnlock("cancelDelegationToken");
|
||||||
}
|
}
|
||||||
getEditLog().logSync();
|
getEditLog().logSync();
|
||||||
|
logAuditEvent(success, operationName, tokenId);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -34,13 +34,16 @@ import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
||||||
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
|
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
|
||||||
import org.apache.hadoop.security.AccessControlException;
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
|
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.security.PrivilegedExceptionAction;
|
||||||
|
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
import org.junit.AfterClass;
|
import org.junit.AfterClass;
|
||||||
import org.junit.Ignore;
|
import org.junit.Ignore;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
import org.mockito.Mock;
|
import org.mockito.Mock;
|
||||||
|
@ -68,6 +71,7 @@ public class TestAuditLoggerWithCommands {
|
||||||
conf = new HdfsConfiguration();
|
conf = new HdfsConfiguration();
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
|
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,true);
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,true);
|
||||||
|
conf.setBoolean(DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
|
||||||
cluster =
|
cluster =
|
||||||
new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
|
new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
@ -566,6 +570,54 @@ public class TestAuditLoggerWithCommands {
|
||||||
cluster.getNamesystem().setFSDirectory(dir);
|
cluster.getNamesystem().setFSDirectory(dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDelegationTokens() throws Exception {
|
||||||
|
Token dt = fs.getDelegationToken("foo");
|
||||||
|
final String getDT =
|
||||||
|
".*src=HDFS_DELEGATION_TOKEN token 1.*with renewer foo.*";
|
||||||
|
verifyAuditLogs(true, ".*cmd=getDelegationToken" + getDT);
|
||||||
|
|
||||||
|
// renew
|
||||||
|
final UserGroupInformation foo =
|
||||||
|
UserGroupInformation.createUserForTesting("foo", new String[] {});
|
||||||
|
foo.doAs(new PrivilegedExceptionAction<Void>() {
|
||||||
|
@Override
|
||||||
|
public Void run() throws Exception {
|
||||||
|
dt.renew(conf);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
verifyAuditLogs(true, ".*cmd=renewDelegationToken" + getDT);
|
||||||
|
try {
|
||||||
|
dt.renew(conf);
|
||||||
|
fail("Renewing a token with non-renewer should fail");
|
||||||
|
} catch (AccessControlException expected) {
|
||||||
|
}
|
||||||
|
verifyAuditLogs(false, ".*cmd=renewDelegationToken" + getDT);
|
||||||
|
|
||||||
|
// cancel
|
||||||
|
final UserGroupInformation bar =
|
||||||
|
UserGroupInformation.createUserForTesting("bar", new String[] {});
|
||||||
|
try {
|
||||||
|
bar.doAs(new PrivilegedExceptionAction<Void>() {
|
||||||
|
@Override
|
||||||
|
public Void run() throws Exception {
|
||||||
|
dt.cancel(conf);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
fail("Canceling a token with non-renewer should fail");
|
||||||
|
} catch (AccessControlException expected) {
|
||||||
|
}
|
||||||
|
verifyAuditLogs(false, ".*cmd=cancelDelegationToken" + getDT);
|
||||||
|
dt.cancel(conf);
|
||||||
|
verifyAuditLogs(true, ".*cmd=cancelDelegationToken" + getDT);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int verifyAuditLogs(final boolean allowed, final String pattern) {
|
||||||
|
return verifyAuditLogs(".*allowed=" + allowed + pattern);
|
||||||
|
}
|
||||||
|
|
||||||
private int verifyAuditLogs(String pattern) {
|
private int verifyAuditLogs(String pattern) {
|
||||||
int length = auditlog.getOutput().split("\n").length;
|
int length = auditlog.getOutput().split("\n").length;
|
||||||
String lastAudit = auditlog.getOutput().split("\n")[length - 1];
|
String lastAudit = auditlog.getOutput().split("\n")[length - 1];
|
||||||
|
|
Loading…
Reference in New Issue