HADOOP-6861. Adds new non-static methods in Credentials to read and write token storage file. Contributed by Jitendra Pandey & Owen O'Malley.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@966911 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
bd802429c1
commit
eb10b152fe
|
@ -80,6 +80,9 @@ Trunk (unreleased changes)
|
|||
same principal. Now the principal name is a pattern that has _HOST in it.
|
||||
(Kan Zhang & Jitendra Pandey via ddas)
|
||||
|
||||
HADOOP-6861. Adds new non-static methods in Credentials to read and
|
||||
write token storage file. (Jitendra Pandey & Owen O'Malley via ddas)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
@ -57,7 +57,8 @@ public final class WritableUtils {
|
|||
}
|
||||
}
|
||||
|
||||
public static int writeCompressedByteArray(DataOutput out, byte[] bytes) throws IOException {
|
||||
public static int writeCompressedByteArray(DataOutput out,
|
||||
byte[] bytes) throws IOException {
|
||||
if (bytes != null) {
|
||||
ByteArrayOutputStream bos = new ByteArrayOutputStream();
|
||||
GZIPOutputStream gzout = new GZIPOutputStream(bos);
|
||||
|
|
|
@ -19,13 +19,17 @@
|
|||
package org.apache.hadoop.security;
|
||||
|
||||
import java.io.DataInput;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutput;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.io.Text;
|
||||
|
@ -114,24 +118,59 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Convenience method for reading a file, and loading the Tokens
|
||||
* Convenience method for reading a token storage file, and loading the Tokens
|
||||
* therein in the passed UGI
|
||||
* @param filename
|
||||
* @param conf
|
||||
* @param ugi
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void readTokensAndLoadInUGI(String filename, Configuration conf,
|
||||
UserGroupInformation ugi) throws IOException {
|
||||
Path localTokensFile = new Path (filename);
|
||||
FileSystem localFS = FileSystem.getLocal(conf);
|
||||
FSDataInputStream in = localFS.open(localTokensFile);
|
||||
Credentials ts = new Credentials();
|
||||
ts.readFields(in);
|
||||
for (Token<? extends TokenIdentifier> token : ts.getAllTokens()) {
|
||||
ugi.addToken(token);
|
||||
public void readTokenStorageFile(Path filename,
|
||||
Configuration conf) throws IOException {
|
||||
FSDataInputStream in = filename.getFileSystem(conf).open(filename);
|
||||
try {
|
||||
readTokenStorageStream(in);
|
||||
} catch(IOException ioe) {
|
||||
throw new IOException("Exception reading " + filename, ioe);
|
||||
} finally {
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience method for reading a token storage file directly from a
|
||||
* datainputstream
|
||||
*/
|
||||
public void readTokenStorageStream(DataInputStream in) throws IOException {
|
||||
byte[] magic = new byte[TOKEN_STORAGE_MAGIC.length];
|
||||
in.readFully(magic);
|
||||
if (!Arrays.equals(magic, TOKEN_STORAGE_MAGIC)) {
|
||||
throw new IOException("Bad header found in token storage.");
|
||||
}
|
||||
byte version = in.readByte();
|
||||
if (version != TOKEN_STORAGE_VERSION) {
|
||||
throw new IOException("Unknown version " + version +
|
||||
" in token storage.");
|
||||
}
|
||||
readFields(in);
|
||||
}
|
||||
|
||||
private static final byte[] TOKEN_STORAGE_MAGIC = "HDTS".getBytes();
|
||||
private static final byte TOKEN_STORAGE_VERSION = 0;
|
||||
|
||||
public void writeTokenStorageToStream(DataOutputStream os)
|
||||
throws IOException {
|
||||
os.write(TOKEN_STORAGE_MAGIC);
|
||||
os.write(TOKEN_STORAGE_VERSION);
|
||||
write(os);
|
||||
}
|
||||
|
||||
public void writeTokenStorageFile(Path filename,
|
||||
Configuration conf) throws IOException {
|
||||
FSDataOutputStream os = filename.getFileSystem(conf).create(filename);
|
||||
writeTokenStorageToStream(os);
|
||||
os.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores all the keys to DataOutput
|
||||
* @param out
|
||||
|
@ -151,7 +190,8 @@ public class Credentials implements Writable {
|
|||
WritableUtils.writeVInt(out, secretKeysMap.size());
|
||||
for(Map.Entry<Text, byte[]> e : secretKeysMap.entrySet()) {
|
||||
e.getKey().write(out);
|
||||
WritableUtils.writeCompressedByteArray(out, e.getValue());
|
||||
WritableUtils.writeVInt(out, e.getValue().length);
|
||||
out.write(e.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -178,8 +218,23 @@ public class Credentials implements Writable {
|
|||
for(int i=0; i<size; i++) {
|
||||
Text alias = new Text();
|
||||
alias.readFields(in);
|
||||
byte[] key = WritableUtils.readCompressedByteArray(in);
|
||||
secretKeysMap.put(alias, key);
|
||||
int len = WritableUtils.readVInt(in);
|
||||
byte[] value = new byte[len];
|
||||
in.readFully(value);
|
||||
secretKeysMap.put(alias, value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy all of the credentials from one credential object into another.
|
||||
* @param other the credentials to copy
|
||||
*/
|
||||
public void addAll(Credentials other) {
|
||||
for(Map.Entry<Text, byte[]> secret: other.secretKeysMap.entrySet()) {
|
||||
secretKeysMap.put(secret.getKey(), secret.getValue());
|
||||
}
|
||||
for(Map.Entry<Text, Token<?>> token: other.tokenMap.entrySet()){
|
||||
tokenMap.put(token.getKey(), token.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -50,6 +50,7 @@ import org.apache.commons.logging.LogFactory;
|
|||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
import org.apache.hadoop.security.token.TokenIdentifier;
|
||||
|
||||
|
@ -135,6 +136,9 @@ public class UserGroupInformation {
|
|||
private static boolean useKerberos;
|
||||
/** Server-side groups fetching service */
|
||||
private static Groups groups;
|
||||
/** The configuration to use */
|
||||
private static Configuration conf;
|
||||
|
||||
|
||||
public static final long MIN_TIME_BEFORE_RELOGIN = 10 * 60 * 1000L;
|
||||
|
||||
|
@ -188,6 +192,7 @@ public class UserGroupInformation {
|
|||
"configuration", ioe);
|
||||
}
|
||||
isInitialized = true;
|
||||
UserGroupInformation.conf = conf;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -398,9 +403,15 @@ public class UserGroupInformation {
|
|||
login.login();
|
||||
loginUser.setLogin(login);
|
||||
loginUser = new UserGroupInformation(login.getSubject());
|
||||
String tokenFile = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
|
||||
if (tokenFile != null && isSecurityEnabled()) {
|
||||
Credentials.readTokensAndLoadInUGI(tokenFile, new Configuration(), loginUser);
|
||||
String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
|
||||
if (fileLocation != null && isSecurityEnabled()) {
|
||||
// load the token storage file and put all of the tokens into the
|
||||
// user.
|
||||
Credentials cred = new Credentials();
|
||||
cred.readTokenStorageFile(new Path("file:///" + fileLocation), conf);
|
||||
for (Token<?> token: cred.getAllTokens()) {
|
||||
loginUser.addToken(token);
|
||||
}
|
||||
}
|
||||
} catch (LoginException le) {
|
||||
throw new IOException("failure to login", le);
|
||||
|
|
|
@ -317,7 +317,9 @@ public class GenericOptionsParser {
|
|||
throw new FileNotFoundException("File "+fileName+" does not exist.");
|
||||
}
|
||||
LOG.debug("setting conf tokensFile: " + fileName);
|
||||
conf.set("tokenCacheFile", localFs.makeQualified(p).toString());
|
||||
conf.set("mapreduce.job.credentials.json", localFs.makeQualified(p)
|
||||
.toString());
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -126,7 +126,7 @@ public class TestGenericOptionsParser extends TestCase {
|
|||
Path tmpPath = new Path(tmpFile.toString());
|
||||
localFs.create(tmpPath);
|
||||
new GenericOptionsParser(conf, args);
|
||||
String fileName = conf.get("tokenCacheFile");
|
||||
String fileName = conf.get("mapreduce.job.credentials.json");
|
||||
assertNotNull("files is null", fileName);
|
||||
assertEquals("files option does not match",
|
||||
localFs.makeQualified(tmpPath).toString(), fileName);
|
||||
|
|
Loading…
Reference in New Issue