Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-15059. Undoing the switch of Credentials to PB format as default - done via HADOOP-12563 for supporting 2.x to 3.x upgrades.

This commit is contained in:
Vinod Kumar Vavilapalli (I am also known as @tshooter.) 2017-12-08 08:00:21 -08:00
parent ce04340ec7
commit f19638333b
4 changed files with 87 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
View File

@ -27,7 +27,6 @@
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@ -60,6 +59,28 @@
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class Credentials implements Writable {
public enum SerializedFormat {
WRITABLE((byte) 0x00),
PROTOBUF((byte) 0x01);
// Caching to avoid reconstructing the array each time.
private static final SerializedFormat[] FORMATS = values();
final byte value;
SerializedFormat(byte val) {
this.value = val;
}
public static SerializedFormat valueOf(int val) {
try {
return FORMATS[val];
} catch (ArrayIndexOutOfBoundsException e) {
throw new IllegalArgumentException("Unknown credential format: " + val);
}
}
}
private static final Logger LOG = LoggerFactory.getLogger(Credentials.class);
private Map<Text, byte[]> secretKeysMap = new HashMap<Text, byte[]>();
@ -224,63 +245,74 @@ public void readTokenStorageStream(DataInputStream in) throws IOException {
if (!Arrays.equals(magic, TOKEN_STORAGE_MAGIC)) {
throw new IOException("Bad header found in token storage.");
}
byte version = in.readByte();
if (version != TOKEN_STORAGE_VERSION &&
version != OLD_TOKEN_STORAGE_VERSION) {
throw new IOException("Unknown version " + version +
" in token storage.");
SerializedFormat format;
try {
format = SerializedFormat.valueOf(in.readByte());
} catch (IllegalArgumentException e) {
throw new IOException(e);
}
if (version == OLD_TOKEN_STORAGE_VERSION) {
switch (format) {
case WRITABLE:
readFields(in);
} else if (version == TOKEN_STORAGE_VERSION) {
break;
case PROTOBUF:
readProto(in);
break;
default:
throw new IOException("Unsupported format " + format);
}
}
private static final byte[] TOKEN_STORAGE_MAGIC =
"HDTS".getBytes(StandardCharsets.UTF_8);
private static final byte TOKEN_STORAGE_VERSION = 1;
/**
* For backward compatibility.
*/
private static final byte OLD_TOKEN_STORAGE_VERSION = 0;
public void writeTokenStorageToStream(DataOutputStream os)
throws IOException {
// by default store in the oldest supported format for compatibility
writeTokenStorageToStream(os, SerializedFormat.WRITABLE);
}
public void writeTokenStorageToStream(DataOutputStream os,
SerializedFormat format) throws IOException {
switch (format) {
case WRITABLE:
writeWritableOutputStream(os);
break;
case PROTOBUF:
writeProtobufOutputStream(os);
break;
default:
throw new IllegalArgumentException("Unsupported serialized format: "
+ format);
}
}
private void writeWritableOutputStream(DataOutputStream os)
throws IOException {
os.write(TOKEN_STORAGE_MAGIC);
os.write(TOKEN_STORAGE_VERSION);
os.write(SerializedFormat.WRITABLE.value);
write(os);
}
private void writeProtobufOutputStream(DataOutputStream os)
throws IOException {
os.write(TOKEN_STORAGE_MAGIC);
os.write(SerializedFormat.PROTOBUF.value);
writeProto(os);
}
public void writeTokenStorageFile(Path filename,
Configuration conf) throws IOException {
FSDataOutputStream os = filename.getFileSystem(conf).create(filename);
writeTokenStorageToStream(os);
os.close();
// by default store in the oldest supported format for compatibility
writeTokenStorageFile(filename, conf, SerializedFormat.WRITABLE);
}
/**
* For backward compatibility.
*/
public void writeLegacyTokenStorageLocalFile(File f) throws IOException {
writeLegacyOutputStream(new DataOutputStream(new FileOutputStream(f)));
}
/**
* For backward compatibility.
*/
public void writeLegacyTokenStorageFile(Path filename, Configuration conf)
throws IOException {
writeLegacyOutputStream(filename.getFileSystem(conf).create(filename));
}
private void writeLegacyOutputStream(DataOutputStream os) throws IOException {
os.write(TOKEN_STORAGE_MAGIC);
os.write(OLD_TOKEN_STORAGE_VERSION);
write(os);
os.close();
public void writeTokenStorageFile(Path filename, Configuration conf,
SerializedFormat format) throws IOException {
try (FSDataOutputStream os =
filename.getFileSystem(conf).create(filename)) {
writeTokenStorageToStream(os, format);
}
}
/**
@ -312,7 +344,7 @@ public void write(DataOutput out) throws IOException {
* @param out
* @throws IOException
*/
public void writeProto(DataOutput out) throws IOException {
void writeProto(DataOutput out) throws IOException {
CredentialsProto.Builder storage = CredentialsProto.newBuilder();
for (Map.Entry<Text, Token<? extends TokenIdentifier>> e :
tokenMap.entrySet()) {
@ -337,7 +369,7 @@ public void writeProto(DataOutput out) throws IOException {
* Populates keys/values from proto buffer storage.
* @param in - stream ready to read a serialized proto buffer message
*/
public void readProto(DataInput in) throws IOException {
void readProto(DataInput in) throws IOException {
CredentialsProto storage = CredentialsProto.parseDelimitedFrom((DataInputStream)in);
for (CredentialsKVProto kv : storage.getTokensList()) {
addToken(new Text(kv.getAliasBytes().toByteArray()),

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
View File

@ -102,11 +102,13 @@ private static Path fileToPath(File f) {
public static void doFormattedWrite(
File f, String format, Credentials creds, Configuration conf)
throws IOException {
if (format == null || format.equals(FORMAT_PB)) {
creds.writeTokenStorageFile(fileToPath(f), conf);
} else { // if (format != null && format.equals(FORMAT_JAVA)) {
creds.writeLegacyTokenStorageLocalFile(f);
// default to oldest supported format for compatibility
Credentials.SerializedFormat credsFormat =
Credentials.SerializedFormat.WRITABLE;
if (format.equals(FORMAT_PB)) {
credsFormat = Credentials.SerializedFormat.PROTOBUF;
}
creds.writeTokenStorageFile(fileToPath(f), conf, credsFormat);
}
/** Print out a Credentials file from the local filesystem.

15
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
View File

@ -20,7 +20,6 @@
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
@ -29,14 +28,12 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.security.token.DtFetcher;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -54,7 +51,6 @@ public class TestDtUtilShell {
private static Configuration defaultConf = new Configuration();
private static FileSystem localFs = null;
private final String alias = "proxy_ip:1234";
private final String renewer = "yarn";
private final String getUrl = SERVICE_GET.toString() + "://localhost:9000/";
private final String getUrl2 = "http://localhost:9000/";
public static Text SERVICE_GET = new Text("testTokenServiceGet");
@ -111,11 +107,12 @@ public void makeTokenFile(Path tokenPath, boolean legacy, Text service)
Token<? extends TokenIdentifier> tok = (Token<? extends TokenIdentifier>)
new Token(IDENTIFIER, PASSWORD, KIND, service);
creds.addToken(tok.getService(), tok);
Credentials.SerializedFormat format =
Credentials.SerializedFormat.PROTOBUF;
if (legacy) {
creds.writeLegacyTokenStorageLocalFile(new File(tokenPath.toString()));
} else {
creds.writeTokenStorageFile(tokenPath, defaultConf);
format = Credentials.SerializedFormat.WRITABLE;
}
creds.writeTokenStorageFile(tokenPath, defaultConf, format);
}
@Test
@ -284,6 +281,6 @@ public void testFormatProtoFlag() throws Exception {
DataInputStream in = new DataInputStream(
new FileInputStream(tokenFilenameGet));
spyCreds.readTokenStorageStream(in);
Mockito.verify(spyCreds).readProto(in);
Mockito.verify(spyCreds, Mockito.never()).readFields(in);
}
}

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
View File

@ -182,7 +182,8 @@ static void saveDelegationToken(Configuration conf, FileSystem fs,
Credentials cred = new Credentials();
cred.addToken(token.getService(), token);
// dtutil is replacing this tool; preserve legacy functionality
cred.writeLegacyTokenStorageFile(tokenFile, conf);
cred.writeTokenStorageFile(tokenFile, conf,
Credentials.SerializedFormat.WRITABLE);
if (LOG.isDebugEnabled()) {
LOG.debug("Fetched token " + fs.getUri() + " for " +