HADOOP-10430. KeyProvider Metadata should have an optional description, there should be a method to retrieve the metadata from all keys. (tucu)

Conflicts:
	hadoop-common-project/hadoop-common/CHANGES.txt

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1619515 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2014-08-21 18:58:42 +00:00
parent 32134d7386
commit bd3cff6027
7 changed files with 137 additions and 12 deletions

View File

@ -105,6 +105,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10428. JavaKeyStoreProvider should accept keystore password via HADOOP-10428. JavaKeyStoreProvider should accept keystore password via
configuration falling back to ENV VAR. (tucu) configuration falling back to ENV VAR. (tucu)
HADOOP-10430. KeyProvider Metadata should have an optional description,
there should be a method to retrieve the metadata from all keys. (tucu)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd) HADOOP-10838. Byte array native checksumming. (James Thomas via todd)

View File

@ -268,7 +268,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
e); e);
} }
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(), Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
new Date(), 1); options.getDescription(), new Date(), 1);
if (options.getBitLength() != 8 * material.length) { if (options.getBitLength() != 8 * material.length) {
throw new IOException("Wrong key length. Required " + throw new IOException("Wrong key length. Required " +
options.getBitLength() + ", but got " + (8 * material.length)); options.getBitLength() + ", but got " + (8 * material.length));

View File

@ -25,8 +25,11 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.net.URI; import java.net.URI;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat;
import java.util.Date; import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter; import com.google.gson.stream.JsonWriter;
@ -104,21 +107,34 @@ public abstract class KeyProvider {
private final static String CIPHER_FIELD = "cipher"; private final static String CIPHER_FIELD = "cipher";
private final static String BIT_LENGTH_FIELD = "bitLength"; private final static String BIT_LENGTH_FIELD = "bitLength";
private final static String CREATED_FIELD = "created"; private final static String CREATED_FIELD = "created";
private final static String DESCRIPTION_FIELD = "description";
private final static String VERSIONS_FIELD = "versions"; private final static String VERSIONS_FIELD = "versions";
private final String cipher; private final String cipher;
private final int bitLength; private final int bitLength;
private final String description;
private final Date created; private final Date created;
private int versions; private int versions;
protected Metadata(String cipher, int bitLength, protected Metadata(String cipher, int bitLength,
Date created, int versions) { String description, Date created, int versions) {
this.cipher = cipher; this.cipher = cipher;
this.bitLength = bitLength; this.bitLength = bitLength;
this.description = description;
this.created = created; this.created = created;
this.versions = versions; this.versions = versions;
} }
public String toString() {
return MessageFormat.format(
"cipher: {0}, length: {1} description: {2} created: {3} version: {4}",
cipher, bitLength, description, created, versions);
}
public String getDescription() {
return description;
}
public Date getCreated() { public Date getCreated() {
return created; return created;
} }
@ -170,6 +186,9 @@ public abstract class KeyProvider {
if (created != null) { if (created != null) {
writer.name(CREATED_FIELD).value(created.getTime()); writer.name(CREATED_FIELD).value(created.getTime());
} }
if (description != null) {
writer.name(DESCRIPTION_FIELD).value(description);
}
writer.name(VERSIONS_FIELD).value(versions); writer.name(VERSIONS_FIELD).value(versions);
writer.endObject(); writer.endObject();
writer.flush(); writer.flush();
@ -186,6 +205,7 @@ public abstract class KeyProvider {
int bitLength = 0; int bitLength = 0;
Date created = null; Date created = null;
int versions = 0; int versions = 0;
String description = null;
JsonReader reader = new JsonReader(new InputStreamReader JsonReader reader = new JsonReader(new InputStreamReader
(new ByteArrayInputStream(bytes))); (new ByteArrayInputStream(bytes)));
reader.beginObject(); reader.beginObject();
@ -199,12 +219,15 @@ public abstract class KeyProvider {
created = new Date(reader.nextLong()); created = new Date(reader.nextLong());
} else if (VERSIONS_FIELD.equals(field)) { } else if (VERSIONS_FIELD.equals(field)) {
versions = reader.nextInt(); versions = reader.nextInt();
} else if (DESCRIPTION_FIELD.equals(field)) {
description = reader.nextString();
} }
} }
reader.endObject(); reader.endObject();
this.cipher = cipher; this.cipher = cipher;
this.bitLength = bitLength; this.bitLength = bitLength;
this.created = created; this.created = created;
this.description = description;
this.versions = versions; this.versions = versions;
} }
} }
@ -215,6 +238,7 @@ public abstract class KeyProvider {
public static class Options { public static class Options {
private String cipher; private String cipher;
private int bitLength; private int bitLength;
private String description;
public Options(Configuration conf) { public Options(Configuration conf) {
cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER); cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER);
@ -231,6 +255,11 @@ public abstract class KeyProvider {
return this; return this;
} }
public Options setDescription(String description) {
this.description = description;
return this;
}
protected String getCipher() { protected String getCipher() {
return cipher; return cipher;
} }
@ -238,6 +267,10 @@ public abstract class KeyProvider {
protected int getBitLength() { protected int getBitLength() {
return bitLength; return bitLength;
} }
protected String getDescription() {
return description;
}
} }
/** /**
@ -277,6 +310,24 @@ public abstract class KeyProvider {
*/ */
public abstract List<String> getKeys() throws IOException; public abstract List<String> getKeys() throws IOException;
/**
* Get the key metadata for all keys.
*
* @return a Map with all the keys and their metadata
* @throws IOException
*/
public Map<String, Metadata> getKeysMetadata() throws IOException {
Map<String, Metadata> keysMetadata = new LinkedHashMap<String, Metadata>();
for (String key : getKeys()) {
Metadata meta = getMetadata(key);
if (meta != null) {
keysMetadata.put(key, meta);
}
}
return keysMetadata;
}
/** /**
* Get the key material for all versions of a specific key name. * Get the key material for all versions of a specific key name.
* @return the list of key material * @return the list of key material

View File

@ -23,6 +23,7 @@ import java.io.PrintStream;
import java.security.InvalidParameterException; import java.security.InvalidParameterException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.List; import java.util.List;
import java.util.Map;
import javax.crypto.KeyGenerator; import javax.crypto.KeyGenerator;
@ -45,6 +46,7 @@ public class KeyShell extends Configured implements Tool {
" [" + RollCommand.USAGE + "]\n" + " [" + RollCommand.USAGE + "]\n" +
" [" + DeleteCommand.USAGE + "]\n" + " [" + DeleteCommand.USAGE + "]\n" +
" [" + ListCommand.USAGE + "]\n"; " [" + ListCommand.USAGE + "]\n";
private static final String LIST_METADATA = "keyShell.list.metadata";
private boolean interactive = false; private boolean interactive = false;
private Command command = null; private Command command = null;
@ -121,6 +123,8 @@ public class KeyShell extends Configured implements Tool {
} else if (args[i].equals("--provider")) { } else if (args[i].equals("--provider")) {
userSuppliedProvider = true; userSuppliedProvider = true;
getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]); getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
} else if (args[i].equals("--metadata")) {
getConf().setBoolean(LIST_METADATA, true);
} else if (args[i].equals("-i") || (args[i].equals("--interactive"))) { } else if (args[i].equals("-i") || (args[i].equals("--interactive"))) {
interactive = true; interactive = true;
} else if (args[i].equals("--help")) { } else if (args[i].equals("--help")) {
@ -201,11 +205,15 @@ public class KeyShell extends Configured implements Tool {
} }
private class ListCommand extends Command { private class ListCommand extends Command {
public static final String USAGE = "list <keyname> [--provider] [--help]"; public static final String USAGE =
"list [--provider] [--metadata] [--help]";
public static final String DESC = public static final String DESC =
"The list subcommand displays the keynames contained within \n" + "The list subcommand displays the keynames contained within \n" +
"a particular provider - as configured in core-site.xml or " + "a particular provider - as configured in core-site.xml or " +
"indicated\nthrough the --provider argument."; "indicated\nthrough the --provider argument.\n" +
"If the --metadata option is used, the keys metadata will be printed";
private boolean metadata = false;
public boolean validate() { public boolean validate() {
boolean rc = true; boolean rc = true;
@ -217,16 +225,24 @@ public class KeyShell extends Configured implements Tool {
+ "you MUST use the --provider argument."); + "you MUST use the --provider argument.");
rc = false; rc = false;
} }
metadata = getConf().getBoolean(LIST_METADATA, false);
return rc; return rc;
} }
public void execute() throws IOException { public void execute() throws IOException {
List<String> keys; List<String> keys;
try { try {
keys = provider.getKeys();
out.println("Listing keys for KeyProvider: " + provider.toString()); out.println("Listing keys for KeyProvider: " + provider.toString());
for (String keyName : keys) { if (metadata) {
out.println(keyName); Map<String, Metadata> keysMeta = provider.getKeysMetadata();
for (Map.Entry<String, Metadata> entry : keysMeta.entrySet()) {
out.println(entry.getKey() + " : " + entry.getValue());
}
} else {
keys = provider.getKeys();
for (String keyName : keys) {
out.println(keyName);
}
} }
} catch (IOException e) { } catch (IOException e) {
out.println("Cannot list keys for KeyProvider: " + provider.toString() out.println("Cannot list keys for KeyProvider: " + provider.toString()

View File

@ -89,7 +89,7 @@ public class UserProvider extends KeyProvider {
options.getBitLength() + ", but got " + (8 * material.length)); options.getBitLength() + ", but got " + (8 * material.length));
} }
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(), Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
new Date(), 1); options.getDescription(), new Date(), 1);
cache.put(name, meta); cache.put(name, meta);
String versionName = buildVersionName(name, 0); String versionName = buildVersionName(name, 0);
credentials.addSecretKey(nameT, meta.serialize()); credentials.addSecretKey(nameT, meta.serialize());

View File

@ -32,6 +32,7 @@ import java.util.Date;
import java.util.List; import java.util.List;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertArrayEquals;
@ -67,23 +68,47 @@ public class TestKeyProvider {
@Test @Test
public void testMetadata() throws Exception { public void testMetadata() throws Exception {
//Metadata without description
DateFormat format = new SimpleDateFormat("y/m/d"); DateFormat format = new SimpleDateFormat("y/m/d");
Date date = format.parse("2013/12/25"); Date date = format.parse("2013/12/25");
KeyProvider.Metadata meta = new KeyProvider.Metadata("myCipher", 100, KeyProvider.Metadata meta = new KeyProvider.Metadata("myCipher", 100, null,
date, 123); date, 123);
assertEquals("myCipher", meta.getCipher()); assertEquals("myCipher", meta.getCipher());
assertEquals(100, meta.getBitLength()); assertEquals(100, meta.getBitLength());
assertNull(meta.getDescription());
assertEquals(date, meta.getCreated()); assertEquals(date, meta.getCreated());
assertEquals(123, meta.getVersions()); assertEquals(123, meta.getVersions());
KeyProvider.Metadata second = new KeyProvider.Metadata(meta.serialize()); KeyProvider.Metadata second = new KeyProvider.Metadata(meta.serialize());
assertEquals(meta.getCipher(), second.getCipher()); assertEquals(meta.getCipher(), second.getCipher());
assertEquals(meta.getBitLength(), second.getBitLength()); assertEquals(meta.getBitLength(), second.getBitLength());
assertNull(second.getDescription());
assertEquals(meta.getCreated(), second.getCreated()); assertEquals(meta.getCreated(), second.getCreated());
assertEquals(meta.getVersions(), second.getVersions()); assertEquals(meta.getVersions(), second.getVersions());
int newVersion = second.addVersion(); int newVersion = second.addVersion();
assertEquals(123, newVersion); assertEquals(123, newVersion);
assertEquals(124, second.getVersions()); assertEquals(124, second.getVersions());
assertEquals(123, meta.getVersions()); assertEquals(123, meta.getVersions());
//Metadata with description
format = new SimpleDateFormat("y/m/d");
date = format.parse("2013/12/25");
meta = new KeyProvider.Metadata("myCipher", 100,
"description", date, 123);
assertEquals("myCipher", meta.getCipher());
assertEquals(100, meta.getBitLength());
assertEquals("description", meta.getDescription());
assertEquals(date, meta.getCreated());
assertEquals(123, meta.getVersions());
second = new KeyProvider.Metadata(meta.serialize());
assertEquals(meta.getCipher(), second.getCipher());
assertEquals(meta.getBitLength(), second.getBitLength());
assertEquals(meta.getDescription(), second.getDescription());
assertEquals(meta.getCreated(), second.getCreated());
assertEquals(meta.getVersions(), second.getVersions());
newVersion = second.addVersion();
assertEquals(123, newVersion);
assertEquals(124, second.getVersions());
assertEquals(123, meta.getVersions());
} }
@Test @Test
@ -95,9 +120,11 @@ public class TestKeyProvider {
assertEquals("myCipher", options.getCipher()); assertEquals("myCipher", options.getCipher());
assertEquals(512, options.getBitLength()); assertEquals(512, options.getBitLength());
options.setCipher("yourCipher"); options.setCipher("yourCipher");
options.setDescription("description");
options.setBitLength(128); options.setBitLength(128);
assertEquals("yourCipher", options.getCipher()); assertEquals("yourCipher", options.getCipher());
assertEquals(128, options.getBitLength()); assertEquals(128, options.getBitLength());
assertEquals("description", options.getDescription());
options = KeyProvider.options(new Configuration()); options = KeyProvider.options(new Configuration());
assertEquals(KeyProvider.DEFAULT_CIPHER, options.getCipher()); assertEquals(KeyProvider.DEFAULT_CIPHER, options.getCipher());
assertEquals(KeyProvider.DEFAULT_BITLENGTH, options.getBitLength()); assertEquals(KeyProvider.DEFAULT_BITLENGTH, options.getBitLength());
@ -139,7 +166,7 @@ public class TestKeyProvider {
@Override @Override
public Metadata getMetadata(String name) throws IOException { public Metadata getMetadata(String name) throws IOException {
return new Metadata(CIPHER, 128, new Date(), 0); return new Metadata(CIPHER, 128, "description", new Date(), 0);
} }
@Override @Override

View File

@ -22,23 +22,42 @@ import static org.junit.Assert.*;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.PrintStream; import java.io.PrintStream;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
public class TestKeyShell { public class TestKeyShell {
private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
private static final File tmpDir =
new File(System.getProperty("test.build.data", "/tmp"), "key"); private static File tmpDir;
private PrintStream initialStdOut;
private PrintStream initialStdErr;
@Before @Before
public void setup() throws Exception { public void setup() throws Exception {
outContent.reset();
errContent.reset();
tmpDir = new File(System.getProperty("test.build.data", "target"),
UUID.randomUUID().toString());
tmpDir.mkdirs();
initialStdOut = System.out;
initialStdErr = System.err;
System.setOut(new PrintStream(outContent)); System.setOut(new PrintStream(outContent));
System.setErr(new PrintStream(errContent)); System.setErr(new PrintStream(errContent));
} }
@After
public void cleanUp() throws Exception {
System.setOut(initialStdOut);
System.setErr(initialStdErr);
}
@Test @Test
public void testKeySuccessfulKeyLifecycle() throws Exception { public void testKeySuccessfulKeyLifecycle() throws Exception {
outContent.reset(); outContent.reset();
@ -59,6 +78,15 @@ public class TestKeyShell {
assertEquals(0, rc); assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1")); assertTrue(outContent.toString().contains("key1"));
outContent.reset();
String[] args2a = {"list", "--metadata", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
rc = ks.run(args2a);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1"));
assertTrue(outContent.toString().contains("description"));
assertTrue(outContent.toString().contains("created"));
outContent.reset(); outContent.reset();
String[] args3 = {"roll", "key1", "--provider", String[] args3 = {"roll", "key1", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"}; "jceks://file" + tmpDir + "/keystore.jceks"};