HADOOP-10430. KeyProvider Metadata should have an optional description, there should be a method to retrieve the metadata from all keys. (tucu)

Conflicts:
	hadoop-common-project/hadoop-common/CHANGES.txt

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1619515 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -105,6 +105,9 @@ Release 2.6.0 - UNRELEASED
     HADOOP-10428. JavaKeyStoreProvider should accept keystore password via
     configuration falling back to ENV VAR. (tucu)
 
+    HADOOP-10430. KeyProvider Metadata should have an optional description,
+    there should be a method to retrieve the metadata from all keys. (tucu)
+
   OPTIMIZATIONS
 
     HADOOP-10838. Byte array native checksumming. (James Thomas via todd)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java

@@ -268,7 +268,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
             e);
       }
       Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
-          new Date(), 1);
+          options.getDescription(), new Date(), 1);
       if (options.getBitLength() != 8 * material.length) {
         throw new IOException("Wrong key length. Required " +
             options.getBitLength() + ", but got " + (8 * material.length));

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java

@@ -25,8 +25,11 @@ import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.net.URI;
 import java.security.NoSuchAlgorithmException;
+import java.text.MessageFormat;
 import java.util.Date;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.google.gson.stream.JsonReader;
 import com.google.gson.stream.JsonWriter;
@@ -104,21 +107,34 @@ public abstract class KeyProvider {
     private final static String CIPHER_FIELD = "cipher";
     private final static String BIT_LENGTH_FIELD = "bitLength";
     private final static String CREATED_FIELD = "created";
+    private final static String DESCRIPTION_FIELD = "description";
     private final static String VERSIONS_FIELD = "versions";
 
     private final String cipher;
     private final int bitLength;
+    private final String description;
     private final Date created;
     private int versions;
 
     protected Metadata(String cipher, int bitLength,
-                       Date created, int versions) {
+                       String description, Date created, int versions) {
       this.cipher = cipher;
       this.bitLength = bitLength;
+      this.description = description;
       this.created = created;
       this.versions = versions;
     }
 
+    public String toString() {
+      return MessageFormat.format(
+          "cipher: {0}, length: {1} description: {2} created: {3} version: {4}",
+          cipher, bitLength, description, created, versions);
+    }
+
+    public String getDescription() {
+      return description;
+    }
+
     public Date getCreated() {
       return created;
     }
@@ -170,6 +186,9 @@ public abstract class KeyProvider {
       if (created != null) {
         writer.name(CREATED_FIELD).value(created.getTime());
       }
+      if (description != null) {
+        writer.name(DESCRIPTION_FIELD).value(description);
+      }
       writer.name(VERSIONS_FIELD).value(versions);
       writer.endObject();
       writer.flush();
@@ -186,6 +205,7 @@ public abstract class KeyProvider {
       int bitLength = 0;
       Date created = null;
       int versions = 0;
+      String description = null;
       JsonReader reader = new JsonReader(new InputStreamReader
           (new ByteArrayInputStream(bytes)));
       reader.beginObject();
@@ -199,12 +219,15 @@ public abstract class KeyProvider {
           created = new Date(reader.nextLong());
         } else if (VERSIONS_FIELD.equals(field)) {
           versions = reader.nextInt();
+        } else if (DESCRIPTION_FIELD.equals(field)) {
+          description = reader.nextString();
         }
       }
       reader.endObject();
       this.cipher = cipher;
       this.bitLength = bitLength;
       this.created = created;
+      this.description = description;
       this.versions = versions;
     }
   }
@@ -215,6 +238,7 @@ public abstract class KeyProvider {
   public static class Options {
     private String cipher;
     private int bitLength;
+    private String description;
 
     public Options(Configuration conf) {
       cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER);
@@ -231,6 +255,11 @@ public abstract class KeyProvider {
       return this;
     }
 
+    public Options setDescription(String description) {
+      this.description = description;
+      return this;
+    }
+
     protected String getCipher() {
       return cipher;
     }
@@ -238,6 +267,10 @@ public abstract class KeyProvider {
     protected int getBitLength() {
       return bitLength;
     }
+
+    protected String getDescription() {
+      return description;
+    }
   }
 
   /**
@@ -277,6 +310,24 @@ public abstract class KeyProvider {
    */
   public abstract List<String> getKeys() throws IOException;
 
+
+  /**
+   * Get the key metadata for all keys.
+   *
+   * @return a Map with all the keys and their metadata
+   * @throws IOException
+   */
+  public Map<String, Metadata> getKeysMetadata() throws IOException {
+    Map<String, Metadata> keysMetadata = new LinkedHashMap<String, Metadata>();
+    for (String key : getKeys()) {
+      Metadata meta = getMetadata(key);
+      if (meta != null) {
+        keysMetadata.put(key, meta);
+      }
+    }
+    return keysMetadata;
+  }
+
   /**
    * Get the key material for all versions of a specific key name.
    * @return the list of key material

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java

@@ -23,6 +23,7 @@ import java.io.PrintStream;
 import java.security.InvalidParameterException;
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
+import java.util.Map;
 
 import javax.crypto.KeyGenerator;
 
@@ -45,6 +46,7 @@ public class KeyShell extends Configured implements Tool {
       "   [" + RollCommand.USAGE + "]\n" +
       "   [" + DeleteCommand.USAGE + "]\n" +
       "   [" + ListCommand.USAGE + "]\n";
+  private static final String LIST_METADATA = "keyShell.list.metadata";
 
   private boolean interactive = false;
   private Command command = null;
@@ -121,6 +123,8 @@ public class KeyShell extends Configured implements Tool {
       } else if (args[i].equals("--provider")) {
         userSuppliedProvider = true;
         getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
+      } else if (args[i].equals("--metadata")) {
+        getConf().setBoolean(LIST_METADATA, true);
       } else if (args[i].equals("-i") || (args[i].equals("--interactive"))) {
         interactive = true;
       } else if (args[i].equals("--help")) {
@@ -201,11 +205,15 @@ public class KeyShell extends Configured implements Tool {
   }
 
   private class ListCommand extends Command {
-    public static final String USAGE = "list <keyname> [--provider] [--help]";
+    public static final String USAGE =
+        "list [--provider] [--metadata] [--help]";
     public static final String DESC =
         "The list subcommand displays the keynames contained within \n" +
         "a particular provider - as configured in core-site.xml or " +
-        "indicated\nthrough the --provider argument.";
+        "indicated\nthrough the --provider argument.\n" +
+        "If the --metadata option is used, the keys metadata will be printed";
+
+    private boolean metadata = false;
 
     public boolean validate() {
       boolean rc = true;
@@ -217,17 +225,25 @@ public class KeyShell extends Configured implements Tool {
             + "you MUST use the --provider argument.");
         rc = false;
       }
+      metadata = getConf().getBoolean(LIST_METADATA, false);
       return rc;
     }
 
     public void execute() throws IOException {
       List<String> keys;
       try {
-        keys = provider.getKeys();
         out.println("Listing keys for KeyProvider: " + provider.toString());
+        if (metadata) {
+          Map<String, Metadata> keysMeta = provider.getKeysMetadata();
+          for (Map.Entry<String, Metadata> entry : keysMeta.entrySet()) {
+            out.println(entry.getKey() + " : " + entry.getValue());
+          }
+        } else {
+          keys = provider.getKeys();
           for (String keyName : keys) {
             out.println(keyName);
           }
+        }
       } catch (IOException e) {
         out.println("Cannot list keys for KeyProvider: " + provider.toString()
             + ": " + e.getMessage());

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java

@@ -89,7 +89,7 @@ public class UserProvider extends KeyProvider {
           options.getBitLength() + ", but got " + (8 * material.length));
     }
     Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
-        new Date(), 1);
+        options.getDescription(), new Date(), 1);
     cache.put(name, meta);
     String versionName = buildVersionName(name, 0);
     credentials.addSecretKey(nameT, meta.serialize());

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java

@@ -32,6 +32,7 @@ import java.util.Date;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertArrayEquals;
 
@@ -67,23 +68,47 @@ public class TestKeyProvider {
 
   @Test
   public void testMetadata() throws Exception {
+    //Metadata without description
     DateFormat format = new SimpleDateFormat("y/m/d");
     Date date = format.parse("2013/12/25");
-    KeyProvider.Metadata meta = new KeyProvider.Metadata("myCipher", 100,
+    KeyProvider.Metadata meta = new KeyProvider.Metadata("myCipher", 100, null,
         date, 123);
     assertEquals("myCipher", meta.getCipher());
     assertEquals(100, meta.getBitLength());
+    assertNull(meta.getDescription());
     assertEquals(date, meta.getCreated());
     assertEquals(123, meta.getVersions());
     KeyProvider.Metadata second = new KeyProvider.Metadata(meta.serialize());
     assertEquals(meta.getCipher(), second.getCipher());
     assertEquals(meta.getBitLength(), second.getBitLength());
+    assertNull(second.getDescription());
     assertEquals(meta.getCreated(), second.getCreated());
     assertEquals(meta.getVersions(), second.getVersions());
     int newVersion = second.addVersion();
     assertEquals(123, newVersion);
     assertEquals(124, second.getVersions());
     assertEquals(123, meta.getVersions());
+
+    //Metadata with description
+    format = new SimpleDateFormat("y/m/d");
+    date = format.parse("2013/12/25");
+    meta = new KeyProvider.Metadata("myCipher", 100,
+        "description", date, 123);
+    assertEquals("myCipher", meta.getCipher());
+    assertEquals(100, meta.getBitLength());
+    assertEquals("description", meta.getDescription());
+    assertEquals(date, meta.getCreated());
+    assertEquals(123, meta.getVersions());
+    second = new KeyProvider.Metadata(meta.serialize());
+    assertEquals(meta.getCipher(), second.getCipher());
+    assertEquals(meta.getBitLength(), second.getBitLength());
+    assertEquals(meta.getDescription(), second.getDescription());
+    assertEquals(meta.getCreated(), second.getCreated());
+    assertEquals(meta.getVersions(), second.getVersions());
+    newVersion = second.addVersion();
+    assertEquals(123, newVersion);
+    assertEquals(124, second.getVersions());
+    assertEquals(123, meta.getVersions());
   }
 
   @Test
@@ -95,9 +120,11 @@ public class TestKeyProvider {
     assertEquals("myCipher", options.getCipher());
     assertEquals(512, options.getBitLength());
     options.setCipher("yourCipher");
+    options.setDescription("description");
     options.setBitLength(128);
     assertEquals("yourCipher", options.getCipher());
     assertEquals(128, options.getBitLength());
+    assertEquals("description", options.getDescription());
     options = KeyProvider.options(new Configuration());
     assertEquals(KeyProvider.DEFAULT_CIPHER, options.getCipher());
     assertEquals(KeyProvider.DEFAULT_BITLENGTH, options.getBitLength());
@@ -139,7 +166,7 @@ public class TestKeyProvider {
 
     @Override
     public Metadata getMetadata(String name) throws IOException {
-      return new Metadata(CIPHER, 128, new Date(), 0);
+      return new Metadata(CIPHER, 128, "description", new Date(), 0);
     }
 
     @Override

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java

@@ -22,23 +22,42 @@ import static org.junit.Assert.*;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.PrintStream;
+import java.util.UUID;
 
 import org.apache.hadoop.conf.Configuration;
+import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 public class TestKeyShell {
   private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
   private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
-  private static final File tmpDir =
-      new File(System.getProperty("test.build.data", "/tmp"), "key");
+
+  private static File tmpDir;
+
+  private PrintStream initialStdOut;
+  private PrintStream initialStdErr;
 
   @Before
   public void setup() throws Exception {
+    outContent.reset();
+    errContent.reset();
+    tmpDir = new File(System.getProperty("test.build.data", "target"),
+        UUID.randomUUID().toString());
+    tmpDir.mkdirs();
+    initialStdOut = System.out;
+    initialStdErr = System.err;
     System.setOut(new PrintStream(outContent));
     System.setErr(new PrintStream(errContent));
   }
 
+  @After
+  public void cleanUp() throws Exception {
+    System.setOut(initialStdOut);
+    System.setErr(initialStdErr);
+  }
+
   @Test
   public void testKeySuccessfulKeyLifecycle() throws Exception {
     outContent.reset();
@@ -59,6 +78,15 @@ public class TestKeyShell {
     assertEquals(0, rc);
     assertTrue(outContent.toString().contains("key1"));
 
+    outContent.reset();
+    String[] args2a = {"list", "--metadata", "--provider",
+                      "jceks://file" + tmpDir + "/keystore.jceks"};
+    rc = ks.run(args2a);
+    assertEquals(0, rc);
+    assertTrue(outContent.toString().contains("key1"));
+    assertTrue(outContent.toString().contains("description"));
+    assertTrue(outContent.toString().contains("created"));
+
     outContent.reset();
     String[] args3 = {"roll", "key1", "--provider",
         "jceks://file" + tmpDir + "/keystore.jceks"};