HADOOP-10177. Create CLI tools for managing keys. (Larry McCay via omalley)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1558867 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Owen O'Malley 2014-01-16 17:48:50 +00:00
parent 3a6f8b8785
commit e05ff82bd9
7 changed files with 675 additions and 4 deletions

View File

@ -98,7 +98,8 @@ Trunk (Unreleased)
HADOOP-8844. Add a plaintext fs -text test-case. HADOOP-8844. Add a plaintext fs -text test-case.
(Akira AJISAKA via harsh) (Akira AJISAKA via harsh)
HADOOP-9432 Add support for markdown .md files in site documentation (stevel) HADOOP-9432 Add support for markdown .md files in site documentation
(stevel)
HADOOP-9186. test-patch.sh should report build failure to JIRA. HADOOP-9186. test-patch.sh should report build failure to JIRA.
(Binglin Chang via Colin Patrick McCabe) (Binglin Chang via Colin Patrick McCabe)
@ -110,6 +111,8 @@ Trunk (Unreleased)
HADOOP-10201. Add listing to KeyProvider API. (Larry McCay via omalley) HADOOP-10201. Add listing to KeyProvider API. (Larry McCay via omalley)
HADOOP-10177. Create CLI tools for managing keys. (Larry McCay via omalley)
BUG FIXES BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled. HADOOP-9451. Fault single-layer config if node group topology is enabled.
@ -117,8 +120,8 @@ Trunk (Unreleased)
HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang) HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName. HADOOP-8177. MBeans shouldn't try to register when it fails to create
(Devaraj K via umamahesh) MBeanName. (Devaraj K via umamahesh)
HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc
(Jon Eagles via bobby) (Jon Eagles via bobby)

View File

@ -104,6 +104,8 @@ case $COMMAND in
CLASS=org.apache.hadoop.util.VersionInfo CLASS=org.apache.hadoop.util.VersionInfo
elif [ "$COMMAND" = "jar" ] ; then elif [ "$COMMAND" = "jar" ] ; then
CLASS=org.apache.hadoop.util.RunJar CLASS=org.apache.hadoop.util.RunJar
elif [ "$COMMAND" = "key" ] ; then
CLASS=org.apache.hadoop.crypto.key.KeyShell
elif [ "$COMMAND" = "checknative" ] ; then elif [ "$COMMAND" = "checknative" ] ; then
CLASS=org.apache.hadoop.util.NativeLibraryChecker CLASS=org.apache.hadoop.util.NativeLibraryChecker
elif [ "$COMMAND" = "distcp" ] ; then elif [ "$COMMAND" = "distcp" ] ; then

View File

@ -77,7 +77,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException { private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
this.uri = uri; this.uri = uri;
path = unnestUri(uri); path = unnestUri(uri);
fs = FileSystem.get(conf); fs = path.getFileSystem(conf);
// Get the password from the user's environment // Get the password from the user's environment
String pw = System.getenv(KEYSTORE_PASSWORD_NAME); String pw = System.getenv(KEYSTORE_PASSWORD_NAME);
if (pw == null) { if (pw == null) {

View File

@ -244,6 +244,17 @@ public abstract class KeyProvider {
return new Options(conf); return new Options(conf);
} }
/**
* Indicates whether this provider represents a store
* that is intended for transient use - such as the UserProvider
* is. These providers are generally used to provide access to
* keying material rather than for long term storage.
* @return true if transient, false otherwise
*/
public boolean isTransient() {
return false;
}
/** /**
* Get the key material for a specific version of the key. This method is used * Get the key material for a specific version of the key. This method is used
* when decrypting data. * when decrypting data.

View File

@ -0,0 +1,474 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.crypto.key;
import java.io.IOException;
import java.io.PrintStream;
import java.security.InvalidParameterException;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import javax.crypto.KeyGenerator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.crypto.key.KeyProvider.Metadata;
import org.apache.hadoop.crypto.key.KeyProvider.Options;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* This program is the CLI utility for the KeyProvider facilities in Hadoop.
*/
public class KeyShell extends Configured implements Tool {
final static private String USAGE_PREFIX = "Usage: hadoop key " +
"[generic options]\n";
final static private String COMMANDS =
" [--help]\n" +
" [" + CreateCommand.USAGE + "]\n" +
" [" + RollCommand.USAGE + "]\n" +
" [" + DeleteCommand.USAGE + "]\n" +
" [" + ListCommand.USAGE + "]\n";
private boolean interactive = false;
private Command command = null;
/** allows stdout to be captured if necessary */
public PrintStream out = System.out;
/** allows stderr to be captured if necessary */
public PrintStream err = System.err;
private boolean userSuppliedProvider = false;
@Override
public int run(String[] args) throws Exception {
int exitCode = 0;
try {
exitCode = init(args);
if (exitCode != 0) {
return exitCode;
}
if (command.validate()) {
command.execute();
} else {
exitCode = -1;
}
} catch (Exception e) {
e.printStackTrace(err);
return -1;
}
return exitCode;
}
/**
* Parse the command line arguments and initialize the data
* <pre>
* % hadoop key create keyName [--size size] [--cipher algorithm]
* [--provider providerPath]
* % hadoop key roll keyName [--provider providerPath]
* % hadoop key list [-provider providerPath]
* % hadoop key delete keyName [--provider providerPath] [-i]
* </pre>
* @param args
* @return
* @throws IOException
*/
private int init(String[] args) throws IOException {
for (int i = 0; i < args.length; i++) { // parse command line
if (args[i].equals("create")) {
String keyName = args[++i];
command = new CreateCommand(keyName);
if (keyName.equals("--help")) {
printKeyShellUsage();
return -1;
}
} else if (args[i].equals("delete")) {
String keyName = args[++i];
command = new DeleteCommand(keyName);
if (keyName.equals("--help")) {
printKeyShellUsage();
return -1;
}
} else if (args[i].equals("roll")) {
String keyName = args[++i];
command = new RollCommand(keyName);
if (keyName.equals("--help")) {
printKeyShellUsage();
return -1;
}
} else if (args[i].equals("list")) {
command = new ListCommand();
} else if (args[i].equals("--size")) {
getConf().set(KeyProvider.DEFAULT_BITLENGTH_NAME, args[++i]);
} else if (args[i].equals("--cipher")) {
getConf().set(KeyProvider.DEFAULT_CIPHER_NAME, args[++i]);
} else if (args[i].equals("--provider")) {
userSuppliedProvider = true;
getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
} else if (args[i].equals("-i") || (args[i].equals("--interactive"))) {
interactive = true;
} else if (args[i].equals("--help")) {
printKeyShellUsage();
return -1;
} else {
printKeyShellUsage();
ToolRunner.printGenericCommandUsage(System.err);
return -1;
}
}
return 0;
}
private void printKeyShellUsage() {
out.println(USAGE_PREFIX + COMMANDS);
if (command != null) {
out.println(command.getUsage());
}
else {
out.println("=========================================================" +
"======");
out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC);
out.println("=========================================================" +
"======");
out.println(RollCommand.USAGE + ":\n\n" + RollCommand.DESC);
out.println("=========================================================" +
"======");
out.println(DeleteCommand.USAGE + ":\n\n" + DeleteCommand.DESC);
out.println("=========================================================" +
"======");
out.println(ListCommand.USAGE + ":\n\n" + ListCommand.DESC);
}
}
private abstract class Command {
protected KeyProvider provider = null;
public boolean validate() {
return true;
}
protected KeyProvider getKeyProvider() {
KeyProvider provider = null;
List<KeyProvider> providers;
try {
providers = KeyProviderFactory.getProviders(getConf());
if (userSuppliedProvider) {
provider = providers.get(0);
}
else {
for (KeyProvider p : providers) {
if (!p.isTransient()) {
provider = p;
break;
}
}
}
} catch (IOException e) {
e.printStackTrace(err);
}
return provider;
}
protected byte[] generateKey(int size, String algorithm)
throws NoSuchAlgorithmException {
out.println("Generating key using size: " + size + " and algorithm: "
+ algorithm);
KeyGenerator keyGenerator = KeyGenerator.getInstance(algorithm);
keyGenerator.init(size);
byte[] key = keyGenerator.generateKey().getEncoded();
return key;
}
protected void printProviderWritten() {
out.println(provider.getClass().getName() + " has been updated.");
}
protected void warnIfTransientProvider() {
if (provider.isTransient()) {
out.println("WARNING: you are modifying a transient provider.");
}
}
public abstract void execute() throws Exception;
public abstract String getUsage();
}
private class ListCommand extends Command {
public static final String USAGE = "list <keyname> [--provider] [--help]";
public static final String DESC =
"The list subcommand displays the keynames contained within \n" +
"a particular provider - as configured in core-site.xml or " +
"indicated\nthrough the --provider argument.";
public boolean validate() {
boolean rc = true;
provider = getKeyProvider();
if (provider == null) {
out.println("There are no non-transient KeyProviders configured.\n"
+ "Consider using the --provider option to indicate the provider\n"
+ "to use. If you want to list a transient provider then you\n"
+ "you MUST use the --provider argument.");
rc = false;
}
return rc;
}
public void execute() throws IOException {
List<String> keys;
try {
keys = provider.getKeys();
out.println("Listing keys for KeyProvider: " + provider.toString());
for (String keyName : keys) {
out.println(keyName);
}
} catch (IOException e) {
out.println("Cannot list keys for KeyProvider: " + provider.toString()
+ ": " + e.getMessage());
throw e;
}
}
@Override
public String getUsage() {
return USAGE + ":\n\n" + DESC;
}
}
private class RollCommand extends Command {
public static final String USAGE = "roll <keyname> [--provider] [--help]";
public static final String DESC =
"The roll subcommand creates a new version of the key specified\n" +
"through the <keyname> argument within the provider indicated using\n" +
"the --provider argument";
String keyName = null;
public RollCommand(String keyName) {
this.keyName = keyName;
}
public boolean validate() {
boolean rc = true;
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured.\n"
+ "Key will not be rolled.\n"
+ "Consider using the --provider option to indicate the provider"
+ " to use.");
rc = false;
}
if (keyName == null) {
out.println("There is no keyName specified. Please provide the" +
"mandatory <keyname>. See the usage description with --help.");
rc = false;
}
return rc;
}
public void execute() throws NoSuchAlgorithmException, IOException {
try {
Metadata md = provider.getMetadata(keyName);
warnIfTransientProvider();
out.println("Rolling key version from KeyProvider: "
+ provider.toString() + " for key name: " + keyName);
try {
byte[] material = null;
material = generateKey(md.getBitLength(), md.getAlgorithm());
provider.rollNewVersion(keyName, material);
out.println(keyName + " has been successfully rolled.");
provider.flush();
printProviderWritten();
} catch (NoSuchAlgorithmException e) {
out.println("Cannot roll key: " + keyName + " within KeyProvider: "
+ provider.toString());
throw e;
}
} catch (IOException e1) {
out.println("Cannot roll key: " + keyName + " within KeyProvider: "
+ provider.toString());
throw e1;
}
}
@Override
public String getUsage() {
return USAGE + ":\n\n" + DESC;
}
}
private class DeleteCommand extends Command {
public static final String USAGE = "delete <keyname> [--provider] [--help]";
public static final String DESC =
"The delete subcommand deletes all of the versions of the key\n" +
"specified as the <keyname> argument from within the provider\n" +
"indicated through the --provider argument";
String keyName = null;
boolean cont = true;
public DeleteCommand(String keyName) {
this.keyName = keyName;
}
@Override
public boolean validate() {
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured.\n"
+ "Nothing will be deleted.\n"
+ "Consider using the --provider option to indicate the provider"
+ " to use.");
return false;
}
if (keyName == null) {
out.println("There is no keyName specified. Please provide the" +
"mandatory <keyname>. See the usage description with --help.");
return false;
}
if (interactive) {
try {
cont = ToolRunner
.confirmPrompt("You are about to DELETE all versions of "
+ "the key: " + keyName + " from KeyProvider "
+ provider.toString() + ". Continue?:");
if (!cont) {
out.println("Nothing has been be deleted.");
}
return cont;
} catch (IOException e) {
out.println(keyName + " will not be deleted.");
e.printStackTrace(err);
}
}
return true;
}
public void execute() throws IOException {
warnIfTransientProvider();
out.println("Deleting key: " + keyName + " from KeyProvider: "
+ provider.toString());
if (cont) {
try {
provider.deleteKey(keyName);
out.println(keyName + " has been successfully deleted.");
provider.flush();
printProviderWritten();
} catch (IOException e) {
out.println(keyName + "has NOT been deleted.");
throw e;
}
}
}
@Override
public String getUsage() {
return USAGE + ":\n\n" + DESC;
}
}
private class CreateCommand extends Command {
public static final String USAGE = "create <keyname> [--cipher] " +
"[--size] [--provider] [--help]";
public static final String DESC =
"The create subcommand creates a new key for the name specified\n" +
"as the <keyname> argument within the provider indicated through\n" +
"the --provider argument. You may also indicate the specific\n" +
"cipher through the --cipher argument. The default for cipher is\n" +
"currently \"AES/CTR/NoPadding\". The default keysize is \"256\".\n" +
"You may also indicate the requested key length through the --size\n" +
"argument.";
String keyName = null;
public CreateCommand(String keyName) {
this.keyName = keyName;
}
public boolean validate() {
boolean rc = true;
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured.\nKey" +
" will not be created.\n"
+ "Consider using the --provider option to indicate the provider" +
" to use.");
rc = false;
}
if (keyName == null) {
out.println("There is no keyName specified. Please provide the" +
"mandatory <keyname>. See the usage description with --help.");
rc = false;
}
return rc;
}
public void execute() throws IOException, NoSuchAlgorithmException {
warnIfTransientProvider();
try {
Options options = KeyProvider.options(getConf());
String alg = getAlgorithm(options.getCipher());
byte[] material = generateKey(options.getBitLength(), alg);
provider.createKey(keyName, material, options);
out.println(keyName + " has been successfully created.");
provider.flush();
printProviderWritten();
} catch (InvalidParameterException e) {
out.println(keyName + " has NOT been created. " + e.getMessage());
throw e;
} catch (IOException e) {
out.println(keyName + " has NOT been created. " + e.getMessage());
throw e;
} catch (NoSuchAlgorithmException e) {
out.println(keyName + " has NOT been created. " + e.getMessage());
throw e;
}
}
/**
* Get the algorithm from the cipher.
* @return the algorithm name
*/
public String getAlgorithm(String cipher) {
int slash = cipher.indexOf('/');
if (slash == - 1) {
return cipher;
} else {
return cipher.substring(0, slash);
}
}
@Override
public String getUsage() {
return USAGE + ":\n\n" + DESC;
}
}
/**
* Main program.
*
* @param args
* Command line arguments
* @throws Exception
*/
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new KeyShell(), args);
System.exit(res);
}
}

View File

@ -49,6 +49,11 @@ public class UserProvider extends KeyProvider {
credentials = user.getCredentials(); credentials = user.getCredentials();
} }
@Override
public boolean isTransient() {
return true;
}
@Override @Override
public KeyVersion getKeyVersion(String versionName) { public KeyVersion getKeyVersion(String versionName) {
byte[] bytes = credentials.getSecretKey(new Text(versionName)); byte[] bytes = credentials.getSecretKey(new Text(versionName));

View File

@ -0,0 +1,176 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.crypto.key;
import static org.junit.Assert.*;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import org.apache.hadoop.conf.Configuration;
import org.junit.Before;
import org.junit.Test;
public class TestKeyShell {
private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
private static final File tmpDir =
new File(System.getProperty("test.build.data", "/tmp"), "key");
@Before
public void setup() throws Exception {
System.setOut(new PrintStream(outContent));
System.setErr(new PrintStream(errContent));
}
@Test
public void testKeySuccessfulKeyLifecycle() throws Exception {
outContent.flush();
String[] args1 = {"create", "key1", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
int rc = 0;
KeyShell ks = new KeyShell();
ks.setConf(new Configuration());
rc = ks.run(args1);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1 has been successfully " +
"created."));
outContent.flush();
String[] args2 = {"list", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
rc = ks.run(args2);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1"));
outContent.flush();
String[] args3 = {"roll", "key1", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
rc = ks.run(args3);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1 has been successfully " +
"rolled."));
outContent.flush();
String[] args4 = {"delete", "key1", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
rc = ks.run(args4);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1 has been successfully " +
"deleted."));
outContent.flush();
String[] args5 = {"list", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
rc = ks.run(args5);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1"));
}
@Test
public void testInvalidKeySize() throws Exception {
String[] args1 = {"create", "key1", "--size", "56", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
int rc = 0;
KeyShell ks = new KeyShell();
ks.setConf(new Configuration());
rc = ks.run(args1);
assertEquals(-1, rc);
assertTrue(outContent.toString().contains("key1 has NOT been created."));
}
@Test
public void testInvalidCipher() throws Exception {
String[] args1 = {"create", "key1", "--cipher", "LJM", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
int rc = 0;
KeyShell ks = new KeyShell();
ks.setConf(new Configuration());
rc = ks.run(args1);
assertEquals(-1, rc);
assertTrue(outContent.toString().contains("key1 has NOT been created."));
}
@Test
public void testInvalidProvider() throws Exception {
String[] args1 = {"create", "key1", "--cipher", "AES", "--provider",
"sdff://file/tmp/keystore.jceks"};
int rc = 0;
KeyShell ks = new KeyShell();
ks.setConf(new Configuration());
rc = ks.run(args1);
assertEquals(-1, rc);
assertTrue(outContent.toString().contains("There are no valid " +
"KeyProviders configured."));
}
@Test
public void testTransientProviderWarning() throws Exception {
String[] args1 = {"create", "key1", "--cipher", "AES", "--provider",
"user:///"};
int rc = 0;
KeyShell ks = new KeyShell();
ks.setConf(new Configuration());
rc = ks.run(args1);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("WARNING: you are modifying a " +
"transient provider."));
}
@Test
public void testTransientProviderOnlyConfig() throws Exception {
String[] args1 = {"create", "key1"};
int rc = 0;
KeyShell ks = new KeyShell();
Configuration config = new Configuration();
config.set(KeyProviderFactory.KEY_PROVIDER_PATH, "user:///");
ks.setConf(config);
rc = ks.run(args1);
assertEquals(-1, rc);
assertTrue(outContent.toString().contains("There are no valid " +
"KeyProviders configured."));
}
@Test
public void testFullCipher() throws Exception {
String[] args1 = {"create", "key1", "--cipher", "AES/CBC/pkcs5Padding",
"--provider", "jceks://file" + tmpDir + "/keystore.jceks"};
int rc = 0;
KeyShell ks = new KeyShell();
ks.setConf(new Configuration());
rc = ks.run(args1);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1 has been successfully " +
"created."));
outContent.flush();
String[] args2 = {"delete", "key1", "--provider",
"jceks://file" + tmpDir + "/keystore.jceks"};
rc = ks.run(args2);
assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1 has been successfully " +
"deleted."));
}
}