Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12942. hadoop credential commands non-obviously use password of "none" (Mike Yoder via lmccay)

This commit is contained in:
Larry McCay 2016-05-11 14:38:21 -04:00
parent e9ee258d04
commit ee2c2afdc3
13 changed files with 515 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
View File

@ -44,6 +44,7 @@
import java.io.Serializable; import java.io.Serializable;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.net.URL;
import java.security.GeneralSecurityException;
import java.security.Key; import java.security.Key;
import java.security.KeyStore; import java.security.KeyStore;
import java.security.KeyStoreException; import java.security.KeyStoreException;
@ -88,7 +89,7 @@
@InterfaceAudience.Private @InterfaceAudience.Private
public class JavaKeyStoreProvider extends KeyProvider { public class JavaKeyStoreProvider extends KeyProvider {
private static final String KEY_METADATA = "KeyMetadata"; private static final String KEY_METADATA = "KeyMetadata";
private static Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(JavaKeyStoreProvider.class); LoggerFactory.getLogger(JavaKeyStoreProvider.class);
public static final String SCHEME_NAME = "jceks"; public static final String SCHEME_NAME = "jceks";
@ -103,8 +104,8 @@ public class JavaKeyStoreProvider extends KeyProvider {
private final URI uri; private final URI uri;
private final Path path; private final Path path;
private final FileSystem fs; private final FileSystem fs;
private final FsPermission permissions; private FsPermission permissions;
private final KeyStore keyStore; private KeyStore keyStore;
private char[] password; private char[] password;
private boolean changed = false; private boolean changed = false;
private Lock readLock; private Lock readLock;
@ -131,13 +132,28 @@ private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
this.uri = uri; this.uri = uri;
path = ProviderUtils.unnestUri(uri); path = ProviderUtils.unnestUri(uri);
fs = path.getFileSystem(conf); fs = path.getFileSystem(conf);
locateKeystore();
ReadWriteLock lock = new ReentrantReadWriteLock(true);
readLock = lock.readLock();
writeLock = lock.writeLock();
}
/**
* The password is either found in the environment or in a file. This
* routine implements the logic for locating the password in these
* locations.
* @return The password as a char []; null if not found.
* @throws IOException
*/
private char[] locatePassword() throws IOException {
char[] pass = null;
// Get the password file from the conf, if not present from the user's // Get the password file from the conf, if not present from the user's
// environment var // environment var
if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) { if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) {
password = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray(); pass = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray();
} }
if (password == null) { if (pass == null) {
String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY); String pwFile = getConf().get(KEYSTORE_PASSWORD_FILE_KEY);
if (pwFile != null) { if (pwFile != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader(); ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL pwdFile = cl.getResource(pwFile); URL pwdFile = cl.getResource(pwFile);
@ -146,14 +162,23 @@ private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
throw new IOException("Password file does not exists"); throw new IOException("Password file does not exists");
} }
try (InputStream is = pwdFile.openStream()) { try (InputStream is = pwdFile.openStream()) {
password = IOUtils.toString(is).trim().toCharArray(); pass = IOUtils.toString(is).trim().toCharArray();
} }
} }
} }
if (password == null) { return pass;
password = KEYSTORE_PASSWORD_DEFAULT; }
}
/**
* Open up and initialize the keyStore.
* @throws IOException
*/
private void locateKeystore() throws IOException {
try { try {
password = locatePassword();
if (password == null) {
password = KEYSTORE_PASSWORD_DEFAULT;
}
Path oldPath = constructOldPath(path); Path oldPath = constructOldPath(path);
Path newPath = constructNewPath(path); Path newPath = constructNewPath(path);
keyStore = KeyStore.getInstance(SCHEME_NAME); keyStore = KeyStore.getInstance(SCHEME_NAME);
@ -175,19 +200,14 @@ private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
permissions = perm; permissions = perm;
} catch (KeyStoreException e) { } catch (KeyStoreException e) {
throw new IOException("Can't create keystore", e); throw new IOException("Can't create keystore", e);
} catch (NoSuchAlgorithmException e) { } catch (GeneralSecurityException e) {
throw new IOException("Can't load keystore " + path, e);
} catch (CertificateException e) {
throw new IOException("Can't load keystore " + path, e); throw new IOException("Can't load keystore " + path, e);
} }
ReadWriteLock lock = new ReentrantReadWriteLock(true);
readLock = lock.readLock();
writeLock = lock.writeLock();
} }
/** /**
* Try loading from the user specified path, else load from the backup * Try loading from the user specified path, else load from the backup
* path in case Exception is not due to bad/wrong password * path in case Exception is not due to bad/wrong password.
* @param path Actual path to load from * @param path Actual path to load from
* @param backupPath Backup path (_OLD) * @param backupPath Backup path (_OLD)
* @return The permissions of the loaded file * @return The permissions of the loaded file
@ -256,7 +276,7 @@ private FsPermission tryLoadIncompleteFlush(Path oldPath, Path newPath)
if (perm == null) { if (perm == null) {
keyStore.load(null, password); keyStore.load(null, password);
LOG.debug("KeyStore initialized anew successfully !!"); LOG.debug("KeyStore initialized anew successfully !!");
perm = new FsPermission("700"); perm = new FsPermission("600");
} }
return perm; return perm;
} }
@ -321,6 +341,40 @@ private Path constructOldPath(Path path) {
return oldPath; return oldPath;
} }
@Override
public boolean needsPassword() throws IOException {
return (null == locatePassword());
}
@VisibleForTesting
public static final String NO_PASSWORD_WARN =
"WARNING: You have accepted the use of the default provider password\n" +
"by not configuring a password in one of the two following locations:\n";
public static final String NO_PASSWORD_ERROR =
"ERROR: The provider cannot find a password in the expected " +
"locations.\nPlease supply a password using one of the " +
"following two mechanisms:\n";
@VisibleForTesting public static final String NO_PASSWORD_INSTRUCTIONS =
" o In the environment variable " +
KEYSTORE_PASSWORD_ENV_VAR + "\n" +
" o In a file referred to by the configuration entry\n" +
" " + KEYSTORE_PASSWORD_FILE_KEY + ".\n" +
"Please review the documentation regarding provider passwords at\n" +
"http://hadoop.apache.org/docs/current/hadoop-project-dist/" +
"hadoop-common/CredentialProviderAPI.html#Keystore_Passwords\n";
@VisibleForTesting public static final String NO_PASSWORD_CONT =
"Continuing with the default provider password.\n";
@Override
public String noPasswordWarning() {
return NO_PASSWORD_WARN + NO_PASSWORD_INSTRUCTIONS + NO_PASSWORD_CONT;
}
@Override
public String noPasswordError() {
return NO_PASSWORD_ERROR + NO_PASSWORD_INSTRUCTIONS;
}
@Override @Override
public KeyVersion getKeyVersion(String versionName) throws IOException { public KeyVersion getKeyVersion(String versionName) throws IOException {
readLock.lock(); readLock.lock();

32
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
View File

@ -607,4 +607,36 @@ public static KeyProvider findProvider(List<KeyProvider> providerList,
} }
throw new IOException("Can't find KeyProvider for key " + keyName); throw new IOException("Can't find KeyProvider for key " + keyName);
} }
/**
* Does this provider require a password? This means that a password is
* required for normal operation, and it has not been found through normal
* means. If true, the password should be provided by the caller using
* setPassword().
* @return Whether or not the provider requires a password
* @throws IOException
*/
public boolean needsPassword() throws IOException {
return false;
}
/**
* If a password for the provider is needed, but is not provided, this will
* return a warning and instructions for supplying said password to the
* provider.
* @return A warning and instructions for supplying the password
*/
public String noPasswordWarning() {
return null;
}
/**
* If a password for the provider is needed, but is not provided, this will
* return an error message and instructions for supplying said password to
* the provider.
* @return An error message and instructions for supplying the password
*/
public String noPasswordError() {
return null;
}
} }

126
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
View File

@ -26,6 +26,7 @@
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.crypto.key.KeyProvider.Metadata; import org.apache.hadoop.crypto.key.KeyProvider.Metadata;
@ -46,14 +47,22 @@ public class KeyShell extends Configured implements Tool {
" [" + DeleteCommand.USAGE + "]\n" + " [" + DeleteCommand.USAGE + "]\n" +
" [" + ListCommand.USAGE + "]\n"; " [" + ListCommand.USAGE + "]\n";
private static final String LIST_METADATA = "keyShell.list.metadata"; private static final String LIST_METADATA = "keyShell.list.metadata";
@VisibleForTesting public static final String NO_VALID_PROVIDERS =
"There are no valid (non-transient) providers configured.\n" +
"No action has been taken. Use the -provider option to specify\n" +
"a provider. If you want to use a transient provider then you\n" +
"MUST use the -provider argument.";
private boolean interactive = true; private boolean interactive = true;
private Command command = null; private Command command = null;
/** allows stdout to be captured if necessary */ /** If true, fail if the provider requires a password and none is given. */
public PrintStream out = System.out; private boolean strict = false;
/** allows stderr to be captured if necessary */
public PrintStream err = System.err; /** allows stdout to be captured if necessary. */
@VisibleForTesting public PrintStream out = System.out;
/** allows stderr to be captured if necessary. */
@VisibleForTesting public PrintStream err = System.err;
private boolean userSuppliedProvider = false; private boolean userSuppliedProvider = false;
@ -76,7 +85,7 @@ public int run(String[] args) throws Exception {
return exitCode; return exitCode;
} }
if (command.validate()) { if (command.validate()) {
command.execute(); command.execute();
} else { } else {
exitCode = 1; exitCode = 1;
} }
@ -88,7 +97,7 @@ public int run(String[] args) throws Exception {
} }
/** /**
* Parse the command line arguments and initialize the data * Parse the command line arguments and initialize the data.
* <pre> * <pre>
* % hadoop key create keyName [-size size] [-cipher algorithm] * % hadoop key create keyName [-size size] [-cipher algorithm]
* [-provider providerPath] * [-provider providerPath]
@ -171,6 +180,8 @@ private int init(String[] args) throws IOException {
getConf().setBoolean(LIST_METADATA, true); getConf().setBoolean(LIST_METADATA, true);
} else if ("-f".equals(args[i]) || ("-force".equals(args[i]))) { } else if ("-f".equals(args[i]) || ("-force".equals(args[i]))) {
interactive = false; interactive = false;
} else if (args[i].equals("-strict")) {
strict = true;
} else if ("-help".equals(args[i])) { } else if ("-help".equals(args[i])) {
printKeyShellUsage(); printKeyShellUsage();
return 1; return 1;
@ -199,7 +210,7 @@ private void printKeyShellUsage() {
out.println(command.getUsage()); out.println(command.getUsage());
} else { } else {
out.println("=========================================================" + out.println("=========================================================" +
"======"); "======");
out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC); out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC);
out.println("=========================================================" + out.println("=========================================================" +
"======"); "======");
@ -221,16 +232,16 @@ public boolean validate() {
} }
protected KeyProvider getKeyProvider() { protected KeyProvider getKeyProvider() {
KeyProvider provider = null; KeyProvider prov = null;
List<KeyProvider> providers; List<KeyProvider> providers;
try { try {
providers = KeyProviderFactory.getProviders(getConf()); providers = KeyProviderFactory.getProviders(getConf());
if (userSuppliedProvider) { if (userSuppliedProvider) {
provider = providers.get(0); prov = providers.get(0);
} else { } else {
for (KeyProvider p : providers) { for (KeyProvider p : providers) {
if (!p.isTransient()) { if (!p.isTransient()) {
provider = p; prov = p;
break; break;
} }
} }
@ -238,11 +249,14 @@ protected KeyProvider getKeyProvider() {
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(err); e.printStackTrace(err);
} }
return provider; if (prov == null) {
out.println(NO_VALID_PROVIDERS);
}
return prov;
} }
protected void printProviderWritten() { protected void printProviderWritten() {
out.println(provider + " has been updated."); out.println(provider + " has been updated.");
} }
protected void warnIfTransientProvider() { protected void warnIfTransientProvider() {
@ -258,12 +272,13 @@ protected void warnIfTransientProvider() {
private class ListCommand extends Command { private class ListCommand extends Command {
public static final String USAGE = public static final String USAGE =
"list [-provider <provider>] [-metadata] [-help]"; "list [-provider <provider>] [-strict] [-metadata] [-help]";
public static final String DESC = public static final String DESC =
"The list subcommand displays the keynames contained within\n" + "The list subcommand displays the keynames contained within\n" +
"a particular provider as configured in core-site.xml or\n" + "a particular provider as configured in core-site.xml or\n" +
"specified with the -provider argument. -metadata displays\n" + "specified with the -provider argument. -metadata displays\n" +
"the metadata."; "the metadata. If -strict is supplied, fail immediately if\n" +
"the provider requires a password and none is given.";
private boolean metadata = false; private boolean metadata = false;
@ -271,10 +286,6 @@ public boolean validate() {
boolean rc = true; boolean rc = true;
provider = getKeyProvider(); provider = getKeyProvider();
if (provider == null) { if (provider == null) {
out.println("There are no non-transient KeyProviders configured.\n"
+ "Use the -provider option to specify a provider. If you\n"
+ "want to list a transient provider then you must use the\n"
+ "-provider argument.");
rc = false; rc = false;
} }
metadata = getConf().getBoolean(LIST_METADATA, false); metadata = getConf().getBoolean(LIST_METADATA, false);
@ -310,12 +321,15 @@ public String getUsage() {
} }
private class RollCommand extends Command { private class RollCommand extends Command {
public static final String USAGE = "roll <keyname> [-provider <provider>] [-help]"; public static final String USAGE =
"roll <keyname> [-provider <provider>] [-strict] [-help]";
public static final String DESC = public static final String DESC =
"The roll subcommand creates a new version for the specified key\n" + "The roll subcommand creates a new version for the specified key\n" +
"within the provider indicated using the -provider argument\n"; "within the provider indicated using the -provider argument.\n" +
"If -strict is supplied, fail immediately if the provider requires\n" +
"a password and none is given.";
String keyName = null; private String keyName = null;
public RollCommand(String keyName) { public RollCommand(String keyName) {
this.keyName = keyName; this.keyName = keyName;
@ -325,14 +339,11 @@ public boolean validate() {
boolean rc = true; boolean rc = true;
provider = getKeyProvider(); provider = getKeyProvider();
if (provider == null) { if (provider == null) {
out.println("There are no valid KeyProviders configured. The key\n" +
"has not been rolled. Use the -provider option to specify\n" +
"a provider.");
rc = false; rc = false;
} }
if (keyName == null) { if (keyName == null) {
out.println("Please provide a <keyname>.\n" + out.println("Please provide a <keyname>.\n" +
"See the usage description by using -help."); "See the usage description by using -help.");
rc = false; rc = false;
} }
return rc; return rc;
@ -368,15 +379,17 @@ public String getUsage() {
private class DeleteCommand extends Command { private class DeleteCommand extends Command {
public static final String USAGE = public static final String USAGE =
"delete <keyname> [-provider <provider>] [-f] [-help]"; "delete <keyname> [-provider <provider>] [-strict] [-f] [-help]";
public static final String DESC = public static final String DESC =
"The delete subcommand deletes all versions of the key\n" + "The delete subcommand deletes all versions of the key\n" +
"specified by the <keyname> argument from within the\n" + "specified by the <keyname> argument from within the\n" +
"provider specified by -provider. The command asks for\n" + "provider specified by -provider. The command asks for\n" +
"user confirmation unless -f is specified."; "user confirmation unless -f is specified. If -strict is\n" +
"supplied, fail immediately if the provider requires a\n" +
"password and none is given.";
String keyName = null; private String keyName = null;
boolean cont = true; private boolean cont = true;
public DeleteCommand(String keyName) { public DeleteCommand(String keyName) {
this.keyName = keyName; this.keyName = keyName;
@ -386,8 +399,6 @@ public DeleteCommand(String keyName) {
public boolean validate() { public boolean validate() {
provider = getKeyProvider(); provider = getKeyProvider();
if (provider == null) { if (provider == null) {
out.println("There are no valid KeyProviders configured. Nothing\n"
+ "was deleted. Use the -provider option to specify a provider.");
return false; return false;
} }
if (keyName == null) { if (keyName == null) {
@ -438,22 +449,23 @@ public String getUsage() {
private class CreateCommand extends Command { private class CreateCommand extends Command {
public static final String USAGE = public static final String USAGE =
"create <keyname> [-cipher <cipher>] [-size <size>]\n" + "create <keyname> [-cipher <cipher>] [-size <size>]\n" +
" [-description <description>]\n" + " [-description <description>]\n" +
" [-attr <attribute=value>]\n" + " [-attr <attribute=value>]\n" +
" [-provider <provider>] [-help]"; " [-provider <provider>] [-strict]\n" +
" [-help]";
public static final String DESC = public static final String DESC =
"The create subcommand creates a new key for the name specified\n" + "The create subcommand creates a new key for the name specified\n" +
"by the <keyname> argument within the provider specified by the\n" + "by the <keyname> argument within the provider specified by the\n" +
"-provider argument. You may specify a cipher with the -cipher\n" + "-provider argument. You may specify a cipher with the -cipher\n" +
"argument. The default cipher is currently \"AES/CTR/NoPadding\".\n" + "argument. The default cipher is currently \"AES/CTR/NoPadding\".\n" +
"The default keysize is 128. You may specify the requested key\n" + "The default keysize is 128. You may specify the requested key\n" +
"length using the -size argument. Arbitrary attribute=value\n" + "length using the -size argument. Arbitrary attribute=value\n" +
"style attributes may be specified using the -attr argument.\n" + "style attributes may be specified using the -attr argument.\n" +
"-attr may be specified multiple times, once per attribute.\n"; "-attr may be specified multiple times, once per attribute.\n";
final String keyName; private final String keyName;
final Options options; private final Options options;
public CreateCommand(String keyName, Options options) { public CreateCommand(String keyName, Options options) {
this.keyName = keyName; this.keyName = keyName;
@ -462,16 +474,24 @@ public CreateCommand(String keyName, Options options) {
public boolean validate() { public boolean validate() {
boolean rc = true; boolean rc = true;
provider = getKeyProvider(); try {
if (provider == null) { provider = getKeyProvider();
out.println("There are no valid KeyProviders configured. No key\n" + if (provider == null) {
" was created. You can use the -provider option to specify\n" + rc = false;
" a provider to use."); } else if (provider.needsPassword()) {
rc = false; if (strict) {
out.println(provider.noPasswordError());
rc = false;
} else {
out.println(provider.noPasswordWarning());
}
}
} catch (IOException e) {
e.printStackTrace(err);
} }
if (keyName == null) { if (keyName == null) {
out.println("Please provide a <keyname>. See the usage description" + out.println("Please provide a <keyname>. See the usage description" +
" with -help."); " with -help.");
rc = false; rc = false;
} }
return rc; return rc;

150
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.security.alias; package org.apache.hadoop.security.alias;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
@ -34,6 +35,7 @@
import java.io.OutputStream; import java.io.OutputStream;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.net.URL;
import java.security.GeneralSecurityException;
import java.security.KeyStore; import java.security.KeyStore;
import java.security.KeyStoreException; import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
@ -70,60 +72,28 @@ public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
private Path path; private Path path;
private final URI uri; private final URI uri;
private final KeyStore keyStore; private KeyStore keyStore;
private char[] password = null; private char[] password = null;
private boolean changed = false; private boolean changed = false;
private Lock readLock; private Lock readLock;
private Lock writeLock; private Lock writeLock;
private final Configuration conf;
protected AbstractJavaKeyStoreProvider(URI uri, Configuration conf) protected AbstractJavaKeyStoreProvider(URI uri, Configuration conf)
throws IOException { throws IOException {
this.uri = uri; this.uri = uri;
initFileSystem(uri, conf); this.conf = conf;
// Get the password from the user's environment initFileSystem(uri);
if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) { locateKeystore();
password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
}
// if not in ENV get check for file
if (password == null) {
String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
if (pwFile != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL pwdFile = cl.getResource(pwFile);
if (pwdFile != null) {
try (InputStream is = pwdFile.openStream()) {
password = IOUtils.toString(is).trim().toCharArray();
}
}
}
}
if (password == null) {
password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
}
try {
keyStore = KeyStore.getInstance("jceks");
if (keystoreExists()) {
stashOriginalFilePermissions();
try (InputStream in = getInputStreamForFile()) {
keyStore.load(in, password);
}
} else {
createPermissions("700");
// required to create an empty keystore. *sigh*
keyStore.load(null, password);
}
} catch (KeyStoreException e) {
throw new IOException("Can't create keystore", e);
} catch (NoSuchAlgorithmException e) {
throw new IOException("Can't load keystore " + getPathAsString(), e);
} catch (CertificateException e) {
throw new IOException("Can't load keystore " + getPathAsString(), e);
}
ReadWriteLock lock = new ReentrantReadWriteLock(true); ReadWriteLock lock = new ReentrantReadWriteLock(true);
readLock = lock.readLock(); readLock = lock.readLock();
writeLock = lock.writeLock(); writeLock = lock.writeLock();
} }
protected Configuration getConf() {
return conf;
}
public Path getPath() { public Path getPath() {
return path; return path;
} }
@ -189,7 +159,7 @@ protected abstract OutputStream getOutputStreamForKeystore()
protected abstract void stashOriginalFilePermissions() throws IOException; protected abstract void stashOriginalFilePermissions() throws IOException;
protected void initFileSystem(URI keystoreUri, Configuration conf) protected void initFileSystem(URI keystoreUri)
throws IOException { throws IOException {
path = ProviderUtils.unnestUri(keystoreUri); path = ProviderUtils.unnestUri(keystoreUri);
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
@ -332,6 +302,102 @@ public void flush() throws IOException {
} }
} }
/**
* The password is either found in the environment or in a file. This
* routine implements the logic for locating the password in these
* locations.
*
* @return The password as a char []; null if not found.
* @throws IOException
*/
private char[] locatePassword() throws IOException {
char[] pass = null;
if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
pass = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
}
// if not in ENV get check for file
if (pass == null) {
String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
if (pwFile != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL pwdFile = cl.getResource(pwFile);
if (pwdFile != null) {
try (InputStream is = pwdFile.openStream()) {
pass = IOUtils.toString(is).trim().toCharArray();
}
}
}
}
return pass;
}
/**
* Open up and initialize the keyStore.
*
* @throws IOException
*/
private void locateKeystore() throws IOException {
try {
password = locatePassword();
if (password == null) {
password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
}
KeyStore ks;
ks = KeyStore.getInstance("jceks");
if (keystoreExists()) {
stashOriginalFilePermissions();
try (InputStream in = getInputStreamForFile()) {
ks.load(in, password);
}
} else {
createPermissions("600");
// required to create an empty keystore. *sigh*
ks.load(null, password);
}
keyStore = ks;
} catch (KeyStoreException e) {
throw new IOException("Can't create keystore", e);
} catch (GeneralSecurityException e) {
throw new IOException("Can't load keystore " + getPathAsString(), e);
}
}
@Override
public boolean needsPassword() throws IOException {
return (null == locatePassword());
}
@VisibleForTesting
public static final String NO_PASSWORD_WARN =
"WARNING: You have accepted the use of the default provider password\n" +
"by not configuring a password in one of the two following locations:\n";
@VisibleForTesting
public static final String NO_PASSWORD_ERROR =
"ERROR: The provider cannot find a password in the expected " +
"locations.\nPlease supply a password using one of the " +
"following two mechanisms:\n";
@VisibleForTesting
public static final String NO_PASSWORD_INSTRUCTIONS =
" o In the environment variable " +
CREDENTIAL_PASSWORD_NAME + "\n" +
" o In a file referred to by the configuration entry\n" +
" " + KEYSTORE_PASSWORD_FILE_KEY + ".\n" +
"Please review the documentation regarding provider passwords at\n" +
"http://hadoop.apache.org/docs/current/hadoop-project-dist/" +
"hadoop-common/CredentialProviderAPI.html#Keystore_Passwords\n";
@VisibleForTesting public static final String NO_PASSWORD_CONT =
"Continuing with the default provider password.\n";
@Override
public String noPasswordWarning() {
return NO_PASSWORD_WARN + NO_PASSWORD_INSTRUCTIONS + NO_PASSWORD_CONT;
}
@Override
public String noPasswordError() {
return NO_PASSWORD_ERROR + NO_PASSWORD_INSTRUCTIONS;
}
@Override @Override
public String toString() { public String toString() {
return uri.toString(); return uri.toString();

37
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
View File

@ -34,7 +34,7 @@
@InterfaceStability.Unstable @InterfaceStability.Unstable
public abstract class CredentialProvider { public abstract class CredentialProvider {
public static final String CLEAR_TEXT_FALLBACK public static final String CLEAR_TEXT_FALLBACK
= "hadoop.security.credential.clear-text-fallback"; = "hadoop.security.credential.clear-text-fallback";
/** /**
* The combination of both the alias and the actual credential value. * The combination of both the alias and the actual credential value.
@ -85,7 +85,8 @@ public boolean isTransient() {
} }
/** /**
* Ensures that any changes to the credentials are written to persistent store. * Ensures that any changes to the credentials are written to persistent
* store.
* @throws IOException * @throws IOException
*/ */
public abstract void flush() throws IOException; public abstract void flush() throws IOException;
@ -121,4 +122,36 @@ public abstract CredentialEntry createCredentialEntry(String name,
* @throws IOException * @throws IOException
*/ */
public abstract void deleteCredentialEntry(String name) throws IOException; public abstract void deleteCredentialEntry(String name) throws IOException;
/**
* Does this provider require a password? This means that a password is
* required for normal operation, and it has not been found through normal
* means. If true, the password should be provided by the caller using
* setPassword().
* @return Whether or not the provider requires a password
* @throws IOException
*/
public boolean needsPassword() throws IOException {
return false;
}
/**
* If a password for the provider is needed, but is not provided, this will
* return a warning and instructions for supplying said password to the
* provider.
* @return A warning and instructions for supplying the password
*/
public String noPasswordWarning() {
return null;
}
/**
* If a password for the provider is needed, but is not provided, this will
* return an error message and instructions for supplying said password to
* the provider.
* @return An error message and instructions for supplying the password
*/
public String noPasswordError() {
return null;
}
} }

168
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
View File

@ -26,6 +26,7 @@
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.Tool;
@ -37,24 +38,34 @@
*/ */
public class CredentialShell extends Configured implements Tool { public class CredentialShell extends Configured implements Tool {
final static private String USAGE_PREFIX = "Usage: hadoop credential " + final static private String USAGE_PREFIX = "Usage: hadoop credential " +
"[generic options]\n"; "[generic options]\n";
final static private String COMMANDS = final static private String COMMANDS =
" [--help]\n" + " [-help]\n" +
" [" + CreateCommand.USAGE + "]\n" + " [" + CreateCommand.USAGE + "]\n" +
" [" + DeleteCommand.USAGE + "]\n" + " [" + DeleteCommand.USAGE + "]\n" +
" [" + ListCommand.USAGE + "]\n"; " [" + ListCommand.USAGE + "]\n";
@VisibleForTesting
public static final String NO_VALID_PROVIDERS =
"There are no valid (non-transient) providers configured.\n" +
"No action has been taken. Use the -provider option to specify\n" +
"a provider. If you want to use a transient provider then you\n" +
"MUST use the -provider argument.";
private boolean interactive = true; private boolean interactive = true;
private Command command = null; private Command command = null;
/** allows stdout to be captured if necessary */ /** If true, fail if the provider requires a password and none is given. */
public PrintStream out = System.out; private boolean strict = false;
/** allows stderr to be captured if necessary */
public PrintStream err = System.err; /** Allows stdout to be captured if necessary. */
@VisibleForTesting public PrintStream out = System.out;
/** Allows stderr to be captured if necessary. */
@VisibleForTesting public PrintStream err = System.err;
private boolean userSuppliedProvider = false; private boolean userSuppliedProvider = false;
private String value = null; private String value = null;
private PasswordReader passwordReader; private PasswordReader passwordReader;
private boolean isHelp = false;
@Override @Override
public int run(String[] args) throws Exception { public int run(String[] args) throws Exception {
@ -64,10 +75,12 @@ public int run(String[] args) throws Exception {
if (exitCode != 0) { if (exitCode != 0) {
return exitCode; return exitCode;
} }
if (command.validate()) { if (!isHelp) {
if (command.validate()) {
command.execute(); command.execute();
} else { } else {
exitCode = 1; exitCode = 1;
}
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(err); e.printStackTrace(err);
@ -77,7 +90,7 @@ public int run(String[] args) throws Exception {
} }
/** /**
* Parse the command line arguments and initialize the data * Parse the command line arguments and initialize the data.
* <pre> * <pre>
* % hadoop credential create alias [-provider providerPath] * % hadoop credential create alias [-provider providerPath]
* % hadoop credential list [-provider providerPath] * % hadoop credential list [-provider providerPath]
@ -130,6 +143,8 @@ protected int init(String[] args) throws IOException {
args[++i]); args[++i]);
} else if (args[i].equals("-f") || (args[i].equals("-force"))) { } else if (args[i].equals("-f") || (args[i].equals("-force"))) {
interactive = false; interactive = false;
} else if (args[i].equals("-strict")) {
strict = true;
} else if (args[i].equals("-v") || (args[i].equals("-value"))) { } else if (args[i].equals("-v") || (args[i].equals("-value"))) {
value = args[++i]; value = args[++i];
} else if (args[i].equals("-help")) { } else if (args[i].equals("-help")) {
@ -145,13 +160,13 @@ protected int init(String[] args) throws IOException {
} }
private void printCredShellUsage() { private void printCredShellUsage() {
isHelp = true;
out.println(USAGE_PREFIX + COMMANDS); out.println(USAGE_PREFIX + COMMANDS);
if (command != null) { if (command != null) {
out.println(command.getUsage()); out.println(command.getUsage());
} } else {
else {
out.println("=========================================================" + out.println("=========================================================" +
"======"); "======");
out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC); out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC);
out.println("=========================================================" + out.println("=========================================================" +
"======"); "======");
@ -170,17 +185,16 @@ public boolean validate() {
} }
protected CredentialProvider getCredentialProvider() { protected CredentialProvider getCredentialProvider() {
CredentialProvider provider = null; CredentialProvider prov = null;
List<CredentialProvider> providers; List<CredentialProvider> providers;
try { try {
providers = CredentialProviderFactory.getProviders(getConf()); providers = CredentialProviderFactory.getProviders(getConf());
if (userSuppliedProvider) { if (userSuppliedProvider) {
provider = providers.get(0); prov = providers.get(0);
} } else {
else {
for (CredentialProvider p : providers) { for (CredentialProvider p : providers) {
if (!p.isTransient()) { if (!p.isTransient()) {
provider = p; prov = p;
break; break;
} }
} }
@ -188,11 +202,14 @@ protected CredentialProvider getCredentialProvider() {
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(err); e.printStackTrace(err);
} }
return provider; if (prov == null) {
out.println(NO_VALID_PROVIDERS);
}
return prov;
} }
protected void printProviderWritten() { protected void printProviderWritten() {
out.println(provider.getClass().getName() + " has been updated."); out.println("Provider " + provider.toString() + " has been updated.");
} }
protected void warnIfTransientProvider() { protected void warnIfTransientProvider() {
@ -207,35 +224,32 @@ protected void warnIfTransientProvider() {
} }
private class ListCommand extends Command { private class ListCommand extends Command {
public static final String USAGE = "list [-provider provider-path]"; public static final String USAGE =
"list [-provider provider-path] [-strict]";
public static final String DESC = public static final String DESC =
"The list subcommand displays the aliases contained within \n" + "The list subcommand displays the aliases contained within \n" +
"a particular provider - as configured in core-site.xml or " + "a particular provider - as configured in core-site.xml or\n" +
"indicated\nthrough the -provider argument."; "indicated through the -provider argument. If -strict is supplied,\n" +
"fail immediately if the provider requires a password and none is\n" +
"provided.";
public boolean validate() { public boolean validate() {
boolean rc = true;
provider = getCredentialProvider(); provider = getCredentialProvider();
if (provider == null) { return (provider != null);
out.println("There are no non-transient CredentialProviders configured.\n"
+ "Consider using the -provider option to indicate the provider\n"
+ "to use. If you want to list a transient provider then you\n"
+ "you MUST use the -provider argument.");
rc = false;
}
return rc;
} }
public void execute() throws IOException { public void execute() throws IOException {
List<String> aliases; List<String> aliases;
try { try {
aliases = provider.getAliases(); aliases = provider.getAliases();
out.println("Listing aliases for CredentialProvider: " + provider.toString()); out.println("Listing aliases for CredentialProvider: " +
provider.toString());
for (String alias : aliases) { for (String alias : aliases) {
out.println(alias); out.println(alias);
} }
} catch (IOException e) { } catch (IOException e) {
out.println("Cannot list aliases for CredentialProvider: " + provider.toString() out.println("Cannot list aliases for CredentialProvider: " +
provider.toString()
+ ": " + e.getMessage()); + ": " + e.getMessage());
throw e; throw e;
} }
@ -249,15 +263,17 @@ public String getUsage() {
private class DeleteCommand extends Command { private class DeleteCommand extends Command {
public static final String USAGE = public static final String USAGE =
"delete <alias> [-f] [-provider provider-path]"; "delete <alias> [-f] [-provider provider-path] [-strict]";
public static final String DESC = public static final String DESC =
"The delete subcommand deletes the credential\n" + "The delete subcommand deletes the credential\n" +
"specified as the <alias> argument from within the provider\n" + "specified as the <alias> argument from within the provider\n" +
"indicated through the -provider argument. The command asks for\n" + "indicated through the -provider argument. The command asks for\n" +
"confirmation unless the -f option is specified."; "confirmation unless the -f option is specified. If -strict is\n" +
"supplied, fail immediately if the provider requires a password\n" +
"and none is given.";
String alias = null; private String alias = null;
boolean cont = true; private boolean cont = true;
public DeleteCommand(String alias) { public DeleteCommand(String alias) {
this.alias = alias; this.alias = alias;
@ -267,10 +283,6 @@ public DeleteCommand(String alias) {
public boolean validate() { public boolean validate() {
provider = getCredentialProvider(); provider = getCredentialProvider();
if (provider == null) { if (provider == null) {
out.println("There are no valid CredentialProviders configured.\n"
+ "Nothing will be deleted.\n"
+ "Consider using the -provider option to indicate the provider"
+ " to use.");
return false; return false;
} }
if (alias == null) { if (alias == null) {
@ -298,16 +310,17 @@ public boolean validate() {
public void execute() throws IOException { public void execute() throws IOException {
warnIfTransientProvider(); warnIfTransientProvider();
out.println("Deleting credential: " + alias + " from CredentialProvider: " out.println("Deleting credential: " + alias +
+ provider.toString()); " from CredentialProvider: " + provider.toString());
if (cont) { if (cont) {
try { try {
provider.deleteCredentialEntry(alias); provider.deleteCredentialEntry(alias);
out.println(alias + " has been successfully deleted."); out.println("Credential " + alias +
" has been successfully deleted.");
provider.flush(); provider.flush();
printProviderWritten(); printProviderWritten();
} catch (IOException e) { } catch (IOException e) {
out.println(alias + " has NOT been deleted."); out.println("Credential " + alias + " has NOT been deleted.");
throw e; throw e;
} }
} }
@ -320,14 +333,17 @@ public String getUsage() {
} }
private class CreateCommand extends Command { private class CreateCommand extends Command {
public static final String USAGE = public static final String USAGE = "create <alias> [-value alias-value] " +
"create <alias> [-provider provider-path]"; "[-provider provider-path] [-strict]";
public static final String DESC = public static final String DESC =
"The create subcommand creates a new credential for the name specified\n" + "The create subcommand creates a new credential for the name\n" +
"as the <alias> argument within the provider indicated through\n" + "specified as the <alias> argument within the provider indicated\n" +
"the -provider argument."; "through the -provider argument. If -strict is supplied, fail\n" +
"immediately if the provider requires a password and none is given.\n" +
"If -value is provided, use that for the value of the credential\n" +
"instead of prompting the user.";
String alias = null; private String alias = null;
public CreateCommand(String alias) { public CreateCommand(String alias) {
this.alias = alias; this.alias = alias;
@ -335,13 +351,20 @@ public CreateCommand(String alias) {
public boolean validate() { public boolean validate() {
boolean rc = true; boolean rc = true;
provider = getCredentialProvider(); try {
if (provider == null) { provider = getCredentialProvider();
out.println("There are no valid CredentialProviders configured." + if (provider == null) {
"\nCredential will not be created.\n" rc = false;
+ "Consider using the -provider option to indicate the provider" + } else if (provider.needsPassword()) {
" to use."); if (strict) {
rc = false; out.println(provider.noPasswordError());
rc = false;
} else {
out.println(provider.noPasswordWarning());
}
}
} catch (IOException e) {
e.printStackTrace(err);
} }
if (alias == null) { if (alias == null) {
out.println("There is no alias specified. Please provide the" + out.println("There is no alias specified. Please provide the" +
@ -358,19 +381,20 @@ public void execute() throws IOException, NoSuchAlgorithmException {
if (value != null) { if (value != null) {
// testing only // testing only
credential = value.toCharArray(); credential = value.toCharArray();
} } else {
else { credential = promptForCredential();
credential = promptForCredential();
} }
provider.createCredentialEntry(alias, credential); provider.createCredentialEntry(alias, credential);
out.println(alias + " has been successfully created.");
provider.flush(); provider.flush();
out.println(alias + " has been successfully created.");
printProviderWritten(); printProviderWritten();
} catch (InvalidParameterException e) { } catch (InvalidParameterException e) {
out.println(alias + " has NOT been created. " + e.getMessage()); out.println("Credential " + alias + " has NOT been created. " +
e.getMessage());
throw e; throw e;
} catch (IOException e) { } catch (IOException e) {
out.println(alias + " has NOT been created. " + e.getMessage()); out.println("Credential " + alias + " has NOT been created. " +
e.getMessage());
throw e; throw e;
} }
} }
@ -391,16 +415,20 @@ protected char[] promptForCredential() throws IOException {
boolean noMatch; boolean noMatch;
do { do {
char[] newPassword1 = c.readPassword("Enter password: "); char[] newPassword1 = c.readPassword("Enter alias password: ");
char[] newPassword2 = c.readPassword("Enter password again: "); char[] newPassword2 = c.readPassword("Enter alias password again: ");
noMatch = !Arrays.equals(newPassword1, newPassword2); noMatch = !Arrays.equals(newPassword1, newPassword2);
if (noMatch) { if (noMatch) {
if (newPassword1 != null) Arrays.fill(newPassword1, ' '); if (newPassword1 != null) {
Arrays.fill(newPassword1, ' ');
}
c.format("Passwords don't match. Try again.%n"); c.format("Passwords don't match. Try again.%n");
} else { } else {
cred = newPassword1; cred = newPassword1;
} }
if (newPassword2 != null) Arrays.fill(newPassword2, ' '); if (newPassword2 != null) {
Arrays.fill(newPassword2, ' ');
}
} while (noMatch); } while (noMatch);
return cred; return cred;
} }
@ -416,7 +444,7 @@ public void setPasswordReader(PasswordReader reader) {
passwordReader = reader; passwordReader = reader;
} }
// to facilitate testing since Console is a final class... /** To facilitate testing since Console is a final class. */
public static class PasswordReader { public static class PasswordReader {
public char[] readPassword(String prompt) { public char[] readPassword(String prompt) {
Console console = System.console(); Console console = System.console();

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
View File

@ -83,10 +83,10 @@ protected void stashOriginalFilePermissions() throws IOException {
permissions = s.getPermission(); permissions = s.getPermission();
} }
protected void initFileSystem(URI uri, Configuration conf) protected void initFileSystem(URI uri)
throws IOException { throws IOException {
super.initFileSystem(uri, conf); super.initFileSystem(uri);
fs = getPath().getFileSystem(conf); fs = getPath().getFileSystem(getConf());
} }
/** /**

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
View File

@ -121,9 +121,9 @@ protected void stashOriginalFilePermissions() throws IOException {
} }
@Override @Override
protected void initFileSystem(URI uri, Configuration conf) protected void initFileSystem(URI uri)
throws IOException { throws IOException {
super.initFileSystem(uri, conf); super.initFileSystem(uri);
try { try {
file = new File(new URI(getPath().toString())); file = new File(new URI(getPath().toString()));
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {

18
hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
View File

@ -89,9 +89,9 @@ Usage: `hadoop credential <subcommand> [options]`
| COMMAND\_OPTION | Description | | COMMAND\_OPTION | Description |
|:---- |:---- | |:---- |:---- |
| create *alias* [-provider *provider-path*] | Prompts the user for a credential to be stored as the given alias. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. | | create *alias* [-provider *provider-path*] [-strict] [-value *credential-value*] | Prompts the user for a credential to be stored as the given alias. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. The `-strict` flag will cause the command to fail if the provider uses a default password. Use `-value` flag to supply the credential value (a.k.a. the alias password) instead of being prompted. |
| delete *alias* [-provider *provider-path*] [-f] | Deletes the credential with the provided alias. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. The command asks for confirmation unless `-f` is specified | | delete *alias* [-provider *provider-path*] [-strict] [-f] | Deletes the credential with the provided alias. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. The `-strict` flag will cause the command to fail if the provider uses a default password. The command asks for confirmation unless `-f` is specified |
| list [-provider *provider-path*] | Lists all of the credential aliases The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. | | list [-provider *provider-path*] [-strict] | Lists all of the credential aliases The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. The `-strict` flag will cause the command to fail if the provider uses a default password. |
Command to manage credentials, passwords and secrets within credential providers. Command to manage credentials, passwords and secrets within credential providers.
@ -101,6 +101,8 @@ indicates that the current user's credentials file should be consulted through t
When utilizing the credential command it will often be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used. Otherwise, given a path of multiple providers, the first non-transient provider will be used. This may or may not be the one that you intended. When utilizing the credential command it will often be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used. Otherwise, given a path of multiple providers, the first non-transient provider will be used. This may or may not be the one that you intended.
Providers frequently require that a password or other secret is supplied. If the provider requires a password and is unable to find one, it will use a default password and emit a warning message that the default password is being used. If the `-strict` flag is supplied, the warning message becomes an error message and the command returns immediately with an error status.
Example: `hadoop credential list -provider jceks://file/tmp/test.jceks` Example: `hadoop credential list -provider jceks://file/tmp/test.jceks`
### `distcp` ### `distcp`
@ -125,14 +127,16 @@ Usage: `hadoop key <subcommand> [options]`
| COMMAND\_OPTION | Description | | COMMAND\_OPTION | Description |
|:---- |:---- | |:---- |:---- |
| create *keyname* [-cipher *cipher*] [-size *size*] [-description *description*] [-attr *attribute=value*] [-provider *provider*] [-help] | Creates a new key for the name specified by the *keyname* argument within the provider specified by the `-provider` argument. You may specify a cipher with the `-cipher` argument. The default cipher is currently "AES/CTR/NoPadding". The default keysize is 128. You may specify the requested key length using the `-size` argument. Arbitrary attribute=value style attributes may be specified using the `-attr` argument. `-attr` may be specified multiple times, once per attribute. | | create *keyname* [-cipher *cipher*] [-size *size*] [-description *description*] [-attr *attribute=value*] [-provider *provider*] [-strict] [-help] | Creates a new key for the name specified by the *keyname* argument within the provider specified by the `-provider` argument. The `-strict` flag will cause the command to fail if the provider uses a default password. You may specify a cipher with the `-cipher` argument. The default cipher is currently "AES/CTR/NoPadding". The default keysize is 128. You may specify the requested key length using the `-size` argument. Arbitrary attribute=value style attributes may be specified using the `-attr` argument. `-attr` may be specified multiple times, once per attribute. |
| roll *keyname* [-provider *provider*] [-help] | Creates a new version for the specified key within the provider indicated using the `-provider` argument | | roll *keyname* [-provider *provider*] [-strict] [-help] | Creates a new version for the specified key within the provider indicated using the `-provider` argument. The `-strict` flag will cause the command to fail if the provider uses a default password. |
| delete *keyname* [-provider *provider*] [-f] [-help] | Deletes all versions of the key specified by the *keyname* argument from within the provider specified by `-provider`. The command asks for user confirmation unless `-f` is specified. | | delete *keyname* [-provider *provider*] [-strict] [-f] [-help] | Deletes all versions of the key specified by the *keyname* argument from within the provider specified by `-provider`. The `-strict` flag will cause the command to fail if the provider uses a default password. The command asks for user confirmation unless `-f` is specified. |
| list [-provider *provider*] [-metadata] [-help] | Displays the keynames contained within a particular provider as configured in core-site.xml or specified with the `-provider` argument. `-metadata` displays the metadata. | | list [-provider *provider*] [-strict] [-metadata] [-help] | Displays the keynames contained within a particular provider as configured in core-site.xml or specified with the `-provider` argument. The `-strict` flag will cause the command to fail if the provider uses a default password. `-metadata` displays the metadata. |
| -help | Prints usage of this command | | -help | Prints usage of this command |
Manage keys via the KeyProvider. For details on KeyProviders, see the [Transparent Encryption Guide](../hadoop-hdfs/TransparentEncryption.html). Manage keys via the KeyProvider. For details on KeyProviders, see the [Transparent Encryption Guide](../hadoop-hdfs/TransparentEncryption.html).
Providers frequently require that a password or other secret is supplied. If the provider requires a password and is unable to find one, it will use a default password and emit a warning message that the default password is being used. If the `-strict` flag is supplied, the warning message becomes an error message and the command returns immediately with an error status.
NOTE: Some KeyProviders (e.g. org.apache.hadoop.crypto.key.JavaKeyStoreProvider) does not support uppercase key names. NOTE: Some KeyProviders (e.g. org.apache.hadoop.crypto.key.JavaKeyStoreProvider) does not support uppercase key names.
### `trace` ### `trace`

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
View File

@ -267,7 +267,7 @@ public void testJksProvider() throws Exception {
Path path = ProviderUtils.unnestUri(new URI(ourUrl)); Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf); FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path); FileStatus s = fs.getFileStatus(path);
assertTrue(s.getPermission().toString().equals("rwx------")); assertTrue(s.getPermission().toString().equals("rw-------"));
assertTrue(file + " should exist", file.isFile()); assertTrue(file + " should exist", file.isFile());
// Corrupt file and Check if JKS can reload from _OLD file // Corrupt file and Check if JKS can reload from _OLD file

32
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
View File

@ -114,6 +114,12 @@ public void testKeySuccessfulKeyLifecycle() throws Exception {
assertEquals(0, rc); assertEquals(0, rc);
assertTrue(outContent.toString().contains(keyName + " has been " + assertTrue(outContent.toString().contains(keyName + " has been " +
"successfully created")); "successfully created"));
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_WARN));
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_CONT));
String listOut = listKeys(ks, false); String listOut = listKeys(ks, false);
assertTrue(listOut.contains(keyName)); assertTrue(listOut.contains(keyName));
@ -128,7 +134,7 @@ public void testKeySuccessfulKeyLifecycle() throws Exception {
rc = ks.run(args2); rc = ks.run(args2);
assertEquals(0, rc); assertEquals(0, rc);
assertTrue(outContent.toString().contains("key1 has been successfully " + assertTrue(outContent.toString().contains("key1 has been successfully " +
"rolled.")); "rolled."));
deleteKey(ks, keyName); deleteKey(ks, keyName);
@ -191,8 +197,7 @@ public void testInvalidProvider() throws Exception {
ks.setConf(new Configuration()); ks.setConf(new Configuration());
rc = ks.run(args1); rc = ks.run(args1);
assertEquals(1, rc); assertEquals(1, rc);
assertTrue(outContent.toString().contains("There are no valid " + assertTrue(outContent.toString().contains(KeyShell.NO_VALID_PROVIDERS));
"KeyProviders configured."));
} }
@Test @Test
@ -206,7 +211,7 @@ public void testTransientProviderWarning() throws Exception {
rc = ks.run(args1); rc = ks.run(args1);
assertEquals(0, rc); assertEquals(0, rc);
assertTrue(outContent.toString().contains("WARNING: you are modifying a " + assertTrue(outContent.toString().contains("WARNING: you are modifying a " +
"transient provider.")); "transient provider."));
} }
@Test @Test
@ -220,8 +225,23 @@ public void testTransientProviderOnlyConfig() throws Exception {
ks.setConf(config); ks.setConf(config);
rc = ks.run(args1); rc = ks.run(args1);
assertEquals(1, rc); assertEquals(1, rc);
assertTrue(outContent.toString().contains("There are no valid " + assertTrue(outContent.toString().contains(KeyShell.NO_VALID_PROVIDERS));
"KeyProviders configured.")); }
@Test
public void testStrict() throws Exception {
outContent.reset();
int rc = 0;
KeyShell ks = new KeyShell();
ks.setConf(new Configuration());
final String[] args1 = {"create", "hello", "-provider", jceksProvider,
"-strict"};
rc = ks.run(args1);
assertEquals(1, rc);
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_ERROR));
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
} }
@Test @Test

55
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
View File

@ -63,6 +63,12 @@ public void testCredentialSuccessfulLifecycle() throws Exception {
assertEquals(outContent.toString(), 0, rc); assertEquals(outContent.toString(), 0, rc);
assertTrue(outContent.toString().contains("credential1 has been successfully " + assertTrue(outContent.toString().contains("credential1 has been successfully " +
"created.")); "created."));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_WARN));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_CONT));
outContent.reset(); outContent.reset();
String[] args2 = {"list", "-provider", String[] args2 = {"list", "-provider",
@ -97,8 +103,8 @@ public void testInvalidProvider() throws Exception {
cs.setConf(new Configuration()); cs.setConf(new Configuration());
rc = cs.run(args1); rc = cs.run(args1);
assertEquals(1, rc); assertEquals(1, rc);
assertTrue(outContent.toString().contains("There are no valid " + assertTrue(outContent.toString().contains(
"CredentialProviders configured.")); CredentialShell.NO_VALID_PROVIDERS));
} }
@Test @Test
@ -132,8 +138,8 @@ public void testTransientProviderOnlyConfig() throws Exception {
cs.setConf(config); cs.setConf(config);
rc = cs.run(args1); rc = cs.run(args1);
assertEquals(1, rc); assertEquals(1, rc);
assertTrue(outContent.toString().contains("There are no valid " + assertTrue(outContent.toString().contains(
"CredentialProviders configured.")); CredentialShell.NO_VALID_PROVIDERS));
} }
@Test @Test
@ -225,6 +231,47 @@ public void testEmptyArgForCommands() throws Exception {
assertEquals("Expected empty argument on " + cmd + " to return 1", 1, assertEquals("Expected empty argument on " + cmd + " to return 1", 1,
shell.init(new String[] { cmd })); shell.init(new String[] { cmd }));
} }
}
@Test
public void testStrict() throws Exception {
outContent.reset();
String[] args1 = {"create", "credential1", "-value", "p@ssw0rd",
"-provider", jceksProvider, "-strict"};
int rc = 1;
CredentialShell cs = new CredentialShell();
cs.setConf(new Configuration());
rc = cs.run(args1);
assertEquals(outContent.toString(), 1, rc);
assertFalse(outContent.toString().contains("credential1 has been " +
"successfully created."));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_ERROR));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
}
@Test
public void testHelp() throws Exception {
outContent.reset();
String[] args1 = {"-help"};
int rc = 0;
CredentialShell cs = new CredentialShell();
cs.setConf(new Configuration());
rc = cs.run(args1);
assertEquals(outContent.toString(), 0, rc);
assertTrue(outContent.toString().contains("Usage"));
}
@Test
public void testHelpCreate() throws Exception {
outContent.reset();
String[] args1 = {"create", "-help"};
int rc = 0;
CredentialShell cs = new CredentialShell();
cs.setConf(new Configuration());
rc = cs.run(args1);
assertEquals(outContent.toString(), 0, rc);
assertTrue(outContent.toString().contains("Usage"));
} }
} }

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
View File

@ -214,7 +214,7 @@ public void testJksProvider() throws Exception {
Path path = ProviderUtils.unnestUri(new URI(ourUrl)); Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf); FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path); FileStatus s = fs.getFileStatus(path);
assertTrue(s.getPermission().toString().equals("rwx------")); assertTrue(s.getPermission().toString().equals("rw-------"));
assertTrue(file + " should exist", file.isFile()); assertTrue(file + " should exist", file.isFile());
// check permission retention after explicit change // check permission retention after explicit change
@ -236,7 +236,8 @@ public void testLocalJksProvider() throws Exception {
Path path = ProviderUtils.unnestUri(new URI(ourUrl)); Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf); FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path); FileStatus s = fs.getFileStatus(path);
assertTrue("Unexpected permissions: " + s.getPermission().toString(), s.getPermission().toString().equals("rwx------")); assertTrue("Unexpected permissions: " + s.getPermission().toString(),
s.getPermission().toString().equals("rw-------"));
assertTrue(file + " should exist", file.isFile()); assertTrue(file + " should exist", file.isFile());
// check permission retention after explicit change // check permission retention after explicit change