Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12942. hadoop credential commands non-obviously use password of "none" (Mike Yoder via lmccay)

This commit is contained in:
Larry McCay 2016-05-11 14:38:21 -04:00
parent e9ee258d04
commit ee2c2afdc3
13 changed files with 515 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
View File

@ -44,6 +44,7 @@ import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.net.URI;
import java.net.URL;
import java.security.GeneralSecurityException;
import java.security.Key;
import java.security.KeyStore;
import java.security.KeyStoreException;
@ -88,7 +89,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
@InterfaceAudience.Private
public class JavaKeyStoreProvider extends KeyProvider {
private static final String KEY_METADATA = "KeyMetadata";
private static Logger LOG =
private static final Logger LOG =
LoggerFactory.getLogger(JavaKeyStoreProvider.class);
public static final String SCHEME_NAME = "jceks";
@ -103,8 +104,8 @@ public class JavaKeyStoreProvider extends KeyProvider {
private final URI uri;
private final Path path;
private final FileSystem fs;
private final FsPermission permissions;
private final KeyStore keyStore;
private FsPermission permissions;
private KeyStore keyStore;
private char[] password;
private boolean changed = false;
private Lock readLock;
@ -131,13 +132,28 @@ public class JavaKeyStoreProvider extends KeyProvider {
this.uri = uri;
path = ProviderUtils.unnestUri(uri);
fs = path.getFileSystem(conf);
locateKeystore();
ReadWriteLock lock = new ReentrantReadWriteLock(true);
readLock = lock.readLock();
writeLock = lock.writeLock();
}
/**
* The password is either found in the environment or in a file. This
* routine implements the logic for locating the password in these
* locations.
* @return The password as a char []; null if not found.
* @throws IOException
*/
private char[] locatePassword() throws IOException {
char[] pass = null;
// Get the password file from the conf, if not present from the user's
// environment var
if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) {
password = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray();
pass = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray();
}
if (password == null) {
String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
if (pass == null) {
String pwFile = getConf().get(KEYSTORE_PASSWORD_FILE_KEY);
if (pwFile != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL pwdFile = cl.getResource(pwFile);
@ -146,14 +162,23 @@ public class JavaKeyStoreProvider extends KeyProvider {
throw new IOException("Password file does not exists");
}
try (InputStream is = pwdFile.openStream()) {
password = IOUtils.toString(is).trim().toCharArray();
pass = IOUtils.toString(is).trim().toCharArray();
}
}
}
return pass;
}
/**
* Open up and initialize the keyStore.
* @throws IOException
*/
private void locateKeystore() throws IOException {
try {
password = locatePassword();
if (password == null) {
password = KEYSTORE_PASSWORD_DEFAULT;
}
try {
Path oldPath = constructOldPath(path);
Path newPath = constructNewPath(path);
keyStore = KeyStore.getInstance(SCHEME_NAME);
@ -175,19 +200,14 @@ public class JavaKeyStoreProvider extends KeyProvider {
permissions = perm;
} catch (KeyStoreException e) {
throw new IOException("Can't create keystore", e);
} catch (NoSuchAlgorithmException e) {
throw new IOException("Can't load keystore " + path, e);
} catch (CertificateException e) {
} catch (GeneralSecurityException e) {
throw new IOException("Can't load keystore " + path, e);
}
ReadWriteLock lock = new ReentrantReadWriteLock(true);
readLock = lock.readLock();
writeLock = lock.writeLock();
}
/**
* Try loading from the user specified path, else load from the backup
* path in case Exception is not due to bad/wrong password
* path in case Exception is not due to bad/wrong password.
* @param path Actual path to load from
* @param backupPath Backup path (_OLD)
* @return The permissions of the loaded file
@ -256,7 +276,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
if (perm == null) {
keyStore.load(null, password);
LOG.debug("KeyStore initialized anew successfully !!");
perm = new FsPermission("700");
perm = new FsPermission("600");
}
return perm;
}
@ -321,6 +341,40 @@ public class JavaKeyStoreProvider extends KeyProvider {
return oldPath;
}
@Override
public boolean needsPassword() throws IOException {
return (null == locatePassword());
}
@VisibleForTesting
public static final String NO_PASSWORD_WARN =
"WARNING: You have accepted the use of the default provider password\n" +
"by not configuring a password in one of the two following locations:\n";
public static final String NO_PASSWORD_ERROR =
"ERROR: The provider cannot find a password in the expected " +
"locations.\nPlease supply a password using one of the " +
"following two mechanisms:\n";
@VisibleForTesting public static final String NO_PASSWORD_INSTRUCTIONS =
" o In the environment variable " +
KEYSTORE_PASSWORD_ENV_VAR + "\n" +
" o In a file referred to by the configuration entry\n" +
" " + KEYSTORE_PASSWORD_FILE_KEY + ".\n" +
"Please review the documentation regarding provider passwords at\n" +
"http://hadoop.apache.org/docs/current/hadoop-project-dist/" +
"hadoop-common/CredentialProviderAPI.html#Keystore_Passwords\n";
@VisibleForTesting public static final String NO_PASSWORD_CONT =
"Continuing with the default provider password.\n";
@Override
public String noPasswordWarning() {
return NO_PASSWORD_WARN + NO_PASSWORD_INSTRUCTIONS + NO_PASSWORD_CONT;
}
@Override
public String noPasswordError() {
return NO_PASSWORD_ERROR + NO_PASSWORD_INSTRUCTIONS;
}
@Override
public KeyVersion getKeyVersion(String versionName) throws IOException {
readLock.lock();

32
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
View File

@ -607,4 +607,36 @@ public abstract class KeyProvider {
}
throw new IOException("Can't find KeyProvider for key " + keyName);
}
/**
* Does this provider require a password? This means that a password is
* required for normal operation, and it has not been found through normal
* means. If true, the password should be provided by the caller using
* setPassword().
* @return Whether or not the provider requires a password
* @throws IOException
*/
public boolean needsPassword() throws IOException {
return false;
}
/**
* If a password for the provider is needed, but is not provided, this will
* return a warning and instructions for supplying said password to the
* provider.
* @return A warning and instructions for supplying the password
*/
public String noPasswordWarning() {
return null;
}
/**
* If a password for the provider is needed, but is not provided, this will
* return an error message and instructions for supplying said password to
* the provider.
* @return An error message and instructions for supplying the password
*/
public String noPasswordError() {
return null;
}
}

86
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
View File

@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.crypto.key.KeyProvider.Metadata;
@ -46,14 +47,22 @@ public class KeyShell extends Configured implements Tool {
" [" + DeleteCommand.USAGE + "]\n" +
" [" + ListCommand.USAGE + "]\n";
private static final String LIST_METADATA = "keyShell.list.metadata";
@VisibleForTesting public static final String NO_VALID_PROVIDERS =
"There are no valid (non-transient) providers configured.\n" +
"No action has been taken. Use the -provider option to specify\n" +
"a provider. If you want to use a transient provider then you\n" +
"MUST use the -provider argument.";
private boolean interactive = true;
private Command command = null;
/** allows stdout to be captured if necessary */
public PrintStream out = System.out;
/** allows stderr to be captured if necessary */
public PrintStream err = System.err;
/** If true, fail if the provider requires a password and none is given. */
private boolean strict = false;
/** allows stdout to be captured if necessary. */
@VisibleForTesting public PrintStream out = System.out;
/** allows stderr to be captured if necessary. */
@VisibleForTesting public PrintStream err = System.err;
private boolean userSuppliedProvider = false;
@ -88,7 +97,7 @@ public class KeyShell extends Configured implements Tool {
}
/**
* Parse the command line arguments and initialize the data
* Parse the command line arguments and initialize the data.
* <pre>
* % hadoop key create keyName [-size size] [-cipher algorithm]
* [-provider providerPath]
@ -171,6 +180,8 @@ public class KeyShell extends Configured implements Tool {
getConf().setBoolean(LIST_METADATA, true);
} else if ("-f".equals(args[i]) || ("-force".equals(args[i]))) {
interactive = false;
} else if (args[i].equals("-strict")) {
strict = true;
} else if ("-help".equals(args[i])) {
printKeyShellUsage();
return 1;
@ -221,16 +232,16 @@ public class KeyShell extends Configured implements Tool {
}
protected KeyProvider getKeyProvider() {
KeyProvider provider = null;
KeyProvider prov = null;
List<KeyProvider> providers;
try {
providers = KeyProviderFactory.getProviders(getConf());
if (userSuppliedProvider) {
provider = providers.get(0);
prov = providers.get(0);
} else {
for (KeyProvider p : providers) {
if (!p.isTransient()) {
provider = p;
prov = p;
break;
}
}
@ -238,7 +249,10 @@ public class KeyShell extends Configured implements Tool {
} catch (IOException e) {
e.printStackTrace(err);
}
return provider;
if (prov == null) {
out.println(NO_VALID_PROVIDERS);
}
return prov;
}
protected void printProviderWritten() {
@ -258,12 +272,13 @@ public class KeyShell extends Configured implements Tool {
private class ListCommand extends Command {
public static final String USAGE =
"list [-provider <provider>] [-metadata] [-help]";
"list [-provider <provider>] [-strict] [-metadata] [-help]";
public static final String DESC =
"The list subcommand displays the keynames contained within\n" +
"a particular provider as configured in core-site.xml or\n" +
"specified with the -provider argument. -metadata displays\n" +
"the metadata.";
"the metadata. If -strict is supplied, fail immediately if\n" +
"the provider requires a password and none is given.";
private boolean metadata = false;
@ -271,10 +286,6 @@ public class KeyShell extends Configured implements Tool {
boolean rc = true;
provider = getKeyProvider();
if (provider == null) {
out.println("There are no non-transient KeyProviders configured.\n"
+ "Use the -provider option to specify a provider. If you\n"
+ "want to list a transient provider then you must use the\n"
+ "-provider argument.");
rc = false;
}
metadata = getConf().getBoolean(LIST_METADATA, false);
@ -310,12 +321,15 @@ public class KeyShell extends Configured implements Tool {
}
private class RollCommand extends Command {
public static final String USAGE = "roll <keyname> [-provider <provider>] [-help]";
public static final String USAGE =
"roll <keyname> [-provider <provider>] [-strict] [-help]";
public static final String DESC =
"The roll subcommand creates a new version for the specified key\n" +
"within the provider indicated using the -provider argument\n";
"within the provider indicated using the -provider argument.\n" +
"If -strict is supplied, fail immediately if the provider requires\n" +
"a password and none is given.";
String keyName = null;
private String keyName = null;
public RollCommand(String keyName) {
this.keyName = keyName;
@ -325,9 +339,6 @@ public class KeyShell extends Configured implements Tool {
boolean rc = true;
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured. The key\n" +
"has not been rolled. Use the -provider option to specify\n" +
"a provider.");
rc = false;
}
if (keyName == null) {
@ -368,15 +379,17 @@ public class KeyShell extends Configured implements Tool {
private class DeleteCommand extends Command {
public static final String USAGE =
"delete <keyname> [-provider <provider>] [-f] [-help]";
"delete <keyname> [-provider <provider>] [-strict] [-f] [-help]";
public static final String DESC =
"The delete subcommand deletes all versions of the key\n" +
"specified by the <keyname> argument from within the\n" +
"provider specified by -provider. The command asks for\n" +
"user confirmation unless -f is specified.";
"user confirmation unless -f is specified. If -strict is\n" +
"supplied, fail immediately if the provider requires a\n" +
"password and none is given.";
String keyName = null;
boolean cont = true;
private String keyName = null;
private boolean cont = true;
public DeleteCommand(String keyName) {
this.keyName = keyName;
@ -386,8 +399,6 @@ public class KeyShell extends Configured implements Tool {
public boolean validate() {
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured. Nothing\n"
+ "was deleted. Use the -provider option to specify a provider.");
return false;
}
if (keyName == null) {
@ -441,7 +452,8 @@ public class KeyShell extends Configured implements Tool {
"create <keyname> [-cipher <cipher>] [-size <size>]\n" +
" [-description <description>]\n" +
" [-attr <attribute=value>]\n" +
" [-provider <provider>] [-help]";
" [-provider <provider>] [-strict]\n" +
" [-help]";
public static final String DESC =
"The create subcommand creates a new key for the name specified\n" +
"by the <keyname> argument within the provider specified by the\n" +
@ -452,8 +464,8 @@ public class KeyShell extends Configured implements Tool {
"style attributes may be specified using the -attr argument.\n" +
"-attr may be specified multiple times, once per attribute.\n";
final String keyName;
final Options options;
private final String keyName;
private final Options options;
public CreateCommand(String keyName, Options options) {
this.keyName = keyName;
@ -462,12 +474,20 @@ public class KeyShell extends Configured implements Tool {
public boolean validate() {
boolean rc = true;
try {
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured. No key\n" +
" was created. You can use the -provider option to specify\n" +
" a provider to use.");
rc = false;
} else if (provider.needsPassword()) {
if (strict) {
out.println(provider.noPasswordError());
rc = false;
} else {
out.println(provider.noPasswordWarning());
}
}
} catch (IOException e) {
e.printStackTrace(err);
}
if (keyName == null) {
out.println("Please provide a <keyname>. See the usage description" +

150
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.security.alias;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.io.IOUtils;
@ -34,6 +35,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URL;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
@ -70,60 +72,28 @@ public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
private Path path;
private final URI uri;
private final KeyStore keyStore;
private KeyStore keyStore;
private char[] password = null;
private boolean changed = false;
private Lock readLock;
private Lock writeLock;
private final Configuration conf;
protected AbstractJavaKeyStoreProvider(URI uri, Configuration conf)
throws IOException {
this.uri = uri;
initFileSystem(uri, conf);
// Get the password from the user's environment
if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
}
// if not in ENV get check for file
if (password == null) {
String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
if (pwFile != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL pwdFile = cl.getResource(pwFile);
if (pwdFile != null) {
try (InputStream is = pwdFile.openStream()) {
password = IOUtils.toString(is).trim().toCharArray();
}
}
}
}
if (password == null) {
password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
}
try {
keyStore = KeyStore.getInstance("jceks");
if (keystoreExists()) {
stashOriginalFilePermissions();
try (InputStream in = getInputStreamForFile()) {
keyStore.load(in, password);
}
} else {
createPermissions("700");
// required to create an empty keystore. *sigh*
keyStore.load(null, password);
}
} catch (KeyStoreException e) {
throw new IOException("Can't create keystore", e);
} catch (NoSuchAlgorithmException e) {
throw new IOException("Can't load keystore " + getPathAsString(), e);
} catch (CertificateException e) {
throw new IOException("Can't load keystore " + getPathAsString(), e);
}
this.conf = conf;
initFileSystem(uri);
locateKeystore();
ReadWriteLock lock = new ReentrantReadWriteLock(true);
readLock = lock.readLock();
writeLock = lock.writeLock();
}
protected Configuration getConf() {
return conf;
}
public Path getPath() {
return path;
}
@ -189,7 +159,7 @@ public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
protected abstract void stashOriginalFilePermissions() throws IOException;
protected void initFileSystem(URI keystoreUri, Configuration conf)
protected void initFileSystem(URI keystoreUri)
throws IOException {
path = ProviderUtils.unnestUri(keystoreUri);
if (LOG.isDebugEnabled()) {
@ -332,6 +302,102 @@ public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
}
}
/**
* The password is either found in the environment or in a file. This
* routine implements the logic for locating the password in these
* locations.
*
* @return The password as a char []; null if not found.
* @throws IOException
*/
private char[] locatePassword() throws IOException {
char[] pass = null;
if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
pass = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
}
// if not in ENV get check for file
if (pass == null) {
String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
if (pwFile != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL pwdFile = cl.getResource(pwFile);
if (pwdFile != null) {
try (InputStream is = pwdFile.openStream()) {
pass = IOUtils.toString(is).trim().toCharArray();
}
}
}
}
return pass;
}
/**
* Open up and initialize the keyStore.
*
* @throws IOException
*/
private void locateKeystore() throws IOException {
try {
password = locatePassword();
if (password == null) {
password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
}
KeyStore ks;
ks = KeyStore.getInstance("jceks");
if (keystoreExists()) {
stashOriginalFilePermissions();
try (InputStream in = getInputStreamForFile()) {
ks.load(in, password);
}
} else {
createPermissions("600");
// required to create an empty keystore. *sigh*
ks.load(null, password);
}
keyStore = ks;
} catch (KeyStoreException e) {
throw new IOException("Can't create keystore", e);
} catch (GeneralSecurityException e) {
throw new IOException("Can't load keystore " + getPathAsString(), e);
}
}
@Override
public boolean needsPassword() throws IOException {
return (null == locatePassword());
}
@VisibleForTesting
public static final String NO_PASSWORD_WARN =
"WARNING: You have accepted the use of the default provider password\n" +
"by not configuring a password in one of the two following locations:\n";
@VisibleForTesting
public static final String NO_PASSWORD_ERROR =
"ERROR: The provider cannot find a password in the expected " +
"locations.\nPlease supply a password using one of the " +
"following two mechanisms:\n";
@VisibleForTesting
public static final String NO_PASSWORD_INSTRUCTIONS =
" o In the environment variable " +
CREDENTIAL_PASSWORD_NAME + "\n" +
" o In a file referred to by the configuration entry\n" +
" " + KEYSTORE_PASSWORD_FILE_KEY + ".\n" +
"Please review the documentation regarding provider passwords at\n" +
"http://hadoop.apache.org/docs/current/hadoop-project-dist/" +
"hadoop-common/CredentialProviderAPI.html#Keystore_Passwords\n";
@VisibleForTesting public static final String NO_PASSWORD_CONT =
"Continuing with the default provider password.\n";
@Override
public String noPasswordWarning() {
return NO_PASSWORD_WARN + NO_PASSWORD_INSTRUCTIONS + NO_PASSWORD_CONT;
}
@Override
public String noPasswordError() {
return NO_PASSWORD_ERROR + NO_PASSWORD_INSTRUCTIONS;
}
@Override
public String toString() {
return uri.toString();

35
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
View File

@ -85,7 +85,8 @@ public abstract class CredentialProvider {
}
/**
* Ensures that any changes to the credentials are written to persistent store.
* Ensures that any changes to the credentials are written to persistent
* store.
* @throws IOException
*/
public abstract void flush() throws IOException;
@ -121,4 +122,36 @@ public abstract class CredentialProvider {
* @throws IOException
*/
public abstract void deleteCredentialEntry(String name) throws IOException;
/**
* Does this provider require a password? This means that a password is
* required for normal operation, and it has not been found through normal
* means. If true, the password should be provided by the caller using
* setPassword().
* @return Whether or not the provider requires a password
* @throws IOException
*/
public boolean needsPassword() throws IOException {
return false;
}
/**
* If a password for the provider is needed, but is not provided, this will
* return a warning and instructions for supplying said password to the
* provider.
* @return A warning and instructions for supplying the password
*/
public String noPasswordWarning() {
return null;
}
/**
* If a password for the provider is needed, but is not provided, this will
* return an error message and instructions for supplying said password to
* the provider.
* @return An error message and instructions for supplying the password
*/
public String noPasswordError() {
return null;
}
}

150
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
View File

@ -26,6 +26,7 @@ import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
@ -39,22 +40,32 @@ public class CredentialShell extends Configured implements Tool {
final static private String USAGE_PREFIX = "Usage: hadoop credential " +
"[generic options]\n";
final static private String COMMANDS =
" [--help]\n" +
" [-help]\n" +
" [" + CreateCommand.USAGE + "]\n" +
" [" + DeleteCommand.USAGE + "]\n" +
" [" + ListCommand.USAGE + "]\n";
@VisibleForTesting
public static final String NO_VALID_PROVIDERS =
"There are no valid (non-transient) providers configured.\n" +
"No action has been taken. Use the -provider option to specify\n" +
"a provider. If you want to use a transient provider then you\n" +
"MUST use the -provider argument.";
private boolean interactive = true;
private Command command = null;
/** allows stdout to be captured if necessary */
public PrintStream out = System.out;
/** allows stderr to be captured if necessary */
public PrintStream err = System.err;
/** If true, fail if the provider requires a password and none is given. */
private boolean strict = false;
/** Allows stdout to be captured if necessary. */
@VisibleForTesting public PrintStream out = System.out;
/** Allows stderr to be captured if necessary. */
@VisibleForTesting public PrintStream err = System.err;
private boolean userSuppliedProvider = false;
private String value = null;
private PasswordReader passwordReader;
private boolean isHelp = false;
@Override
public int run(String[] args) throws Exception {
@ -64,11 +75,13 @@ public class CredentialShell extends Configured implements Tool {
if (exitCode != 0) {
return exitCode;
}
if (!isHelp) {
if (command.validate()) {
command.execute();
} else {
exitCode = 1;
}
}
} catch (Exception e) {
e.printStackTrace(err);
return 1;
@ -77,7 +90,7 @@ public class CredentialShell extends Configured implements Tool {
}
/**
* Parse the command line arguments and initialize the data
* Parse the command line arguments and initialize the data.
* <pre>
* % hadoop credential create alias [-provider providerPath]
* % hadoop credential list [-provider providerPath]
@ -130,6 +143,8 @@ public class CredentialShell extends Configured implements Tool {
args[++i]);
} else if (args[i].equals("-f") || (args[i].equals("-force"))) {
interactive = false;
} else if (args[i].equals("-strict")) {
strict = true;
} else if (args[i].equals("-v") || (args[i].equals("-value"))) {
value = args[++i];
} else if (args[i].equals("-help")) {
@ -145,11 +160,11 @@ public class CredentialShell extends Configured implements Tool {
}
private void printCredShellUsage() {
isHelp = true;
out.println(USAGE_PREFIX + COMMANDS);
if (command != null) {
out.println(command.getUsage());
}
else {
} else {
out.println("=========================================================" +
"======");
out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC);
@ -170,17 +185,16 @@ public class CredentialShell extends Configured implements Tool {
}
protected CredentialProvider getCredentialProvider() {
CredentialProvider provider = null;
CredentialProvider prov = null;
List<CredentialProvider> providers;
try {
providers = CredentialProviderFactory.getProviders(getConf());
if (userSuppliedProvider) {
provider = providers.get(0);
}
else {
prov = providers.get(0);
} else {
for (CredentialProvider p : providers) {
if (!p.isTransient()) {
provider = p;
prov = p;
break;
}
}
@ -188,11 +202,14 @@ public class CredentialShell extends Configured implements Tool {
} catch (IOException e) {
e.printStackTrace(err);
}
return provider;
if (prov == null) {
out.println(NO_VALID_PROVIDERS);
}
return prov;
}
protected void printProviderWritten() {
out.println(provider.getClass().getName() + " has been updated.");
out.println("Provider " + provider.toString() + " has been updated.");
}
protected void warnIfTransientProvider() {
@ -207,35 +224,32 @@ public class CredentialShell extends Configured implements Tool {
}
private class ListCommand extends Command {
public static final String USAGE = "list [-provider provider-path]";
public static final String USAGE =
"list [-provider provider-path] [-strict]";
public static final String DESC =
"The list subcommand displays the aliases contained within \n" +
"a particular provider - as configured in core-site.xml or " +
"indicated\nthrough the -provider argument.";
"a particular provider - as configured in core-site.xml or\n" +
"indicated through the -provider argument. If -strict is supplied,\n" +
"fail immediately if the provider requires a password and none is\n" +
"provided.";
public boolean validate() {
boolean rc = true;
provider = getCredentialProvider();
if (provider == null) {
out.println("There are no non-transient CredentialProviders configured.\n"
+ "Consider using the -provider option to indicate the provider\n"
+ "to use. If you want to list a transient provider then you\n"
+ "you MUST use the -provider argument.");
rc = false;
}
return rc;
return (provider != null);
}
public void execute() throws IOException {
List<String> aliases;
try {
aliases = provider.getAliases();
out.println("Listing aliases for CredentialProvider: " + provider.toString());
out.println("Listing aliases for CredentialProvider: " +
provider.toString());
for (String alias : aliases) {
out.println(alias);
}
} catch (IOException e) {
out.println("Cannot list aliases for CredentialProvider: " + provider.toString()
out.println("Cannot list aliases for CredentialProvider: " +
provider.toString()
+ ": " + e.getMessage());
throw e;
}
@ -249,15 +263,17 @@ public class CredentialShell extends Configured implements Tool {
private class DeleteCommand extends Command {
public static final String USAGE =
"delete <alias> [-f] [-provider provider-path]";
"delete <alias> [-f] [-provider provider-path] [-strict]";
public static final String DESC =
"The delete subcommand deletes the credential\n" +
"specified as the <alias> argument from within the provider\n" +
"indicated through the -provider argument. The command asks for\n" +
"confirmation unless the -f option is specified.";
"confirmation unless the -f option is specified. If -strict is\n" +
"supplied, fail immediately if the provider requires a password\n" +
"and none is given.";
String alias = null;
boolean cont = true;
private String alias = null;
private boolean cont = true;
public DeleteCommand(String alias) {
this.alias = alias;
@ -267,10 +283,6 @@ public class CredentialShell extends Configured implements Tool {
public boolean validate() {
provider = getCredentialProvider();
if (provider == null) {
out.println("There are no valid CredentialProviders configured.\n"
+ "Nothing will be deleted.\n"
+ "Consider using the -provider option to indicate the provider"
+ " to use.");
return false;
}
if (alias == null) {
@ -298,16 +310,17 @@ public class CredentialShell extends Configured implements Tool {
public void execute() throws IOException {
warnIfTransientProvider();
out.println("Deleting credential: " + alias + " from CredentialProvider: "
+ provider.toString());
out.println("Deleting credential: " + alias +
" from CredentialProvider: " + provider.toString());
if (cont) {
try {
provider.deleteCredentialEntry(alias);
out.println(alias + " has been successfully deleted.");
out.println("Credential " + alias +
" has been successfully deleted.");
provider.flush();
printProviderWritten();
} catch (IOException e) {
out.println(alias + " has NOT been deleted.");
out.println("Credential " + alias + " has NOT been deleted.");
throw e;
}
}
@ -320,14 +333,17 @@ public class CredentialShell extends Configured implements Tool {
}
private class CreateCommand extends Command {
public static final String USAGE =
"create <alias> [-provider provider-path]";
public static final String USAGE = "create <alias> [-value alias-value] " +
"[-provider provider-path] [-strict]";
public static final String DESC =
"The create subcommand creates a new credential for the name specified\n" +
"as the <alias> argument within the provider indicated through\n" +
"the -provider argument.";
"The create subcommand creates a new credential for the name\n" +
"specified as the <alias> argument within the provider indicated\n" +
"through the -provider argument. If -strict is supplied, fail\n" +
"immediately if the provider requires a password and none is given.\n" +
"If -value is provided, use that for the value of the credential\n" +
"instead of prompting the user.";
String alias = null;
private String alias = null;
public CreateCommand(String alias) {
this.alias = alias;
@ -335,13 +351,20 @@ public class CredentialShell extends Configured implements Tool {
public boolean validate() {
boolean rc = true;
try {
provider = getCredentialProvider();
if (provider == null) {
out.println("There are no valid CredentialProviders configured." +
"\nCredential will not be created.\n"
+ "Consider using the -provider option to indicate the provider" +
" to use.");
rc = false;
} else if (provider.needsPassword()) {
if (strict) {
out.println(provider.noPasswordError());
rc = false;
} else {
out.println(provider.noPasswordWarning());
}
}
} catch (IOException e) {
e.printStackTrace(err);
}
if (alias == null) {
out.println("There is no alias specified. Please provide the" +
@ -358,19 +381,20 @@ public class CredentialShell extends Configured implements Tool {
if (value != null) {
// testing only
credential = value.toCharArray();
}
else {
} else {
credential = promptForCredential();
}
provider.createCredentialEntry(alias, credential);
out.println(alias + " has been successfully created.");
provider.flush();
out.println(alias + " has been successfully created.");
printProviderWritten();
} catch (InvalidParameterException e) {
out.println(alias + " has NOT been created. " + e.getMessage());
out.println("Credential " + alias + " has NOT been created. " +
e.getMessage());
throw e;
} catch (IOException e) {
out.println(alias + " has NOT been created. " + e.getMessage());
out.println("Credential " + alias + " has NOT been created. " +
e.getMessage());
throw e;
}
}
@ -391,16 +415,20 @@ public class CredentialShell extends Configured implements Tool {
boolean noMatch;
do {
char[] newPassword1 = c.readPassword("Enter password: ");
char[] newPassword2 = c.readPassword("Enter password again: ");
char[] newPassword1 = c.readPassword("Enter alias password: ");
char[] newPassword2 = c.readPassword("Enter alias password again: ");
noMatch = !Arrays.equals(newPassword1, newPassword2);
if (noMatch) {
if (newPassword1 != null) Arrays.fill(newPassword1, ' ');
if (newPassword1 != null) {
Arrays.fill(newPassword1, ' ');
}
c.format("Passwords don't match. Try again.%n");
} else {
cred = newPassword1;
}
if (newPassword2 != null) Arrays.fill(newPassword2, ' ');
if (newPassword2 != null) {
Arrays.fill(newPassword2, ' ');
}
} while (noMatch);
return cred;
}
@ -416,7 +444,7 @@ public class CredentialShell extends Configured implements Tool {
passwordReader = reader;
}
// to facilitate testing since Console is a final class...
/** To facilitate testing since Console is a final class. */
public static class PasswordReader {
public char[] readPassword(String prompt) {
Console console = System.console();

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
View File

@ -83,10 +83,10 @@ public class JavaKeyStoreProvider extends AbstractJavaKeyStoreProvider {
permissions = s.getPermission();
}
protected void initFileSystem(URI uri, Configuration conf)
protected void initFileSystem(URI uri)
throws IOException {
super.initFileSystem(uri, conf);
fs = getPath().getFileSystem(conf);
super.initFileSystem(uri);
fs = getPath().getFileSystem(getConf());
}
/**

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
View File

@ -121,9 +121,9 @@ public final class LocalJavaKeyStoreProvider extends
}
@Override
protected void initFileSystem(URI uri, Configuration conf)
protected void initFileSystem(URI uri)
throws IOException {
super.initFileSystem(uri, conf);
super.initFileSystem(uri);
try {
file = new File(new URI(getPath().toString()));
if (LOG.isDebugEnabled()) {

18
hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
View File

@ -89,9 +89,9 @@ Usage: `hadoop credential <subcommand> [options]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
| create *alias* [-provider *provider-path*] | Prompts the user for a credential to be stored as the given alias. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. |
| delete *alias* [-provider *provider-path*] [-f] | Deletes the credential with the provided alias. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. The command asks for confirmation unless `-f` is specified |
| list [-provider *provider-path*] | Lists all of the credential aliases The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. |
| create *alias* [-provider *provider-path*] [-strict] [-value *credential-value*] | Prompts the user for a credential to be stored as the given alias. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. The `-strict` flag will cause the command to fail if the provider uses a default password. Use `-value` flag to supply the credential value (a.k.a. the alias password) instead of being prompted. |
| delete *alias* [-provider *provider-path*] [-strict] [-f] | Deletes the credential with the provided alias. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. The `-strict` flag will cause the command to fail if the provider uses a default password. The command asks for confirmation unless `-f` is specified |
| list [-provider *provider-path*] [-strict] | Lists all of the credential aliases The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. The `-strict` flag will cause the command to fail if the provider uses a default password. |
Command to manage credentials, passwords and secrets within credential providers.
@ -101,6 +101,8 @@ indicates that the current user's credentials file should be consulted through t
When utilizing the credential command it will often be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used. Otherwise, given a path of multiple providers, the first non-transient provider will be used. This may or may not be the one that you intended.
Providers frequently require that a password or other secret is supplied. If the provider requires a password and is unable to find one, it will use a default password and emit a warning message that the default password is being used. If the `-strict` flag is supplied, the warning message becomes an error message and the command returns immediately with an error status.
Example: `hadoop credential list -provider jceks://file/tmp/test.jceks`
### `distcp`
@ -125,14 +127,16 @@ Usage: `hadoop key <subcommand> [options]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
| create *keyname* [-cipher *cipher*] [-size *size*] [-description *description*] [-attr *attribute=value*] [-provider *provider*] [-help] | Creates a new key for the name specified by the *keyname* argument within the provider specified by the `-provider` argument. You may specify a cipher with the `-cipher` argument. The default cipher is currently "AES/CTR/NoPadding". The default keysize is 128. You may specify the requested key length using the `-size` argument. Arbitrary attribute=value style attributes may be specified using the `-attr` argument. `-attr` may be specified multiple times, once per attribute. |
| roll *keyname* [-provider *provider*] [-help] | Creates a new version for the specified key within the provider indicated using the `-provider` argument |
| delete *keyname* [-provider *provider*] [-f] [-help] | Deletes all versions of the key specified by the *keyname* argument from within the provider specified by `-provider`. The command asks for user confirmation unless `-f` is specified. |
| list [-provider *provider*] [-metadata] [-help] | Displays the keynames contained within a particular provider as configured in core-site.xml or specified with the `-provider` argument. `-metadata` displays the metadata. |
| create *keyname* [-cipher *cipher*] [-size *size*] [-description *description*] [-attr *attribute=value*] [-provider *provider*] [-strict] [-help] | Creates a new key for the name specified by the *keyname* argument within the provider specified by the `-provider` argument. The `-strict` flag will cause the command to fail if the provider uses a default password. You may specify a cipher with the `-cipher` argument. The default cipher is currently "AES/CTR/NoPadding". The default keysize is 128. You may specify the requested key length using the `-size` argument. Arbitrary attribute=value style attributes may be specified using the `-attr` argument. `-attr` may be specified multiple times, once per attribute. |
| roll *keyname* [-provider *provider*] [-strict] [-help] | Creates a new version for the specified key within the provider indicated using the `-provider` argument. The `-strict` flag will cause the command to fail if the provider uses a default password. |
| delete *keyname* [-provider *provider*] [-strict] [-f] [-help] | Deletes all versions of the key specified by the *keyname* argument from within the provider specified by `-provider`. The `-strict` flag will cause the command to fail if the provider uses a default password. The command asks for user confirmation unless `-f` is specified. |
| list [-provider *provider*] [-strict] [-metadata] [-help] | Displays the keynames contained within a particular provider as configured in core-site.xml or specified with the `-provider` argument. The `-strict` flag will cause the command to fail if the provider uses a default password. `-metadata` displays the metadata. |
| -help | Prints usage of this command |
Manage keys via the KeyProvider. For details on KeyProviders, see the [Transparent Encryption Guide](../hadoop-hdfs/TransparentEncryption.html).
Providers frequently require that a password or other secret is supplied. If the provider requires a password and is unable to find one, it will use a default password and emit a warning message that the default password is being used. If the `-strict` flag is supplied, the warning message becomes an error message and the command returns immediately with an error status.
NOTE: Some KeyProviders (e.g. org.apache.hadoop.crypto.key.JavaKeyStoreProvider) does not support uppercase key names.
### `trace`

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
View File

@ -267,7 +267,7 @@ public class TestKeyProviderFactory {
Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path);
assertTrue(s.getPermission().toString().equals("rwx------"));
assertTrue(s.getPermission().toString().equals("rw-------"));
assertTrue(file + " should exist", file.isFile());
// Corrupt file and Check if JKS can reload from _OLD file

28
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
View File

@ -114,6 +114,12 @@ public class TestKeyShell {
assertEquals(0, rc);
assertTrue(outContent.toString().contains(keyName + " has been " +
"successfully created"));
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_WARN));
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_CONT));
String listOut = listKeys(ks, false);
assertTrue(listOut.contains(keyName));
@ -191,8 +197,7 @@ public class TestKeyShell {
ks.setConf(new Configuration());
rc = ks.run(args1);
assertEquals(1, rc);
assertTrue(outContent.toString().contains("There are no valid " +
"KeyProviders configured."));
assertTrue(outContent.toString().contains(KeyShell.NO_VALID_PROVIDERS));
}
@Test
@ -220,8 +225,23 @@ public class TestKeyShell {
ks.setConf(config);
rc = ks.run(args1);
assertEquals(1, rc);
assertTrue(outContent.toString().contains("There are no valid " +
"KeyProviders configured."));
assertTrue(outContent.toString().contains(KeyShell.NO_VALID_PROVIDERS));
}
@Test
public void testStrict() throws Exception {
outContent.reset();
int rc = 0;
KeyShell ks = new KeyShell();
ks.setConf(new Configuration());
final String[] args1 = {"create", "hello", "-provider", jceksProvider,
"-strict"};
rc = ks.run(args1);
assertEquals(1, rc);
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_ERROR));
assertTrue(outContent.toString()
.contains(JavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
}
@Test

55
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
View File

@ -63,6 +63,12 @@ public class TestCredShell {
assertEquals(outContent.toString(), 0, rc);
assertTrue(outContent.toString().contains("credential1 has been successfully " +
"created."));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_WARN));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_CONT));
outContent.reset();
String[] args2 = {"list", "-provider",
@ -97,8 +103,8 @@ public class TestCredShell {
cs.setConf(new Configuration());
rc = cs.run(args1);
assertEquals(1, rc);
assertTrue(outContent.toString().contains("There are no valid " +
"CredentialProviders configured."));
assertTrue(outContent.toString().contains(
CredentialShell.NO_VALID_PROVIDERS));
}
@Test
@ -132,8 +138,8 @@ public class TestCredShell {
cs.setConf(config);
rc = cs.run(args1);
assertEquals(1, rc);
assertTrue(outContent.toString().contains("There are no valid " +
"CredentialProviders configured."));
assertTrue(outContent.toString().contains(
CredentialShell.NO_VALID_PROVIDERS));
}
@Test
@ -225,6 +231,47 @@ public class TestCredShell {
assertEquals("Expected empty argument on " + cmd + " to return 1", 1,
shell.init(new String[] { cmd }));
}
}
@Test
public void testStrict() throws Exception {
outContent.reset();
String[] args1 = {"create", "credential1", "-value", "p@ssw0rd",
"-provider", jceksProvider, "-strict"};
int rc = 1;
CredentialShell cs = new CredentialShell();
cs.setConf(new Configuration());
rc = cs.run(args1);
assertEquals(outContent.toString(), 1, rc);
assertFalse(outContent.toString().contains("credential1 has been " +
"successfully created."));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_ERROR));
assertTrue(outContent.toString()
.contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
}
@Test
public void testHelp() throws Exception {
outContent.reset();
String[] args1 = {"-help"};
int rc = 0;
CredentialShell cs = new CredentialShell();
cs.setConf(new Configuration());
rc = cs.run(args1);
assertEquals(outContent.toString(), 0, rc);
assertTrue(outContent.toString().contains("Usage"));
}
@Test
public void testHelpCreate() throws Exception {
outContent.reset();
String[] args1 = {"create", "-help"};
int rc = 0;
CredentialShell cs = new CredentialShell();
cs.setConf(new Configuration());
rc = cs.run(args1);
assertEquals(outContent.toString(), 0, rc);
assertTrue(outContent.toString().contains("Usage"));
}
}

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
View File

@ -214,7 +214,7 @@ public class TestCredentialProviderFactory {
Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path);
assertTrue(s.getPermission().toString().equals("rwx------"));
assertTrue(s.getPermission().toString().equals("rw-------"));
assertTrue(file + " should exist", file.isFile());
// check permission retention after explicit change
@ -236,7 +236,8 @@ public class TestCredentialProviderFactory {
Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path);
assertTrue("Unexpected permissions: " + s.getPermission().toString(), s.getPermission().toString().equals("rwx------"));
assertTrue("Unexpected permissions: " + s.getPermission().toString(),
s.getPermission().toString().equals("rw-------"));
assertTrue(file + " should exist", file.isFile());
// check permission retention after explicit change