Merge pull request #22335 from rjernst/keystore

Settings: Add infrastructure for elasticsearch keystore

core/build.gradle

@@ -94,6 +94,8 @@ dependencies {
       exclude group: 'org.elasticsearch', module: 'elasticsearch'
     }
   }
+  testCompile 'com.google.jimfs:jimfs:1.1'
+  testCompile 'com.google.guava:guava:18.0'
 }
 
 if (isEclipse) {

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -30,17 +30,16 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.PidFile;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.inject.CreationException;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.IfConfig;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.env.Environment;
@@ -228,13 +227,36 @@ final class Bootstrap {
         };
     }
 
-    private static Environment initialEnvironment(boolean foreground, Path pidFile, Settings initialSettings) {
+    private static KeyStoreWrapper loadKeyStore(Environment initialEnv) throws BootstrapException {
+        final KeyStoreWrapper keystore;
+        try {
+            keystore = KeyStoreWrapper.load(initialEnv.configFile());
+        } catch (IOException e) {
+            throw new BootstrapException(e);
+        }
+        if (keystore == null) {
+            return null; // no keystore
+        }
+
+        try {
+            keystore.decrypt(new char[0] /* TODO: read password from stdin */);
+        } catch (Exception e) {
+            throw new BootstrapException(e);
+        }
+        return keystore;
+    }
+
+    private static Environment createEnvironment(boolean foreground, Path pidFile,
+                                                 KeyStoreWrapper keystore, Settings initialSettings) {
         Terminal terminal = foreground ? Terminal.DEFAULT : null;
         Settings.Builder builder = Settings.builder();
         if (pidFile != null) {
             builder.put(Environment.PIDFILE_SETTING.getKey(), pidFile);
         }
         builder.put(initialSettings);
+        if (keystore != null) {
+            builder.setKeyStore(keystore);
+        }
         return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal, Collections.emptyMap());
     }
 
@@ -265,7 +287,7 @@ final class Bootstrap {
             final boolean foreground,
             final Path pidFile,
             final boolean quiet,
-            final Settings initialSettings) throws BootstrapException, NodeValidationException, UserException {
+            final Environment initialEnv) throws BootstrapException, NodeValidationException, UserException {
         // Set the system property before anything has a chance to trigger its use
         initLoggerPrefix();
 
@@ -275,7 +297,8 @@ final class Bootstrap {
 
         INSTANCE = new Bootstrap();
 
-        Environment environment = initialEnvironment(foreground, pidFile, initialSettings);
+        final KeyStoreWrapper keystore = loadKeyStore(initialEnv);
+        Environment environment = createEnvironment(foreground, pidFile, keystore, initialEnv.settings());
         try {
             LogConfigurator.configure(environment);
         } catch (IOException e) {
@@ -313,6 +336,13 @@ final class Bootstrap {
 
             INSTANCE.setup(true, environment);
 
+            try {
+                // any secure settings must be read during node construction
+                IOUtils.close(keystore);
+            } catch (IOException e) {
+                throw new BootstrapException(e);
+            }
+
             INSTANCE.start();
 
             if (closeStandardStreams) {

core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java

@@ -26,7 +26,7 @@ import java.nio.file.Path;
  * during bootstrap should explicitly declare the checked exceptions that they can throw, rather
  * than declaring the top-level checked exception {@link Exception}. This exception exists to wrap
  * these checked exceptions so that
- * {@link Bootstrap#init(boolean, Path, boolean, org.elasticsearch.common.settings.Settings)}
+ * {@link Bootstrap#init(boolean, Path, boolean, org.elasticsearch.env.Environment)}
  * does not have to declare all of these checked exceptions.
  */
 class BootstrapException extends Exception {

core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -111,16 +111,16 @@ class Elasticsearch extends EnvironmentAwareCommand {
         final boolean quiet = options.has(quietOption);
 
         try {
-            init(daemonize, pidFile, quiet, env.settings());
+            init(daemonize, pidFile, quiet, env);
         } catch (NodeValidationException e) {
             throw new UserException(ExitCodes.CONFIG, e.getMessage());
         }
     }
 
-    void init(final boolean daemonize, final Path pidFile, final boolean quiet, Settings initialSettings)
+    void init(final boolean daemonize, final Path pidFile, final boolean quiet, Environment initialEnv)
         throws NodeValidationException, UserException {
         try {
-            Bootstrap.init(!daemonize, pidFile, quiet, initialSettings);
+            Bootstrap.init(!daemonize, pidFile, quiet, initialEnv);
         } catch (BootstrapException | RuntimeException e) {
             // format exceptions to the console in a special way
             // to avoid 2MB stacktraces from guice, etc.

core/src/main/java/org/elasticsearch/cli/Terminal.java

@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.nio.charset.Charset;
+import java.util.Locale;
 
 /**
  * A Terminal wraps access to reading input and writing output for a cli.
@@ -92,6 +93,26 @@ public abstract class Terminal {
         }
     }
 
+    /**
+     * Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n'
+     * (or the default empty value) is entered.
+     */
+    public final boolean promptYesNo(String prompt, boolean defaultYes) {
+        String answerPrompt = defaultYes ? " [Y/n]" : " [y/N]";
+        while (true) {
+            String answer = readText(prompt + answerPrompt).toLowerCase(Locale.ROOT);
+            if (answer.isEmpty()) {
+                return defaultYes;
+            }
+            boolean answerYes = answer.equals("y");
+            if (answerYes == false && answer.equals("n") == false) {
+                println("Did not understand answer '" + answer + "'");
+                continue;
+            }
+            return answerYes;
+        }
+    }
+
     private static class ConsoleTerminal extends Terminal {
 
         private static final Console CONSOLE = System.console();

core/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli which adds a string setting.
+ */
+class AddStringKeyStoreCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<Void> stdinOption;
+    private final OptionSpec<Void> forceOption;
+    private final OptionSpec<String> arguments;
+
+    AddStringKeyStoreCommand() {
+        super("Add a string setting to the keystore");
+        this.stdinOption = parser.acceptsAll(Arrays.asList("x", "stdin"), "Read setting value from stdin");
+        this.forceOption = parser.acceptsAll(Arrays.asList("f", "force"), "Overwrite existing setting without prompting");
+        this.arguments = parser.nonOptions("setting name");
+    }
+
+    // pkg private so tests can manipulate
+    InputStream getStdin() {
+        return System.in;
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        String setting = arguments.value(options);
+        if (keystore.getSettings().contains(setting) && options.has(forceOption) == false) {
+            if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without modifying keystore.");
+                return;
+            }
+        }
+
+        final char[] value;
+        if (options.has(stdinOption)) {
+            BufferedReader stdinReader = new BufferedReader(new InputStreamReader(getStdin(), StandardCharsets.UTF_8));
+            value = stdinReader.readLine().toCharArray();
+        } else {
+            value = terminal.readSecret("Enter value for " + setting + ": ");
+        }
+
+        try {
+            keystore.setStringSetting(setting, value);
+        } catch (IllegalArgumentException e) {
+            throw new UserException(ExitCodes.DATA_ERROR, "String value must contain only ASCII");
+        }
+        keystore.save(env.configFile());
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/CreateKeyStoreCommand.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to create a new keystore.
+ */
+class CreateKeyStoreCommand extends EnvironmentAwareCommand {
+
+    CreateKeyStoreCommand() {
+        super("Creates a new elasticsearch keystore");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        if (Files.exists(keystoreFile)) {
+            if (terminal.promptYesNo("An elasticsearch keystore already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without creating keystore.");
+                return;
+            }
+        }
+
+
+        char[] password = new char[0];// terminal.readSecret("Enter passphrase (empty for no passphrase): ");
+        /* TODO: uncomment when entering passwords on startup is supported
+        char[] passwordRepeat = terminal.readSecret("Enter same passphrase again: ");
+        if (Arrays.equals(password, passwordRepeat) == false) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Passphrases are not equal, exiting.");
+        }*/
+
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(password);
+        keystore.save(env.configFile());
+        terminal.println("Created elasticsearch keystore in " + env.configFile());
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.Terminal;
+
+/**
+ * A cli tool for managing secrets in the elasticsearch keystore.
+ */
+public class KeyStoreCli extends MultiCommand {
+
+    private KeyStoreCli() {
+        super("A tool for managing settings stored in the elasticsearch keystore");
+        subcommands.put("create", new CreateKeyStoreCommand());
+        subcommands.put("list", new ListKeyStoreCommand());
+        subcommands.put("add", new AddStringKeyStoreCommand());
+        subcommands.put("remove", new RemoveSettingKeyStoreCommand());
+    }
+
+    public static void main(String[] args) throws Exception {
+        exit(new KeyStoreCli().main(args, Terminal.DEFAULT));
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java

@@ -0,0 +1,278 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.security.auth.DestroyFailedException;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.CharBuffer;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.BufferedChecksumIndexInput;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.util.SetOnce;
+
+/**
+ * A wrapper around a Java KeyStore which provides supplements the keystore with extra metadata.
+ *
+ * Loading a keystore has 2 phases. First, call {@link #load(Path)}. Then call
+ * {@link #decrypt(char[])} with the keystore password, or an empty char array if
+ * {@link #hasPassword()} is {@code false}.  Loading and decrypting should happen
+ * in a single thread. Once decrypted, keys may be read with the wrapper in
+ * multiple threads.
+ */
+public class KeyStoreWrapper implements Closeable {
+
+    /** The name of the keystore file to read and write. */
+    private static final String KEYSTORE_FILENAME = "elasticsearch.keystore";
+
+    /** The version of the metadata written before the keystore data. */
+    private static final int FORMAT_VERSION = 1;
+
+    /** The keystore type for a newly created keystore. */
+    private static final String NEW_KEYSTORE_TYPE = "PKCS12";
+
+    /** The algorithm used to store password for a newly created keystore. */
+    private static final String NEW_KEYSTORE_SECRET_KEY_ALGO = "PBE";//"PBEWithHmacSHA256AndAES_128";
+
+    /** An encoder to check whether string values are ascii. */
+    private static final CharsetEncoder ASCII_ENCODER = StandardCharsets.US_ASCII.newEncoder();
+
+    /** True iff the keystore has a password needed to read. */
+    private final boolean hasPassword;
+
+    /** The type of the keystore, as passed to {@link java.security.KeyStore#getInstance(String)} */
+    private final String type;
+
+    /** A factory necessary for constructing instances of secrets in a {@link KeyStore}. */
+    private final SecretKeyFactory secretFactory;
+
+    /** The raw bytes of the encrypted keystore. */
+    private final byte[] keystoreBytes;
+
+    /** The loaded keystore. See {@link #decrypt(char[])}. */
+    private final SetOnce<KeyStore> keystore = new SetOnce<>();
+
+    /** The password for the keystore. See {@link #decrypt(char[])}. */
+    private final SetOnce<KeyStore.PasswordProtection> keystorePassword = new SetOnce<>();
+
+    /** The setting names contained in the loaded keystore. */
+    private final Set<String> settingNames = new HashSet<>();
+
+    private KeyStoreWrapper(boolean hasPassword, String type, String secretKeyAlgo, byte[] keystoreBytes) {
+        this.hasPassword = hasPassword;
+        this.type = type;
+        try {
+            secretFactory = SecretKeyFactory.getInstance(secretKeyAlgo);
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+        this.keystoreBytes = keystoreBytes;
+    }
+
+    /** Returns a path representing the ES keystore in the given config dir. */
+    static Path keystorePath(Path configDir) {
+        return configDir.resolve(KEYSTORE_FILENAME);
+    }
+
+    /** Constructs a new keystore with the given password. */
+    static KeyStoreWrapper create(char[] password) throws Exception {
+        KeyStoreWrapper wrapper = new KeyStoreWrapper(password.length != 0, NEW_KEYSTORE_TYPE, NEW_KEYSTORE_SECRET_KEY_ALGO, null);
+        KeyStore keyStore = KeyStore.getInstance(NEW_KEYSTORE_TYPE);
+        keyStore.load(null, null);
+        wrapper.keystore.set(keyStore);
+        wrapper.keystorePassword.set(new KeyStore.PasswordProtection(password));
+        return wrapper;
+    }
+
+    /**
+     * Loads information about the Elasticsearch keystore from the provided config directory.
+     *
+     * {@link #decrypt(char[])} must be called before reading or writing any entries.
+     * Returns {@code null} if no keystore exists.
+     */
+    public static KeyStoreWrapper load(Path configDir) throws IOException {
+        Path keystoreFile = keystorePath(configDir);
+        if (Files.exists(keystoreFile) == false) {
+            return null;
+        }
+
+        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
+        try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) {
+            ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput);
+            CodecUtil.checkHeader(input, KEYSTORE_FILENAME, FORMAT_VERSION, FORMAT_VERSION);
+            byte hasPasswordByte = input.readByte();
+            boolean hasPassword = hasPasswordByte == 1;
+            if (hasPassword == false && hasPasswordByte != 0) {
+                throw new IllegalStateException("hasPassword boolean is corrupt: "
+                    + String.format(Locale.ROOT, "%02x", hasPasswordByte));
+            }
+            String type = input.readString();
+            String secretKeyAlgo = input.readString();
+            byte[] keystoreBytes = new byte[input.readInt()];
+            input.readBytes(keystoreBytes, 0, keystoreBytes.length);
+            CodecUtil.checkFooter(input);
+            return new KeyStoreWrapper(hasPassword, type, secretKeyAlgo, keystoreBytes);
+        }
+    }
+
+    /** Returns true iff {@link #decrypt(char[])} has been called. */
+    public boolean isLoaded() {
+        return keystore.get() != null;
+    }
+
+    /** Return true iff calling {@link #decrypt(char[])} requires a non-empty password. */
+    public boolean hasPassword() {
+        return hasPassword;
+    }
+
+    /**
+     * Decrypts the underlying java keystore.
+     *
+     * This may only be called once. The provided password will be zeroed out.
+     */
+    public void decrypt(char[] password) throws GeneralSecurityException, IOException {
+        if (keystore.get() != null) {
+            throw new IllegalStateException("Keystore has already been decrypted");
+        }
+        keystore.set(KeyStore.getInstance(type));
+        try (InputStream in = new ByteArrayInputStream(keystoreBytes)) {
+            keystore.get().load(in, password);
+        } finally {
+            Arrays.fill(keystoreBytes, (byte)0);
+        }
+
+        keystorePassword.set(new KeyStore.PasswordProtection(password));
+        Arrays.fill(password, '\0');
+
+        // convert keystore aliases enum into a set for easy lookup
+        Enumeration<String> aliases = keystore.get().aliases();
+        while (aliases.hasMoreElements()) {
+            settingNames.add(aliases.nextElement());
+        }
+    }
+
+    /** Write the keystore to the given config directory. */
+    void save(Path configDir) throws Exception {
+        char[] password = this.keystorePassword.get().getPassword();
+
+        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
+        // write to tmp file first, then overwrite
+        String tmpFile = KEYSTORE_FILENAME + ".tmp";
+        try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) {
+            CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION);
+            output.writeByte(password.length == 0 ? (byte)0 : (byte)1);
+            output.writeString(type);
+            output.writeString(secretFactory.getAlgorithm());
+
+            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
+            keystore.get().store(keystoreBytesStream, password);
+            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
+            output.writeInt(keystoreBytes.length);
+            output.writeBytes(keystoreBytes, keystoreBytes.length);
+            CodecUtil.writeFooter(output);
+        }
+
+        Path keystoreFile = keystorePath(configDir);
+        Files.move(configDir.resolve(tmpFile), keystoreFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+        PosixFileAttributeView attrs = Files.getFileAttributeView(keystoreFile, PosixFileAttributeView.class);
+        if (attrs != null) {
+            // don't rely on umask: ensure the keystore has minimal permissions
+            attrs.setPermissions(PosixFilePermissions.fromString("rw-------"));
+        }
+    }
+
+    /** Returns the names of all settings in this keystore. */
+    public Set<String> getSettings() {
+        return settingNames;
+    }
+
+    // TODO: make settings accessible only to code that registered the setting
+    /** Retrieve a string setting. The {@link SecureString} should be closed once it is used. */
+    SecureString getStringSetting(String setting) throws GeneralSecurityException {
+        KeyStore.Entry entry = keystore.get().getEntry(setting, keystorePassword.get());
+        if (entry instanceof KeyStore.SecretKeyEntry == false) {
+            throw new IllegalStateException("Secret setting " + setting + " is not a string");
+        }
+        // TODO: only allow getting a setting once?
+        KeyStore.SecretKeyEntry secretKeyEntry = (KeyStore.SecretKeyEntry)entry;
+        PBEKeySpec keySpec = (PBEKeySpec) secretFactory.getKeySpec(secretKeyEntry.getSecretKey(), PBEKeySpec.class);
+        SecureString value = new SecureString(keySpec.getPassword());
+        keySpec.clearPassword();
+        return value;
+    }
+
+    /**
+     * Set a string setting.
+     *
+     * @throws IllegalArgumentException if the value is not ASCII
+     */
+    void setStringSetting(String setting, char[] value) throws GeneralSecurityException {
+        if (ASCII_ENCODER.canEncode(CharBuffer.wrap(value)) == false) {
+            throw new IllegalArgumentException("Value must be ascii");
+        }
+        SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec(value));
+        keystore.get().setEntry(setting, new KeyStore.SecretKeyEntry(secretKey), keystorePassword.get());
+        settingNames.add(setting);
+    }
+
+    /** Remove the given setting from the keystore. */
+    void remove(String setting) throws KeyStoreException {
+        keystore.get().deleteEntry(setting);
+        settingNames.remove(setting);
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            if (keystorePassword.get() != null) {
+                keystorePassword.get().destroy();
+            }
+        } catch (DestroyFailedException e) {
+            throw new IOException(e);
+        }
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/ListKeyStoreCommand.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to list all settings in the keystore.
+ */
+class ListKeyStoreCommand extends EnvironmentAwareCommand {
+
+    ListKeyStoreCommand() {
+        super("List entries in the keystore");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        List<String> sortedEntries = new ArrayList<>(keystore.getSettings());
+        Collections.sort(sortedEntries);
+        for (String entry : sortedEntries) {
+            terminal.println(entry);
+        }
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommand.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.util.List;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to remove a setting.
+ */
+class RemoveSettingKeyStoreCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<String> arguments;
+
+    RemoveSettingKeyStoreCommand() {
+        super("Remove a setting from the keystore");
+        arguments = parser.nonOptions("setting names");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        List<String> settings = arguments.values(options);
+        if (settings.isEmpty()) {
+            throw new UserException(ExitCodes.USAGE, "Must supply at least one setting to remove");
+        }
+
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        for (String setting : arguments.values(options)) {
+            if (keystore.getSettings().contains(setting) == false) {
+                throw new UserException(ExitCodes.CONFIG, "Setting [" + setting + "] does not exist in the keystore.");
+            }
+            keystore.remove(setting);
+        }
+        keystore.save(env.configFile());
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+
+/**
+ * A secure setting.
+ *
+ * This class allows access to settings from the Elasticsearch keystore.
+ */
+public abstract class SecureSetting<T> extends Setting<T> {
+    private static final Set<Property> ALLOWED_PROPERTIES = new HashSet<>(
+        Arrays.asList(Property.Deprecated, Property.Shared)
+    );
+
+    private SecureSetting(String key, Setting.Property... properties) {
+        super(key, (String)null, null, properties);
+        assert assertAllowedProperties(properties);
+    }
+
+    private boolean assertAllowedProperties(Setting.Property... properties) {
+        for (Setting.Property property : properties) {
+            if (ALLOWED_PROPERTIES.contains(property) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public String getDefaultRaw(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public T getDefault(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public String getRaw(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public T get(Settings settings) {
+        checkDeprecation(settings);
+        final KeyStoreWrapper keystore = Objects.requireNonNull(settings.getKeyStore());
+        if (keystore.getSettings().contains(getKey()) == false) {
+            return getFallback(settings);
+        }
+        try {
+            return getSecret(keystore);
+        } catch (GeneralSecurityException e) {
+            throw new RuntimeException("failed to read secure setting " + getKey(), e);
+        }
+    }
+
+    /** Returns the secret setting from the keyStoreReader store. */
+    abstract T getSecret(KeyStoreWrapper keystore) throws GeneralSecurityException;
+
+    /** Returns the value from a fallback setting. Returns null if no fallback exists. */
+    abstract T getFallback(Settings settings);
+
+    // TODO: override toXContent
+
+    /**
+     * A setting which contains a sensitive string.
+     *
+     * This may be any sensitive string, e.g. a username, a password, an auth token, etc.
+     */
+    public static SecureSetting<SecureString> stringSetting(String name, Setting<String> fallback, Property... properties) {
+        return new SecureSetting<SecureString>(name, properties) {
+            @Override
+            protected SecureString getSecret(KeyStoreWrapper keystore) throws GeneralSecurityException {
+                return keystore.getStringSetting(getKey());
+            }
+            @Override
+            SecureString getFallback(Settings settings) {
+                if (fallback != null) {
+                    return new SecureString(fallback.get(settings).toCharArray());
+                }
+                return null;
+            }
+        };
+    }
+
+
+}

core/src/main/java/org/elasticsearch/common/settings/SecureString.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A String implementations which allows clearing the underlying char array.
+ */
+public final class SecureString implements CharSequence, AutoCloseable {
+
+    private char[] chars;
+
+    /**
+     * Constructs a new SecureString which controls the passed in char array.
+     *
+     * Note: When this instance is closed, the array will be zeroed out.
+     */
+    public SecureString(char[] chars) {
+        this.chars = Objects.requireNonNull(chars);
+    }
+
+    /** Constant time equality to avoid potential timing attacks. */
+    @Override
+    public synchronized boolean equals(Object o) {
+        ensureNotClosed();
+        if (this == o) return true;
+        if (o == null || o instanceof CharSequence == false) return false;
+        CharSequence that = (CharSequence) o;
+        if (chars.length != that.length()) {
+            return false;
+        }
+
+        int equals = 0;
+        for (int i = 0; i < chars.length; i++) {
+            equals |= chars[i] ^ that.charAt(i);
+        }
+
+        return equals == 0;
+    }
+
+    @Override
+    public synchronized int hashCode() {
+        return Arrays.hashCode(chars);
+    }
+
+    @Override
+    public synchronized int length() {
+        ensureNotClosed();
+        return chars.length;
+    }
+
+    @Override
+    public synchronized char charAt(int index) {
+        ensureNotClosed();
+        return chars[index];
+    }
+
+    @Override
+    public SecureString subSequence(int start, int end) {
+        throw new UnsupportedOperationException("Cannot get subsequence of SecureString");
+    }
+
+    /**
+     * Convert to a {@link String}. This should only be used with APIs that do not take {@link CharSequence}.
+     */
+    @Override
+    public synchronized String toString() {
+        return new String(chars);
+    }
+
+    /**
+     * Closes the string by clearing the underlying char array.
+     */
+    @Override
+    public synchronized void close() {
+        Arrays.fill(chars, '\0');
+        chars = null;
+    }
+
+    /** Throw an exception if this string has been closed, indicating something is trying to access the data after being closed. */
+    private void ensureNotClosed() {
+        if (chars == null) {
+            throw new IllegalStateException("SecureString has already been closed");
+        }
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -274,7 +274,7 @@ public class Setting<T> extends ToXContentToBytes {
      * Returns the default value string representation for this setting.
      * @param settings a settings object for settings that has a default value depending on another setting if available
      */
-    public final String getDefaultRaw(Settings settings) {
+    public String getDefaultRaw(Settings settings) {
         return defaultValue.apply(settings);
     }
 
@@ -282,7 +282,7 @@ public class Setting<T> extends ToXContentToBytes {
      * Returns the default value for this setting.
      * @param settings a settings object for settings that has a default value depending on another setting if available
      */
-    public final T getDefault(Settings settings) {
+    public T getDefault(Settings settings) {
         return parser.apply(getDefaultRaw(settings));
     }
 
@@ -290,7 +290,7 @@ public class Setting<T> extends ToXContentToBytes {
      * Returns <code>true</code> iff this setting is present in the given settings object. Otherwise <code>false</code>
      */
     public boolean exists(Settings settings) {
-        return settings.get(getKey()) != null;
+        return settings.contains(getKey());
     }
 
     /**
@@ -330,14 +330,19 @@ public class Setting<T> extends ToXContentToBytes {
      * instead. This is useful if the value can't be parsed due to an invalid value to access the actual value.
      */
     public String getRaw(Settings settings) {
+        checkDeprecation(settings);
+        return settings.get(getKey(), defaultValue.apply(settings));
+    }
+
+    /** Logs a deprecation warning if the setting is deprecated and used. */
+    protected void checkDeprecation(Settings settings) {
         // They're using the setting, so we need to tell them to stop
         if (this.isDeprecated() && this.exists(settings)) {
             // It would be convenient to show its replacement key, but replacement is often not so simple
             final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(getClass()));
             deprecationLogger.deprecated("[{}] setting was deprecated in Elasticsearch and it will be removed in a future release! " +
-                    "See the breaking changes lists in the documentation for details", getKey());
+                "See the breaking changes lists in the documentation for details", getKey());
         }
-        return settings.get(getKey(), defaultValue.apply(settings));
     }
 
     /**

core/src/main/java/org/elasticsearch/common/settings/Settings.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.settings;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
@@ -76,10 +77,29 @@ public final class Settings implements ToXContent {
     public static final Settings EMPTY = new Builder().build();
     private static final Pattern ARRAY_PATTERN = Pattern.compile("(.*)\\.\\d+$");
 
+    /** The raw settings from the full key to raw string value. */
     private Map<String, String> settings;
 
-    Settings(Map<String, String> settings) {
+    /** The keystore storage associated with these settings. */
-        this.settings = Collections.unmodifiableMap(settings);
+    private KeyStoreWrapper keystore;
+
+    Settings(Map<String, String> settings, KeyStoreWrapper keystore) {
+        // we use a sorted map for consistent serialization when using getAsMap()
+        this.settings = Collections.unmodifiableSortedMap(new TreeMap<>(settings));
+        this.keystore = keystore;
+    }
+
+    /**
+     * Retrieve the keystore that contains secure settings.
+     */
+    KeyStoreWrapper getKeyStore() {
+        // pkg private so it can only be accessed by local subclasses of SecureSetting
+        return keystore;
+    }
+
+    /** Returns true if the setting exists, false otherwise. */
+    public boolean contains(String key) {
+        return settings.containsKey(key) || keystore != null && keystore.getSettings().contains(key);
     }
 
     /**
@@ -185,16 +205,18 @@ public final class Settings implements ToXContent {
 
     /**
      * A settings that are filtered (and key is removed) with the specified prefix.
+     * Secure settings may not be access through the prefixed settings.
      */
     public Settings getByPrefix(String prefix) {
-        return new Settings(new FilteredMap(this.settings, (k) -> k.startsWith(prefix), prefix));
+        return new Settings(new FilteredMap(this.settings, (k) -> k.startsWith(prefix), prefix), null);
     }
 
     /**
      * Returns a new settings object that contains all setting of the current one filtered by the given settings key predicate.
+     * Secure settings may not be accessed through a filter.
      */
     public Settings filter(Predicate<String> predicate) {
-        return new Settings(new FilteredMap(this.settings, predicate, null));
+        return new Settings(new FilteredMap(this.settings, predicate, null), null);
     }
 
     /**
@@ -456,7 +478,7 @@ public final class Settings implements ToXContent {
         }
         Map<String, Settings> retVal = new LinkedHashMap<>();
         for (Map.Entry<String, Map<String, String>> entry : map.entrySet()) {
-            retVal.put(entry.getKey(), new Settings(Collections.unmodifiableMap(entry.getValue())));
+            retVal.put(entry.getKey(), new Settings(Collections.unmodifiableMap(entry.getValue()), keystore));
         }
         return Collections.unmodifiableMap(retVal);
     }
@@ -591,6 +613,8 @@ public final class Settings implements ToXContent {
         // we use a sorted map for consistent serialization when using getAsMap()
         private final Map<String, String> map = new TreeMap<>();
 
+        private SetOnce<KeyStoreWrapper> keystore = new SetOnce<>();
+
         private Builder() {
 
         }
@@ -613,6 +637,14 @@ public final class Settings implements ToXContent {
             return map.get(key);
         }
 
+        /** Sets the secret store for these settings. */
+        public void setKeyStore(KeyStoreWrapper keystore) {
+            if (keystore.isLoaded()) {
+                throw new IllegalStateException("The keystore wrapper must already be loaded");
+            }
+            this.keystore.set(keystore);
+        }
+
         /**
          * Puts tuples of key value pairs of settings. Simplified version instead of repeating calling
          * put for each one.
@@ -1019,7 +1051,7 @@ public final class Settings implements ToXContent {
          * set on this builder.
          */
         public Settings build() {
-            return new Settings(map);
+            return new Settings(map, keystore.get());
         }
     }
 

core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java

@@ -19,14 +19,6 @@
 
 package org.elasticsearch.node.internal;
 
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.env.Environment;
-
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -39,6 +31,14 @@ import java.util.Set;
 import java.util.function.Function;
 import java.util.function.Predicate;
 
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.env.Environment;
+
 import static org.elasticsearch.common.Strings.cleanPath;
 
 public class InternalSettingsPreparer {

core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java

@@ -152,8 +152,8 @@ public class ElasticsearchCliTests extends ESElasticsearchCliTestCase {
                 ExitCodes.OK,
                 true,
                 output -> {},
-                (foreground, pidFile, quiet, esSettings) -> {
+                (foreground, pidFile, quiet, env) -> {
-                    Map<String, String> settings = esSettings.getAsMap();
+                    Map<String, String> settings = env.settings().getAsMap();
                     assertThat(settings, hasEntry("foo", "bar"));
                     assertThat(settings, hasEntry("baz", "qux"));
                 },

core/src/test/java/org/elasticsearch/cli/TerminalTests.java

@@ -46,6 +46,33 @@ public class TerminalTests extends ESTestCase {
         assertPrinted(terminal, Terminal.Verbosity.NORMAL, "This message contains percent like %20n");
     }
 
+    public void testPromptYesNoDefault() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("");
+        assertTrue(terminal.promptYesNo("Answer?", true));
+        terminal.addTextInput("");
+        assertFalse(terminal.promptYesNo("Answer?", false));
+    }
+
+    public void testPromptYesNoReprompt() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("blah");
+        terminal.addTextInput("y");
+        assertTrue(terminal.promptYesNo("Answer? [Y/n]\nDid not understand answer 'blah'\nAnswer? [Y/n]", true));
+    }
+
+    public void testPromptYesNoCase() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("Y");
+        assertTrue(terminal.promptYesNo("Answer?", false));
+        terminal.addTextInput("y");
+        assertTrue(terminal.promptYesNo("Answer?", false));
+        terminal.addTextInput("N");
+        assertFalse(terminal.promptYesNo("Answer?", true));
+        terminal.addTextInput("n");
+        assertFalse(terminal.promptYesNo("Answer?", true));
+    }
+
     private void assertPrinted(MockTerminal logTerminal, Terminal.Verbosity verbosity, String text) throws Exception {
         logTerminal.println(verbosity, text);
         String output = logTerminal.getOutput();

core/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java

@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
+    InputStream input;
+
+    @Override
+    protected Command newCommand() {
+        return new AddStringKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+            @Override
+            InputStream getStdin() {
+                return input;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testOverwritePromptDefault() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("");
+        execute("foo");
+        assertSecureString("foo", "bar");
+    }
+
+    public void testOverwritePromptExplicitNo() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("n"); // explicit no
+        execute("foo");
+        assertSecureString("foo", "bar");
+    }
+
+    public void testOverwritePromptExplicitYes() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("y");
+        terminal.addSecretInput("newvalue");
+        execute("foo");
+        assertSecureString("foo", "newvalue");
+    }
+
+    public void testOverwriteForceShort() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addSecretInput("newvalue");
+        execute("-f", "foo"); // force
+        assertSecureString("foo", "newvalue");
+    }
+
+    public void testOverwriteForceLong() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addSecretInput("and yet another secret value");
+        execute("--force", "foo"); // force
+        assertSecureString("foo", "and yet another secret value");
+    }
+
+    public void testForceNonExistent() throws Exception {
+        createKeystore("");
+        terminal.addSecretInput("value");
+        execute("--force", "foo"); // force
+        assertSecureString("foo", "value");
+    }
+
+    public void testPromptForValue() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        terminal.addSecretInput("secret value");
+        execute("foo");
+        assertSecureString("foo", "secret value");
+    }
+
+    public void testStdinShort() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        setInput("secret value 1");
+        execute("-x", "foo");
+        assertSecureString("foo", "secret value 1");
+    }
+
+    public void testStdinLong() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        setInput("secret value 2");
+        execute("--stdin", "foo");
+        assertSecureString("foo", "secret value 2");
+    }
+
+    public void testNonAsciiValue() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        terminal.addSecretInput("non-äsčîï");
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertEquals("String value must contain only ASCII", e.getMessage());
+    }
+
+    void setInput(String inputStr) {
+        input = new ByteArrayInputStream(inputStr.getBytes(StandardCharsets.UTF_8));
+    }
+}

core/src/test/java/org/elasticsearch/common/settings/CreateKeyStoreCommandTests.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+public class CreateKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new CreateKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testPosix() throws Exception {
+        execute();
+        Path configDir = env.configFile();
+        assertNotNull(KeyStoreWrapper.load(configDir));
+    }
+
+    public void testNotPosix() throws Exception {
+        setupEnv(false);
+        execute();
+        Path configDir = env.configFile();
+        assertNotNull(KeyStoreWrapper.load(configDir));
+    }
+
+    public void testOverwrite() throws Exception {
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        byte[] content = "not a keystore".getBytes(StandardCharsets.UTF_8);
+        Files.write(keystoreFile, content);
+
+        terminal.addTextInput(""); // default is no
+        execute();
+        assertArrayEquals(content, Files.readAllBytes(keystoreFile));
+
+        terminal.addTextInput("n"); // explicit no
+        execute();
+        assertArrayEquals(content, Files.readAllBytes(keystoreFile));
+
+        terminal.addTextInput("y");
+        execute();
+        assertNotNull(KeyStoreWrapper.load(env.configFile()));
+    }
+}

core/src/test/java/org/elasticsearch/common/settings/KeyStoreCommandTestCase.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.cli.CommandTestCase;
+import org.elasticsearch.common.io.PathUtilsForTesting;
+import org.elasticsearch.env.Environment;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ * Base test case for manipulating the ES keystore.
+ */
+@LuceneTestCase.SuppressFileSystems("*") // we do our own mocking
+public abstract class KeyStoreCommandTestCase extends CommandTestCase {
+
+    Environment env;
+
+    List<FileSystem> fileSystems = new ArrayList<>();
+
+    @After
+    public void closeMockFileSystems() throws IOException {
+        IOUtils.close(fileSystems);
+    }
+
+    @Before
+    public void setupEnv() throws IOException {
+        setupEnv(true); // default to posix, but tests may call setupEnv(false) to overwrite
+    }
+
+    void setupEnv(boolean posix) throws IOException {
+        final Configuration configuration;
+        if (posix) {
+            configuration = Configuration.unix().toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
+        } else {
+            configuration = Configuration.unix();
+        }
+        FileSystem fs = Jimfs.newFileSystem(configuration);
+        fileSystems.add(fs);
+        PathUtilsForTesting.installMock(fs); // restored by restoreFileSystem in ESTestCase
+        Path home = fs.getPath("/", "test-home");
+        Files.createDirectories(home.resolve("config"));
+        env = new Environment(Settings.builder().put("path.home", home).build());
+    }
+
+    KeyStoreWrapper createKeystore(String password, String... settings) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(password.toCharArray());
+        assertEquals(0, settings.length % 2);
+        for (int i = 0; i < settings.length; i += 2) {
+            keystore.setStringSetting(settings[i], settings[i + 1].toCharArray());
+        }
+        keystore.save(env.configFile());
+        return keystore;
+    }
+
+    KeyStoreWrapper loadKeystore(String password) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        keystore.decrypt(password.toCharArray());
+        return keystore;
+    }
+
+    void assertSecureString(String setting, String value) throws Exception {
+        assertSecureString(loadKeystore(""), setting, value);
+    }
+
+    void assertSecureString(KeyStoreWrapper keystore, String setting, String value) throws Exception {
+        assertEquals(value, keystore.getStringSetting(setting).toString());
+    }
+}

core/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class ListKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new ListKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testEmpty() throws Exception {
+        createKeystore("");
+        execute();
+        assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty());
+    }
+
+    public void testOne() throws Exception {
+        createKeystore("", "foo", "bar");
+        execute();
+        assertEquals("foo\n", terminal.getOutput());
+    }
+
+    public void testMultiple() throws Exception {
+        createKeystore("", "foo", "1", "baz", "2", "bar", "3");
+        execute();
+        assertEquals("bar\nbaz\nfoo\n", terminal.getOutput()); // sorted
+    }
+}

core/src/test/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommandTests.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import javax.crypto.SecretKeyFactory;
+import java.security.Provider;
+import java.security.Security;
+import java.util.Map;
+import java.util.Set;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class RemoveSettingKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new RemoveSettingKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testNoSettings() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertThat(e.getMessage(), containsString("Must supply at least one setting"));
+    }
+
+    public void testNonExistentSetting() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.CONFIG, e.exitCode);
+        assertThat(e.getMessage(), containsString("[foo] does not exist"));
+    }
+
+    public void testOne() throws Exception {
+        createKeystore("", "foo", "bar");
+        execute("foo");
+        assertFalse(loadKeystore("").getSettings().contains("foo"));
+    }
+
+    public void testMany() throws Exception {
+        createKeystore("", "foo", "1", "bar", "2", "baz", "3");
+        execute("foo", "baz");
+        Set<String> settings = loadKeystore("").getSettings();
+        assertFalse(settings.contains("foo"));
+        assertFalse(settings.contains("baz"));
+        assertTrue(settings.contains("bar"));
+        assertEquals(1, settings.size());
+    }
+}

distribution/src/main/resources/bin/elasticsearch-keystore

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+CDPATH=""
+SCRIPT="$0"
+
+# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path.
+while [ -h "$SCRIPT" ] ; do
+  ls=`ls -ld "$SCRIPT"`
+  # Drop everything prior to ->
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    SCRIPT="$link"
+  else
+    SCRIPT=`dirname "$SCRIPT"`/"$link"
+  fi
+done
+
+# determine elasticsearch home
+ES_HOME=`dirname "$SCRIPT"`/..
+
+# make ELASTICSEARCH_HOME absolute
+ES_HOME=`cd "$ES_HOME"; pwd`
+
+
+# Sets the default values for elasticsearch variables used in this script
+if [ -z "$CONF_DIR" ]; then
+  CONF_DIR="${path.conf}"
+fi
+
+# The default env file is defined at building/packaging time.
+# For a ${project.name} package, the value is "${path.env}".
+ES_ENV_FILE="${path.env}"
+
+# If an include is specified with the ES_INCLUDE environment variable, use it
+if [ -n "$ES_INCLUDE" ]; then
+    ES_ENV_FILE="$ES_INCLUDE"
+fi
+
+# Source the environment file
+if [ -n "$ES_ENV_FILE" ]; then
+
+  # If the ES_ENV_FILE is not found, try to resolve the path
+  # against the ES_HOME directory
+  if [ ! -f "$ES_ENV_FILE" ]; then
+      ES_ENV_FILE="$ELASTIC_HOME/$ES_ENV_FILE"
+  fi
+
+  . "$ES_ENV_FILE"
+  if [ $? -ne 0 ]; then
+      echo "Unable to source environment file: $ES_ENV_FILE" >&2
+      exit 1
+  fi
+fi
+
+# don't let JAVA_TOOL_OPTIONS slip in (e.g. crazy agents in ubuntu)
+# works around https://bugs.launchpad.net/ubuntu/+source/jayatana/+bug/1441487
+if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then
+    echo "Warning: Ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS"
+    unset JAVA_TOOL_OPTIONS
+fi
+
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi
+
+if [ -x "$JAVA_HOME/bin/java" ]; then
+    JAVA=$JAVA_HOME/bin/java
+else
+    JAVA=`which java`
+fi
+
+if [ ! -x "$JAVA" ]; then
+    echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
+    exit 1
+fi
+
+# full hostname passed through cut for portability on systems that do not support hostname -s
+# export on separate line for shells that do not support combining definition and export
+HOSTNAME=`hostname | cut -d. -f1`
+export HOSTNAME
+
+declare -a args=("$@")
+path_props=(-Des.path.home="$ES_HOME")
+
+if [ -e "$CONF_DIR" ]; then
+  path_props=("${path_props[@]}" -Des.path.conf="$CONF_DIR")
+fi
+
+exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch "${path_props[@]}" -cp "$ES_HOME/lib/*" org.elasticsearch.common.settings.KeyStoreCli "${args[@]}"

distribution/src/main/resources/bin/elasticsearch-keystore.bat

@@ -0,0 +1,30 @@
+@echo off
+
+SETLOCAL enabledelayedexpansion
+
+IF DEFINED JAVA_HOME (
+  set JAVA=%JAVA_HOME%\bin\java.exe
+) ELSE (
+  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
+)
+IF NOT EXIST "%JAVA%" (
+  ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
+  EXIT /B 1
+)
+
+set SCRIPT_DIR=%~dp0
+for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
+
+TITLE Elasticsearch Plugin Manager ${project.version}
+
+SET path_props=-Des.path.home="%ES_HOME%"
+IF DEFINED CONF_DIR (
+  SET path_props=!path_props! -Des.path.conf="%CONF_DIR%"
+)
+
+SET args=%*
+SET HOSTNAME=%COMPUTERNAME%
+
+"%JAVA%" %ES_JAVA_OPTS% !path_props! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.common.settings.KeyStoreCli" !args!
+
+ENDLOCAL

docs/java-rest/configuration.asciidoc

@@ -93,9 +93,9 @@ https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org
 
 [source,java]
 --------------------------------------------------
-KeyStore keyStore = KeyStore.getInstance("jks");
+KeyStore keystore = KeyStore.getInstance("jks");
 try (InputStream is = Files.newInputStream(keyStorePath)) {
-    keyStore.load(is, keyStorePass.toCharArray());
+    keystore.load(is, keyStorePass.toCharArray());
 }
 RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
         .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {

qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java

@@ -41,8 +41,7 @@ public class EvilElasticsearchCliTests extends ESElasticsearchCliTestCase {
                 true,
                 output -> {},
                 (foreground, pidFile, quiet, esSettings) -> {
-                    Map<String, String> settings = esSettings.getAsMap();
+                    Map<String, String> settings = esSettings.settings().getAsMap();
-                    settings.keySet().forEach(System.out::println);
                     assertThat(settings.size(), equalTo(2));
                     assertThat(settings, hasEntry("path.home", value));
                     assertThat(settings, hasKey("path.logs")); // added by env initialization
@@ -55,7 +54,7 @@ public class EvilElasticsearchCliTests extends ESElasticsearchCliTestCase {
                 true,
                 output -> {},
                 (foreground, pidFile, quiet, esSettings) -> {
-                    Map<String, String> settings = esSettings.getAsMap();
+                    Map<String, String> settings = esSettings.settings().getAsMap();
                     assertThat(settings.size(), equalTo(2));
                     assertThat(settings, hasEntry("path.home", commandLineValue));
                     assertThat(settings, hasKey("path.logs")); // added by env initialization

test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java

@@ -35,7 +35,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 abstract class ESElasticsearchCliTestCase extends ESTestCase {
 
     interface InitConsumer {
-        void accept(final boolean foreground, final Path pidFile, final boolean quiet, final Settings initialSettings);
+        void accept(final boolean foreground, final Path pidFile, final boolean quiet, final Environment initialEnv);
     }
 
     void runTest(
@@ -57,9 +57,9 @@ abstract class ESElasticsearchCliTestCase extends ESTestCase {
                     return new Environment(realSettings);
                 }
                 @Override
-                void init(final boolean daemonize, final Path pidFile, final boolean quiet, Settings initialSettings) {
+                void init(final boolean daemonize, final Path pidFile, final boolean quiet, Environment initialEnv) {
                     init.set(true);
-                    initConsumer.accept(!daemonize, pidFile, quiet, initialSettings);
+                    initConsumer.accept(!daemonize, pidFile, quiet, initialEnv);
                 }
 
                 @Override