From a9707a461df93831a7871dfe12069900f817bf8d Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Thu, 29 Jun 2017 14:58:35 -0600 Subject: [PATCH] Use a secure setting for the watcher encryption key (elastic/x-pack-elasticsearch#1831) This commit removes the system key from master and changes watcher to use a secure setting instead for the encryption key. Original commit: elastic/x-pack-elasticsearch@5ac95c60ef346c4815b0c1906d1eee69684da566 --- docs/en/security/getting-started.asciidoc | 17 ---- docs/en/security/index.asciidoc | 7 +- docs/en/security/reference/files.asciidoc | 4 - docs/en/security/release-notes.asciidoc | 6 +- .../tribe-clients-integrations/tribe.asciidoc | 8 -- plugin/bin/x-pack/syskeygen | 2 +- plugin/bin/x-pack/syskeygen.bat | Bin 350 -> 336 bytes .../org/elasticsearch/xpack/XPackPlugin.java | 18 +++- .../xpack/security/Security.java | 13 --- .../xpack/security/crypto/CryptoService.java | 88 ++++++------------ .../security/crypto/tool/SystemKeyTool.java | 7 +- .../EncryptSensitiveDataBootstrapCheck.java | 54 +++++++++++ .../elasticsearch/xpack/watcher/Watcher.java | 12 +++ .../xpack/watcher/watch/Watch.java | 3 +- .../email/EmailSecretsIntegrationTests.java | 26 ++++-- .../security/crypto/CryptoServiceTests.java | 67 ++----------- ...cryptSensitiveDataBootstrapCheckTests.java | 46 +++++++++ .../AbstractWatcherIntegrationTestCase.java | 16 +--- .../HttpSecretsIntegrationTests.java | 28 ++++-- qa/full-cluster-restart/build.gradle | 8 +- qa/rolling-upgrade/build.gradle | 15 +-- 21 files changed, 227 insertions(+), 218 deletions(-) create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheck.java create mode 100644 plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java diff --git a/docs/en/security/getting-started.asciidoc b/docs/en/security/getting-started.asciidoc index d5f49e7aef3..f7d31b0522d 100644 --- a/docs/en/security/getting-started.asciidoc +++ b/docs/en/security/getting-started.asciidoc @@ -73,23 +73,6 @@ curl -XPOST -u elastic 'localhost:9200/_xpack/security/user/johndoe' -H "Content // NOTCONSOLE -- -[[enable-message-authentication]] -. Enable message authentication to verify that messages are not tampered with or corrupted in transit: -.. Run the `syskeygen` tool from `ES_HOME` without any options: -+ -[source, shell] ----------------- -bin/x-pack/syskeygen ----------------- -+ -This creates a system key file in `CONFIG_DIR/x-pack/system_key`. - -.. Copy the generated system key to the rest of the nodes in the cluster. -+ -IMPORTANT: The system key is a symmetric key, so the same key must be on every - node in the cluster. - - [[enable-auditing]] . Enable Auditing to keep track of attempted and successful interactions with your Elasticsearch cluster: diff --git a/docs/en/security/index.asciidoc b/docs/en/security/index.asciidoc index a45b6853d4c..1a531c7f6d6 100644 --- a/docs/en/security/index.asciidoc +++ b/docs/en/security/index.asciidoc @@ -52,11 +52,8 @@ A critical part of security is keeping confidential data confidential. Elasticsearch has built-in protections against accidental data loss and corruption. However, there's nothing to stop deliberate tampering or data interception. {security} preserves the integrity of your data by -<> to and from nodes and -<> to verify that they -have not been tampered with or corrupted in transit during node-to-node -communication. For even greater protection, you can increase the -<> and +<> to and from nodes. +For even greater protection, you can increase the <> and <>. diff --git a/docs/en/security/reference/files.asciidoc b/docs/en/security/reference/files.asciidoc index d1529a07d69..3030669fb3f 100644 --- a/docs/en/security/reference/files.asciidoc +++ b/docs/en/security/reference/files.asciidoc @@ -20,10 +20,6 @@ The {security} uses the following files: * `CONFIG_DIR/x-pack/log4j2.properties` contains audit information (read more <>). -* `CONFIG_DIR/x-pack/system_key` holds a cluster secret key that's used to - authenticate messages during node to node communication. For more information, - see <>. - [[security-files-location]] IMPORTANT: Any files that {security} uses must be stored in the Elasticsearch diff --git a/docs/en/security/release-notes.asciidoc b/docs/en/security/release-notes.asciidoc index 5e037a1f2d0..b783ea8f536 100644 --- a/docs/en/security/release-notes.asciidoc +++ b/docs/en/security/release-notes.asciidoc @@ -102,7 +102,7 @@ March 15, 2016 .Bug Fixes * Enable <> by default. -* Fix issues with <> on certain JDKs that do not support cloning message +* Fix issues with message authentication on certain JDKs that do not support cloning message authentication codes. * Built in <> no longer throw an exception if the `Authorization` header does not contain a basic authentication token. @@ -209,7 +209,7 @@ the correct user to index the audit events. July 21, 2015 .Bug Fixes -* Fixes <> serialization to work with Shield 1.2.1 and earlier. +* Fixes message authentication serialization to work with Shield 1.2.1 and earlier. ** NOTE: if you are upgrading from Shield 1.3.0 or Shield 1.2.2 a {ref-17}/setup-upgrade.html#restart-upgrade[cluster restart upgrade] will be necessary. When upgrading from other versions of Shield, follow the normal upgrade procedure. @@ -245,7 +245,7 @@ June 24, 2015 July 21, 2015 .Bug Fixes -* Fixes <> serialization to work with Shield 1.2.1 and earlier. +* Fixes message authentication serialization to work with Shield 1.2.1 and earlier. ** NOTE: if you are upgrading from Shield 1.2.2 a {ref-17}/setup-upgrade.html#restart-upgrade[cluster restart upgrade] will be necessary. When upgrading from other versions of Shield, follow the normal upgrade procedure. diff --git a/docs/en/security/tribe-clients-integrations/tribe.asciidoc b/docs/en/security/tribe-clients-integrations/tribe.asciidoc index 0d3892a8b08..2402d0a5f75 100644 --- a/docs/en/security/tribe-clients-integrations/tribe.asciidoc +++ b/docs/en/security/tribe-clients-integrations/tribe.asciidoc @@ -15,14 +15,6 @@ To use a tribe node with secured clusters: . Install {xpack} on the tribe node and every node in each connected cluster. -. Enable <> globally. -Generate a system key on one node and copy it to the tribe node and every other -node in each of the connected clusters. -+ -IMPORTANT: For message authentication to work properly across multiple clusters, - the tribe node and all of the connected clusters must share the same - system key. {security} reads the system key from `CONFIG_DIR/x-pack/system_key`. - . Enable encryption globally. To encrypt communications, you must enable <> on every node. + diff --git a/plugin/bin/x-pack/syskeygen b/plugin/bin/x-pack/syskeygen index 4d85cbe9981..478966c7587 100755 --- a/plugin/bin/x-pack/syskeygen +++ b/plugin/bin/x-pack/syskeygen @@ -98,7 +98,7 @@ if [ -e "$CONF_DIR" ]; then fi cd "$ES_HOME" > /dev/null -"$JAVA" $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" org.elasticsearch.xpack.security.crypto.tool.SystemKeyTool $properties "${args[@]}" +"$JAVA" $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" org.elasticsearch.common.settings.EncKeyTool $properties "${args[@]}" status=$? cd - > /dev/null exit $status diff --git a/plugin/bin/x-pack/syskeygen.bat b/plugin/bin/x-pack/syskeygen.bat index 74f2d46bd245b523efc0b31aa04694fa9d9f077a..3cf0afb13a0caa9c8b822388148a5dd8eac36db0 100644 GIT binary patch delta 42 xcmcb|bb)Du9-~rner|4lo?dZkNl9j2da<5sUb1&;Wk`O0j)JNdSAc(j3jj^=4wC=? delta 56 zcmcb>bdPC+9-~1;L1J>YUU6!2X;EfLrCxGTWkE^4UP*p_j$UwOaY<^fcWPw_kf)%k K#TDQm-~s?(bQE9! diff --git a/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java b/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java index 481d98bd32b..72baebc66f8 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java @@ -46,7 +46,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ExecutorBuilder; @@ -98,6 +97,7 @@ import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.SecurityFeatureSet; import org.elasticsearch.xpack.security.authc.AuthenticationService; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; +import org.elasticsearch.xpack.security.crypto.CryptoService; import org.elasticsearch.xpack.ssl.SSLConfigurationReloader; import org.elasticsearch.xpack.ssl.SSLService; import org.elasticsearch.xpack.upgrade.Upgrade; @@ -123,6 +123,9 @@ import java.util.Set; import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.watcher.Watcher.ENCRYPT_SENSITIVE_DATA_SETTING; public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin { @@ -202,6 +205,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I protected Graph graph; protected MachineLearning machineLearning; protected Logstash logstash; + protected CryptoService cryptoService; protected Deprecation deprecation; protected Upgrade upgrade; @@ -229,6 +233,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I } else { this.extensionsService = null; } + cryptoService = ENCRYPT_SENSITIVE_DATA_SETTING.get(settings) ? new CryptoService(settings) : null; } // For tests only @@ -283,7 +288,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I // watcher http stuff Map httpAuthFactories = new HashMap<>(); - httpAuthFactories.put(BasicAuth.TYPE, new BasicAuthFactory(security.getCryptoService())); + httpAuthFactories.put(BasicAuth.TYPE, new BasicAuthFactory(cryptoService)); // TODO: add more auth types, or remove this indirection HttpAuthRegistry httpAuthRegistry = new HttpAuthRegistry(httpAuthFactories); HttpRequestTemplate.Parser httpTemplateParser = new HttpRequestTemplate.Parser(httpAuthRegistry); @@ -296,7 +301,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I components.addAll(notificationComponents); components.addAll(watcher.createComponents(getClock(), scriptService, internalClient, licenseState, - httpClient, httpTemplateParser, threadPool, clusterService, security.getCryptoService(), xContentRegistry, components)); + httpClient, httpTemplateParser, threadPool, clusterService, cryptoService, xContentRegistry, components)); components.addAll(machineLearning.createComponents(internalClient, clusterService, threadPool, xContentRegistry)); @@ -315,7 +320,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I HttpRequestTemplate.Parser httpTemplateParser, ScriptService scriptService, HttpAuthRegistry httpAuthRegistry) { List components = new ArrayList<>(); - components.add(new EmailService(settings, security.getCryptoService(), clusterSettings)); + components.add(new EmailService(settings, cryptoService, clusterSettings)); components.add(new HipChatService(settings, httpClient, clusterSettings)); components.add(new JiraService(settings, httpClient, clusterSettings)); components.add(new SlackService(settings, httpClient, clusterSettings)); @@ -573,7 +578,10 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I @Override public List getBootstrapChecks() { - return security.getBootstrapChecks(); + return Collections.unmodifiableList( + Stream.of(security.getBootstrapChecks(), watcher.getBootstrapChecks()) + .flatMap(Collection::stream) + .collect(Collectors.toList())); } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/Security.java b/plugin/src/main/java/org/elasticsearch/xpack/security/Security.java index 8b91d61aa68..85e7687d353 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -200,7 +200,6 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin { private final boolean enabled; private final boolean transportClientMode; private final XPackLicenseState licenseState; - private final CryptoService cryptoService; private final SSLService sslService; /* what a PITA that we need an extra indirection to initialize this. Yet, once we got rid of guice we can thing about how * to fix this or make it simpler. Today we need several service that are created in createComponents but we need to register @@ -220,18 +219,11 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin { this.enabled = XPackSettings.SECURITY_ENABLED.get(settings); if (enabled && transportClientMode == false) { validateAutoCreateIndex(settings); - cryptoService = new CryptoService(settings, env); - } else { - cryptoService = null; } this.licenseState = licenseState; this.sslService = sslService; } - public CryptoService getCryptoService() { - return cryptoService; - } - public Collection nodeModules() { List modules = new ArrayList<>(); if (enabled == false || transportClientMode) { @@ -254,7 +246,6 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin { if (enabled == false) { modules.add(b -> { - b.bind(CryptoService.class).toProvider(Providers.of(null)); b.bind(Realms.class).toProvider(Providers.of(null)); // for SecurityFeatureSet b.bind(CompositeRolesStore.class).toProvider(Providers.of(null)); // for SecurityFeatureSet b.bind(NativeRoleMappingStore.class).toProvider(Providers.of(null)); // for SecurityFeatureSet @@ -268,7 +259,6 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin { // which might not be the case during Plugin class instantiation. Once nodeModules are pulled // everything should have been loaded modules.add(b -> { - b.bind(CryptoService.class).toInstance(cryptoService); if (XPackSettings.AUDIT_ENABLED.get(settings)) { b.bind(AuditTrail.class).to(AuditTrailService.class); // interface used by some actions... } @@ -474,9 +464,6 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin { settingsList.add(TokenService.DELETE_INTERVAL); settingsList.add(TokenService.DELETE_TIMEOUT); - // encryption settings - CryptoService.addSettings(settingsList); - // hide settings settingsList.add(Setting.listSetting(setting("hide_settings"), Collections.emptyList(), Function.identity(), Property.NodeScope, Property.Filtered)); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/crypto/CryptoService.java b/plugin/src/main/java/org/elasticsearch/xpack/security/crypto/CryptoService.java index e0f6f40f5dd..0a47cfe6abd 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/crypto/CryptoService.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/crypto/CryptoService.java @@ -12,10 +12,8 @@ import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; -import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import java.io.InputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; @@ -25,12 +23,12 @@ import java.util.List; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.security.authc.support.CharArrays; +import org.elasticsearch.xpack.watcher.Watcher; import static org.elasticsearch.xpack.security.Security.setting; @@ -42,14 +40,19 @@ public class CryptoService extends AbstractComponent { public static final String KEY_ALGO = "HmacSHA512"; public static final int KEY_SIZE = 1024; - static final String FILE_NAME = "system_key"; - static final String DEFAULT_ENCRYPTION_ALGORITHM = "AES/CTR/NoPadding"; - static final String DEFAULT_KEY_ALGORITH = "AES"; static final String ENCRYPTED_TEXT_PREFIX = "::es_encrypted::"; - static final int DEFAULT_KEY_LENGTH = 128; - private static final Setting SYSTEM_KEY_REQUIRED_SETTING = - Setting.boolSetting(setting("system_key.required"), false, Property.NodeScope); + // the encryption used in this class was picked when Java 7 was still the min. supported + // version. The use of counter mode was chosen to simplify the need to deal with padding + // and for its speed. 128 bit key length is chosen due to the JCE policy that ships by + // default with the Oracle JDK. + // TODO: with better support in Java 8, we should consider moving to use AES GCM as it + // also provides authentication of the encrypted data, which is something that we are + // missing here. + private static final String DEFAULT_ENCRYPTION_ALGORITHM = "AES/CTR/NoPadding"; + private static final String DEFAULT_KEY_ALGORITH = "AES"; + private static final int DEFAULT_KEY_LENGTH = 128; + private static final Setting ENCRYPTION_ALGO_SETTING = new Setting<>(setting("encryption.algorithm"), s -> DEFAULT_ENCRYPTION_ALGORITHM, s -> s, Property.NodeScope); private static final Setting ENCRYPTION_KEY_LENGTH_SETTING = @@ -65,7 +68,7 @@ public class CryptoService extends AbstractComponent { */ private final SecretKey encryptionKey; - public CryptoService(Settings settings, Environment env) throws IOException { + public CryptoService(Settings settings) throws IOException { super(settings); this.encryptionAlgorithm = ENCRYPTION_ALGO_SETTING.get(settings); final int keyLength = ENCRYPTION_KEY_LENGTH_SETTING.get(settings); @@ -76,17 +79,13 @@ public class CryptoService extends AbstractComponent { throw new IllegalArgumentException("invalid key length [" + keyLength + "]. value must be a multiple of 8"); } - Path keyFile = resolveSystemKey(env); - SecretKey systemKey = readSystemKey(keyFile, SYSTEM_KEY_REQUIRED_SETTING.get(settings)); - + SecretKey systemKey = readSystemKey(Watcher.ENCRYPTION_KEY_SETTING.get(settings)); try { encryptionKey = encryptionKey(systemKey, keyLength, keyAlgorithm); } catch (NoSuchAlgorithmException nsae) { throw new ElasticsearchException("failed to start crypto service. could not load encryption key", nsae); } - if (systemKey != null) { - logger.info("system key [{}] has been loaded", keyFile.toAbsolutePath()); - } + assert encryptionKey != null : "the encryption key should never be null"; } public static byte[] generateKey() { @@ -103,21 +102,14 @@ public class CryptoService extends AbstractComponent { } } - public static Path resolveSystemKey(Environment env) { - return XPackPlugin.resolveConfigFile(env, FILE_NAME); - } - - private static SecretKey readSystemKey(Path file, boolean required) throws IOException { - if (Files.exists(file)) { - byte[] bytes = Files.readAllBytes(file); - return new SecretKeySpec(bytes, KEY_ALGO); + private static SecretKey readSystemKey(InputStream in) throws IOException { + final int keySizeBytes = KEY_SIZE / 8; + final byte[] keyBytes = new byte[keySizeBytes]; + final int read = Streams.readFully(in, keyBytes); + if (read != keySizeBytes) { + throw new IllegalArgumentException("key size did not match expected value; was the key generated with syskeygen?"); } - - if (required) { - throw new FileNotFoundException("[" + file + "] must be present with a valid key"); - } - - return null; + return new SecretKeySpec(keyBytes, KEY_ALGO); } /** @@ -126,15 +118,8 @@ public class CryptoService extends AbstractComponent { * @return character array representing the encrypted data */ public char[] encrypt(char[] chars) { - SecretKey key = this.encryptionKey; - if (key == null) { - logger.warn("encrypt called without a key, returning plain text. run syskeygen and copy same key to all nodes to enable " + - "encryption"); - return chars; - } - byte[] charBytes = CharArrays.toUtf8Bytes(chars); - String base64 = Base64.getEncoder().encodeToString(encryptInternal(charBytes, key)); + String base64 = Base64.getEncoder().encodeToString(encryptInternal(charBytes, encryptionKey)); return ENCRYPTED_TEXT_PREFIX.concat(base64).toCharArray(); } @@ -144,10 +129,6 @@ public class CryptoService extends AbstractComponent { * @return plaintext chars */ public char[] decrypt(char[] chars) { - if (encryptionKey == null) { - return chars; - } - if (!isEncrypted(chars)) { // Not encrypted return chars; @@ -170,18 +151,10 @@ public class CryptoService extends AbstractComponent { * @param chars the chars to check if they are encrypted * @return true is data is encrypted */ - public boolean isEncrypted(char[] chars) { + protected boolean isEncrypted(char[] chars) { return CharArrays.charsBeginsWith(ENCRYPTED_TEXT_PREFIX, chars); } - /** - * Flag for callers to determine if values will actually be encrypted or returned plaintext - * @return true if values will be encrypted - */ - public boolean isEncryptionEnabled() { - return this.encryptionKey != null; - } - private byte[] encryptInternal(byte[] bytes, SecretKey key) { byte[] iv = new byte[ivLength]; secureRandom.nextBytes(iv); @@ -217,7 +190,7 @@ public class CryptoService extends AbstractComponent { } - static Cipher cipher(int mode, String encryptionAlgorithm, SecretKey key, byte[] initializationVector) { + private static Cipher cipher(int mode, String encryptionAlgorithm, SecretKey key, byte[] initializationVector) { try { Cipher cipher = Cipher.getInstance(encryptionAlgorithm); cipher.init(mode, key, new IvParameterSpec(initializationVector)); @@ -227,11 +200,7 @@ public class CryptoService extends AbstractComponent { } } - static SecretKey encryptionKey(SecretKey systemKey, int keyLength, String algorithm) throws NoSuchAlgorithmException { - if (systemKey == null) { - return null; - } - + private static SecretKey encryptionKey(SecretKey systemKey, int keyLength, String algorithm) throws NoSuchAlgorithmException { byte[] bytes = systemKey.getEncoded(); if ((bytes.length * 8) < keyLength) { throw new IllegalArgumentException("at least " + keyLength + " bits should be provided as key data"); @@ -253,6 +222,5 @@ public class CryptoService extends AbstractComponent { settings.add(ENCRYPTION_KEY_LENGTH_SETTING); settings.add(ENCRYPTION_KEY_ALGO_SETTING); settings.add(ENCRYPTION_ALGO_SETTING); - settings.add(SYSTEM_KEY_REQUIRED_SETTING); } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyTool.java b/plugin/src/main/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyTool.java index ff6a7756e78..5a72e71c7e6 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyTool.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyTool.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.security.crypto.CryptoService; import java.nio.file.Files; @@ -61,7 +62,7 @@ public class SystemKeyTool extends EnvironmentAwareCommand { } keyPath = parsePath(args.get(0)); } else { - keyPath = CryptoService.resolveSystemKey(env); + keyPath = env.configFile().resolve(XPackPlugin.NAME).resolve("system_key"); } // write the key @@ -75,7 +76,7 @@ public class SystemKeyTool extends EnvironmentAwareCommand { if (view != null) { view.setPermissions(PERMISSION_OWNER_READ_WRITE); terminal.println("Ensure the generated key can be read by the user that Elasticsearch runs as, " - + "permissions are set to owner read/write only"); + + "permissions are set to owner read/write only"); } } @@ -84,4 +85,4 @@ public class SystemKeyTool extends EnvironmentAwareCommand { return PathUtils.get(path); } -} +} \ No newline at end of file diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheck.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheck.java new file mode 100644 index 00000000000..9da35e4e2ec --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheck.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.watcher; + +import org.elasticsearch.bootstrap.BootstrapCheck; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.XPackPlugin; + +import java.nio.file.Files; +import java.nio.file.Path; + +final class EncryptSensitiveDataBootstrapCheck implements BootstrapCheck { + + private final Settings settings; + private final Environment environment; + + EncryptSensitiveDataBootstrapCheck(Settings settings, Environment environment) { + this.settings = settings; + this.environment = environment; + } + + @Override + public boolean check() { + return Watcher.ENCRYPT_SENSITIVE_DATA_SETTING.get(settings) && Watcher.ENCRYPTION_KEY_SETTING.exists(settings) == false; + } + + @Override + public String errorMessage() { + final Path sysKeyPath = environment.configFile().resolve(XPackPlugin.NAME).resolve("system_key").toAbsolutePath(); + if (Files.exists(sysKeyPath)) { + return "Encryption of sensitive data requires the key to be placed in the secure setting store. Run " + + "'bin/elasticsearch-keystore add-file " + Watcher.ENCRYPTION_KEY_SETTING.getKey() + " " + + environment.configFile().resolve(XPackPlugin.NAME).resolve("system_key").toAbsolutePath() + + "' to import the file.\nAfter importing, the system_key file should be removed from the " + + "filesystem.\nRepeat this on every node in the cluster."; + } else { + return "Encryption of sensitive data requires a key to be placed in the secure setting store. First run the " + + "bin/x-pack/syskeygen tool to generate a key file.\nThen run 'bin/elasticsearch-keystore add-file " + + Watcher.ENCRYPTION_KEY_SETTING.getKey() + " " + + environment.configFile().resolve(XPackPlugin.NAME).resolve("system_key").toAbsolutePath() + "' to import the key into" + + " the secure setting store. Finally, remove the system_key file from the filesystem.\n" + + "Repeat this on every node in the cluster"; + } + } + + @Override + public boolean alwaysEnforce() { + return true; + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 67c7549e840..28b8a60f56a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.watcher; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; @@ -25,12 +26,14 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexModule; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; @@ -153,6 +156,7 @@ import org.elasticsearch.xpack.watcher.watch.Watch; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import java.io.InputStream; import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; @@ -177,6 +181,7 @@ public class Watcher implements ActionPlugin { new Setting<>("index.xpack.watcher.template.version", "", Function.identity(), Setting.Property.IndexScope); public static final Setting ENCRYPT_SENSITIVE_DATA_SETTING = Setting.boolSetting("xpack.watcher.encrypt_sensitive_data", false, Setting.Property.NodeScope); + public static final Setting ENCRYPTION_KEY_SETTING = SecureSetting.secureFile("xpack.watcher.encryption_key", null); public static final Setting MAX_STOP_TIMEOUT_SETTING = Setting.timeSetting("xpack.watcher.stop.timeout", TimeValue.timeValueSeconds(30), Setting.Property.NodeScope); @@ -367,6 +372,7 @@ public class Watcher implements ActionPlugin { settings.add(Setting.intSetting("xpack.watcher.execution.scroll.size", 0, Setting.Property.NodeScope)); settings.add(Setting.intSetting("xpack.watcher.watch.scroll.size", 0, Setting.Property.NodeScope)); settings.add(ENCRYPT_SENSITIVE_DATA_SETTING); + settings.add(ENCRYPTION_KEY_SETTING); settings.add(Setting.simpleString("xpack.watcher.internal.ops.search.default_timeout", Setting.Property.NodeScope)); settings.add(Setting.simpleString("xpack.watcher.internal.ops.bulk.default_timeout", Setting.Property.NodeScope)); @@ -379,6 +385,8 @@ public class Watcher implements ActionPlugin { settings.add(Setting.simpleString("xpack.watcher.execution.scroll.timeout", Setting.Property.NodeScope)); settings.add(Setting.simpleString("xpack.watcher.start_immediately", Setting.Property.NodeScope)); + // encryption settings + CryptoService.addSettings(settings); return settings; } @@ -512,4 +520,8 @@ public class Watcher implements ActionPlugin { return map; }; } + + public List getBootstrapChecks() { + return Collections.singletonList(new EncryptSensitiveDataBootstrapCheck(settings, new Environment(settings))); + } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/watch/Watch.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/watch/Watch.java index 8a4084ee5b3..31ff3da7a40 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/watch/Watch.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/watch/Watch.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.common.secret.Secret; import org.elasticsearch.xpack.security.crypto.CryptoService; import org.elasticsearch.xpack.support.clock.HaltedClock; -import org.elasticsearch.xpack.watcher.Watcher; import org.elasticsearch.xpack.watcher.actions.ActionRegistry; import org.elasticsearch.xpack.watcher.actions.ActionStatus; import org.elasticsearch.xpack.watcher.actions.ActionWrapper; @@ -209,7 +208,7 @@ public class Watch implements ToXContentObject { this.triggerService = triggerService; this.actionRegistry = actionRegistry; this.inputRegistry = inputRegistry; - this.cryptoService = Watcher.ENCRYPT_SENSITIVE_DATA_SETTING.get(settings) ? cryptoService : null; + this.cryptoService = cryptoService; this.defaultInput = new ExecutableNoneInput(logger); this.defaultCondition = AlwaysCondition.INSTANCE; this.defaultActions = Collections.emptyList(); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/notification/email/EmailSecretsIntegrationTests.java b/plugin/src/test/java/org/elasticsearch/xpack/notification/email/EmailSecretsIntegrationTests.java index 622e3900dba..28fe26b2c29 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/notification/email/EmailSecretsIntegrationTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/notification/email/EmailSecretsIntegrationTests.java @@ -6,10 +6,12 @@ package org.elasticsearch.xpack.notification.email; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.xpack.notification.email.support.EmailServer; import org.elasticsearch.xpack.security.crypto.CryptoService; +import org.elasticsearch.xpack.watcher.Watcher; import org.elasticsearch.xpack.watcher.client.WatcherClient; import org.elasticsearch.xpack.watcher.condition.AlwaysCondition; import org.elasticsearch.xpack.watcher.execution.ActionExecutionMode; @@ -43,6 +45,7 @@ public class EmailSecretsIntegrationTests extends AbstractWatcherIntegrationTest private EmailServer server; private Boolean encryptSensitiveData; + private byte[] encryptionKey; @Override public void setUp() throws Exception { @@ -58,15 +61,23 @@ public class EmailSecretsIntegrationTests extends AbstractWatcherIntegrationTest @Override protected Settings nodeSettings(int nodeOrdinal) { if (encryptSensitiveData == null) { - encryptSensitiveData = securityEnabled() && randomBoolean(); + encryptSensitiveData = randomBoolean(); + if (encryptSensitiveData) { + encryptionKey = CryptoService.generateKey(); + } } - return Settings.builder() + Settings.Builder builder = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("xpack.notification.email.account.test.smtp.auth", true) .put("xpack.notification.email.account.test.smtp.port", server.port()) .put("xpack.notification.email.account.test.smtp.host", "localhost") - .put("xpack.watcher.encrypt_sensitive_data", encryptSensitiveData) - .build(); + .put("xpack.watcher.encrypt_sensitive_data", encryptSensitiveData); + if (encryptSensitiveData) { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), encryptionKey); + builder.setSecureSettings(secureSettings); + } + return builder.build(); } public void testEmail() throws Exception { @@ -91,9 +102,12 @@ public class EmailSecretsIntegrationTests extends AbstractWatcherIntegrationTest Map source = response.getSource(); Object value = XContentMapValues.extractValue("actions._email.email.password", source); assertThat(value, notNullValue()); - if (securityEnabled() && encryptSensitiveData) { + if (encryptSensitiveData) { assertThat(value, not(is(EmailServer.PASSWORD))); - CryptoService cryptoService = getInstanceFromMaster(CryptoService.class); + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), encryptionKey); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + CryptoService cryptoService = new CryptoService(settings); assertThat(new String(cryptoService.decrypt(((String) value).toCharArray())), is(EmailServer.PASSWORD)); } else { assertThat(value, is(EmailServer.PASSWORD)); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/crypto/CryptoServiceTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/crypto/CryptoServiceTests.java index 03b4e04b165..44a354e75a0 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/crypto/CryptoServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/crypto/CryptoServiceTests.java @@ -5,45 +5,31 @@ */ package org.elasticsearch.xpack.security.crypto; -import java.io.FileNotFoundException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; +import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.XPackPlugin; +import org.elasticsearch.xpack.watcher.Watcher; import org.junit.Before; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; public class CryptoServiceTests extends ESTestCase { private Settings settings; - private Environment env; - private Path keyFile; @Before public void init() throws Exception { - Path home = createTempDir(); - Path xpackConf = home.resolve("config").resolve(XPackPlugin.NAME); - Files.createDirectories(xpackConf); - keyFile = xpackConf.resolve("system_key"); - Files.write(keyFile, CryptoService.generateKey()); + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), CryptoService.generateKey()); settings = Settings.builder() - .put("resource.reload.interval.high", "2s") - .put("xpack.security.system_key.required", randomBoolean()) - .put("path.home", home) + .setSecureSettings(mockSecureSettings) .build(); - env = new Environment(settings); } public void testEncryptionAndDecryptionChars() throws Exception { - CryptoService service = new CryptoService(settings, env); - assertThat(service.isEncryptionEnabled(), is(true)); + CryptoService service = new CryptoService(settings); final char[] chars = randomAlphaOfLengthBetween(0, 1000).toCharArray(); final char[] encrypted = service.encrypt(chars); assertThat(encrypted, notNullValue()); @@ -53,31 +39,8 @@ public class CryptoServiceTests extends ESTestCase { assertThat(Arrays.equals(chars, decrypted), is(true)); } - public void testEncryptionAndDecryptionCharsWithoutKey() throws Exception { - Files.delete(keyFile); - CryptoService service = new CryptoService(Settings.EMPTY, env); - assertThat(service.isEncryptionEnabled(), is(false)); - final char[] chars = randomAlphaOfLengthBetween(0, 1000).toCharArray(); - final char[] encryptedChars = service.encrypt(chars); - final char[] decryptedChars = service.decrypt(encryptedChars); - assertThat(chars, equalTo(encryptedChars)); - assertThat(chars, equalTo(decryptedChars)); - } - - public void testEncryptionEnabledWithKey() throws Exception { - CryptoService service = new CryptoService(settings, env); - assertThat(service.isEncryptionEnabled(), is(true)); - } - - public void testEncryptionEnabledWithoutKey() throws Exception { - Files.delete(keyFile); - CryptoService service = new CryptoService(Settings.EMPTY, env); - assertThat(service.isEncryptionEnabled(), is(false)); - } - public void testEncryptedChar() throws Exception { - CryptoService service = new CryptoService(settings, env); - assertThat(service.isEncryptionEnabled(), is(true)); + CryptoService service = new CryptoService(settings); assertThat(service.isEncrypted((char[]) null), is(false)); assertThat(service.isEncrypted(new char[0]), is(false)); @@ -86,20 +49,4 @@ public class CryptoServiceTests extends ESTestCase { assertThat(service.isEncrypted(randomAlphaOfLengthBetween(0, 100).toCharArray()), is(false)); assertThat(service.isEncrypted(service.encrypt(randomAlphaOfLength(10).toCharArray())), is(true)); } - - public void testSystemKeyFileRequired() throws Exception { - Files.delete(keyFile); - Settings customSettings = Settings.builder().put(settings).put("xpack.security.system_key.required", true).build(); - FileNotFoundException fnfe = expectThrows(FileNotFoundException.class, () -> new CryptoService(customSettings, env)); - assertThat(fnfe.getMessage(), containsString("must be present with a valid key")); - } - - public void testEmptySystemKeyFile() throws Exception { - // delete and create empty file - Files.delete(keyFile); - Files.createFile(keyFile); - assertTrue(Files.exists(keyFile)); - IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new CryptoService(settings, env)); - assertThat(iae.getMessage(), containsString("Empty key")); - } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java new file mode 100644 index 00000000000..a3830238b4a --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.watcher; + +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.security.crypto.CryptoService; + +public class EncryptSensitiveDataBootstrapCheckTests extends ESTestCase { + + public void testDefaultIsFalse() { + Settings settings = Settings.builder().put("path.home", createTempDir()).build(); + Environment env = new Environment(settings); + EncryptSensitiveDataBootstrapCheck check = new EncryptSensitiveDataBootstrapCheck(settings, env); + assertFalse(check.check()); + assertTrue(check.alwaysEnforce()); + } + + public void testNoKeyInKeystore() { + Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .put(Watcher.ENCRYPT_SENSITIVE_DATA_SETTING.getKey(), true) + .build(); + Environment env = new Environment(settings); + EncryptSensitiveDataBootstrapCheck check = new EncryptSensitiveDataBootstrapCheck(settings, env); + assertTrue(check.check()); + } + + public void testKeyInKeystore() { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), CryptoService.generateKey()); + Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .put(Watcher.ENCRYPT_SENSITIVE_DATA_SETTING.getKey(), true) + .setSecureSettings(secureSettings) + .build(); + Environment env = new Environment(settings); + EncryptSensitiveDataBootstrapCheck check = new EncryptSensitiveDataBootstrapCheck(settings, env); + assertFalse(check.check()); + } +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index c31ce6ea05d..eb0b18f134a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -59,7 +59,6 @@ import org.elasticsearch.xpack.notification.email.Profile; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.authc.file.FileRealm; import org.elasticsearch.xpack.security.authc.support.Hasher; -import org.elasticsearch.xpack.security.crypto.CryptoService; import org.elasticsearch.xpack.support.clock.ClockMock; import org.elasticsearch.xpack.template.TemplateUtils; import org.elasticsearch.xpack.watcher.WatcherState; @@ -163,7 +162,6 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase writeFile(xpackConf, "users", SecuritySettings.USERS); writeFile(xpackConf, "users_roles", SecuritySettings.USER_ROLES); writeFile(xpackConf, "roles.yml", SecuritySettings.ROLES); - writeFile(xpackConf, "system_key", SecuritySettings.systemKey); } catch (final IOException e) { throw new UncheckedIOException(e); } @@ -691,9 +689,6 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase private static final String TEST_PASSWORD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString(TEST_PASSWORD.toCharArray()))); static boolean auditLogsEnabled = SystemPropertyUtil.getBoolean("tests.audit_logs", true); - static byte[] systemKey = generateKey(); // must be the same for all nodes - - public static final String IP_FILTER = "allow: all\n"; public static final String USERS = "transport_client:" + TEST_PASSWORD_HASHED + "\n" + @@ -712,7 +707,7 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/state', 'cluster:monitor/health', 'cluster:monitor/stats'," + " 'cluster:admin/settings/update', 'cluster:admin/repository/delete', 'cluster:monitor/nodes/liveness'," + " 'indices:admin/template/get', 'indices:admin/template/put', 'indices:admin/template/delete'," + - " 'cluster:admin/script/put' ]\n" + + " 'cluster:admin/script/put', 'cluster:monitor/task' ]\n" + " indices:\n" + " - names: '*'\n" + " privileges: [ all ]\n" + @@ -729,7 +724,6 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase if (!enabled) { return builder.put("xpack.security.enabled", false).build(); } - builder.put("xpack.security.enabled", true) .put("xpack.security.authc.realms.esusers.type", FileRealm.TYPE) .put("xpack.security.authc.realms.esusers.order", 0) @@ -742,14 +736,6 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase builder.put(NetworkModule.HTTP_TYPE_KEY, Security.NAME4); return builder.build(); } - - static byte[] generateKey() { - try { - return CryptoService.generateKey(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } } /** diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java index e50952c66cf..6249f9a85e1 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.watcher.test.integration; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.test.http.MockResponse; @@ -14,6 +15,7 @@ import org.elasticsearch.xpack.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.common.http.auth.basic.ApplicableBasicAuth; import org.elasticsearch.xpack.common.http.auth.basic.BasicAuth; import org.elasticsearch.xpack.security.crypto.CryptoService; +import org.elasticsearch.xpack.watcher.Watcher; import org.elasticsearch.xpack.watcher.client.WatcherClient; import org.elasticsearch.xpack.watcher.condition.AlwaysCondition; import org.elasticsearch.xpack.watcher.execution.ActionExecutionMode; @@ -31,7 +33,6 @@ import org.junit.Before; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.loggingAction; import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.webhookAction; import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder; @@ -52,8 +53,9 @@ public class HttpSecretsIntegrationTests extends AbstractWatcherIntegrationTestC static final String USERNAME = "_user"; static final String PASSWORD = "_passwd"; - private MockWebServer webServer = new MockWebServer();; + private MockWebServer webServer = new MockWebServer(); private static Boolean encryptSensitiveData; + private static byte[] encryptionKey; @Before public void init() throws Exception { @@ -68,12 +70,18 @@ public class HttpSecretsIntegrationTests extends AbstractWatcherIntegrationTestC @Override protected Settings nodeSettings(int nodeOrdinal) { if (encryptSensitiveData == null) { - encryptSensitiveData = securityEnabled() && randomBoolean(); + encryptSensitiveData = randomBoolean(); + if (encryptSensitiveData) { + encryptionKey = CryptoService.generateKey(); + } } if (encryptSensitiveData) { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), encryptionKey); return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("xpack.watcher.encrypt_sensitive_data", encryptSensitiveData) + .setSecureSettings(secureSettings) .build(); } return super.nodeSettings(nodeOrdinal); @@ -99,9 +107,12 @@ public class HttpSecretsIntegrationTests extends AbstractWatcherIntegrationTestC Map source = response.getSource(); Object value = XContentMapValues.extractValue("input.http.request.auth.basic.password", source); assertThat(value, notNullValue()); - if (securityEnabled() && encryptSensitiveData) { + if (encryptSensitiveData) { assertThat(value, not(is((Object) PASSWORD))); - CryptoService cryptoService = getInstanceFromMaster(CryptoService.class); + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), encryptionKey); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + CryptoService cryptoService = new CryptoService(settings); assertThat(new String(cryptoService.decrypt(((String) value).toCharArray())), is(PASSWORD)); } else { assertThat(value, is((Object) PASSWORD)); @@ -164,9 +175,12 @@ public class HttpSecretsIntegrationTests extends AbstractWatcherIntegrationTestC Object value = XContentMapValues.extractValue("actions._webhook.webhook.auth.basic.password", source); assertThat(value, notNullValue()); - if (securityEnabled() && encryptSensitiveData) { + if (encryptSensitiveData) { assertThat(value, not(is((Object) PASSWORD))); - CryptoService cryptoService = getInstanceFromMaster(CryptoService.class); + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setFile(Watcher.ENCRYPTION_KEY_SETTING.getKey(), encryptionKey); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + CryptoService cryptoService = new CryptoService(settings); assertThat(new String(cryptoService.decrypt(((String) value).toCharArray())), is(PASSWORD)); } else { assertThat(value, is((Object) PASSWORD)); diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index 5422361579f..c326f0d9671 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -217,6 +217,7 @@ subprojects { } extraConfigFile 'x-pack/system_key', "${mainProject.projectDir}/src/test/resources/system_key" + setting 'xpack.watcher.encrypt_sensitive_data', 'true' } } @@ -245,9 +246,10 @@ subprojects { dependsOn copyTestNodeKeystore extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') if (withSystemKey) { - setting 'xpack.security.system_key.required', 'true' - extraConfigFile 'x-pack/system_key', - "${mainProject.projectDir}/src/test/resources/system_key" + setting 'xpack.watcher.encrypt_sensitive_data', 'true' + setupCommand 'create-elasticsearch-keystore', 'bin/elasticsearch-keystore', 'create' + setupCommand 'add-key-elasticsearch-keystore', + 'bin/elasticsearch-keystore', 'add-file', 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" } } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index aa4b20fb44f..5b1050b7018 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -220,6 +220,7 @@ subprojects { } extraConfigFile 'x-pack/system_key', "${mainProject.projectDir}/src/test/resources/system_key" + setting 'xpack.watcher.encrypt_sensitive_data', 'true' } } @@ -244,9 +245,10 @@ subprojects { dependsOn copyTestNodeKeystore extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') if (withSystemKey) { - setting 'xpack.security.system_key.required', 'true' - extraConfigFile 'x-pack/system_key', - "${mainProject.projectDir}/src/test/resources/system_key" + setting 'xpack.watcher.encrypt_sensitive_data', 'true' + setupCommand 'create-elasticsearch-keystore', 'bin/elasticsearch-keystore', 'create' + setupCommand 'add-key-elasticsearch-keystore', + 'bin/elasticsearch-keystore', 'add-file', 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" } } @@ -271,9 +273,10 @@ subprojects { dependsOn copyTestNodeKeystore extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') if (withSystemKey) { - setting 'xpack.security.system_key.required', 'true' - extraConfigFile 'x-pack/system_key', - "${mainProject.projectDir}/src/test/resources/system_key" + setting 'xpack.watcher.encrypt_sensitive_data', 'true' + setupCommand 'create-elasticsearch-keystore', 'bin/elasticsearch-keystore', 'create' + setupCommand 'add-key-elasticsearch-keystore', + 'bin/elasticsearch-keystore', 'add-file', 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" } }