From e21d58541ad91be9e2d621de59de0e0756c72258 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Fri, 2 Aug 2019 12:29:43 +1000 Subject: [PATCH] Improve errors when TLS files cannot be read (#45122) This change improves the exception messages that are thrown when the system cannot read TLS resources such as keystores, truststores, certificates, keys or certificate-chains (CAs). This change specifically handles: - Files that do not exist - Files that cannot be read due to file-system permissions - Files that cannot be read due to the ES security-manager Backport of: #44787 --- .../xpack/core/ssl/CertParsingUtils.java | 6 +- .../xpack/core/ssl/KeyConfig.java | 29 ++ .../xpack/core/ssl/PEMKeyConfig.java | 37 +- .../xpack/core/ssl/PEMTrustConfig.java | 17 +- .../xpack/core/ssl/PemUtils.java | 7 +- .../xpack/core/ssl/SSLService.java | 33 +- .../xpack/core/ssl/StoreKeyConfig.java | 37 +- .../xpack/core/ssl/StoreTrustConfig.java | 17 +- .../xpack/core/ssl/TrustConfig.java | 56 ++- .../org/elasticsearch/test/TestMatchers.java | 36 ++ .../xpack/core/ssl/CertParsingUtilsTests.java | 3 +- .../ssl/SSLConfigurationReloaderTests.java | 7 +- .../xpack/core/ssl/SSLServiceTests.java | 17 +- .../xpack/core/ssl/StoreKeyConfigTests.java | 3 +- .../xpack/ssl/SSLErrorMessageTests.java | 322 ++++++++++++++++++ .../SSLServiceErrorMessageTests/README.txt | 106 ++++++ .../ssl/SSLServiceErrorMessageTests/ca1.crt | 23 ++ .../ssl/SSLServiceErrorMessageTests/ca1.jks | Bin 0 -> 2313 bytes .../ssl/SSLServiceErrorMessageTests/ca1.p12 | Bin 0 -> 2647 bytes .../SSLServiceErrorMessageTests/cert1a.crt | 21 ++ .../SSLServiceErrorMessageTests/cert1a.jks | Bin 0 -> 4186 bytes .../SSLServiceErrorMessageTests/cert1a.key | 27 ++ .../SSLServiceErrorMessageTests/cert1a.p12 | Bin 0 -> 3615 bytes 23 files changed, 752 insertions(+), 52 deletions(-) create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageTests.java create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/README.txt create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.crt create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.jks create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.p12 create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/cert1a.crt create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/cert1a.jks create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/cert1a.key create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/cert1a.p12 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java index 0a902aba22c..7f16c6097ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java @@ -63,6 +63,10 @@ public class CertParsingUtils { return PathUtils.get(path).normalize(); } + static List resolvePaths(List certPaths, @Nullable Environment environment) { + return certPaths.stream().map(p -> resolvePath(p, environment)).collect(Collectors.toList()); + } + public static KeyStore readKeyStore(Path path, String type, char[] password) throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException { try (InputStream in = Files.newInputStream(path)) { @@ -82,7 +86,7 @@ public class CertParsingUtils { */ public static Certificate[] readCertificates(List certPaths, @Nullable Environment environment) throws CertificateException, IOException { - final List resolvedPaths = certPaths.stream().map(p -> resolvePath(p, environment)).collect(Collectors.toList()); + final List resolvedPaths = resolvePaths(certPaths, environment); return readCertificates(resolvedPaths); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/KeyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/KeyConfig.java index 823f0da367e..a0aeac73107 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/KeyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/KeyConfig.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.ssl; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.env.Environment; import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo; @@ -12,7 +13,10 @@ import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo; import javax.net.ssl.X509ExtendedKeyManager; import javax.net.ssl.X509ExtendedTrustManager; +import java.io.IOException; +import java.nio.file.AccessDeniedException; import java.nio.file.Path; +import java.security.AccessControlException; import java.security.PrivateKey; import java.util.Collection; import java.util.Collections; @@ -64,6 +68,31 @@ abstract class KeyConfig extends TrustConfig { abstract X509ExtendedKeyManager createKeyManager(@Nullable Environment environment); + /** + * generate a new exception caused by a missing file, that is required for this key config + */ + static ElasticsearchException missingKeyConfigFile(IOException cause, String fileType, Path path) { + return new ElasticsearchException( + "failed to initialize SSL KeyManager - " + fileType + " file [{}] does not exist", cause, path.toAbsolutePath()); + } + + /** + * generate a new exception caused by an unreadable file (i.e. file-system access denied), that is required for this key config + */ + static ElasticsearchException unreadableKeyConfigFile(AccessDeniedException cause, String fileType, Path path) { + return new ElasticsearchException( + "failed to initialize SSL KeyManager - not permitted to read " + fileType + " file [{}]", cause, path.toAbsolutePath()); + } + + /** + * generate a new exception caused by a blocked file (i.e. security-manager access denied), that is required for this key config + */ + static ElasticsearchException blockedKeyConfigFile(AccessControlException cause, Environment environment, String fileType, Path path) { + return new ElasticsearchException( + "failed to initialize SSL KeyManager - access to read {} file [{}] is blocked;" + + " SSL resources should be placed in the [{}] directory", cause, fileType, path, environment.configFile()); + } + abstract List privateKeys(@Nullable Environment environment); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PEMKeyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PEMKeyConfig.java index 3cc147c0a0b..21ee39e5703 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PEMKeyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PEMKeyConfig.java @@ -14,9 +14,13 @@ import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo; import javax.net.ssl.X509ExtendedKeyManager; import javax.net.ssl.X509ExtendedTrustManager; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.security.AccessControlException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; @@ -35,6 +39,9 @@ import java.util.Objects; */ class PEMKeyConfig extends KeyConfig { + private static final String CERTIFICATE_FILE = "certificate"; + private static final String KEY_FILE = "key"; + private final String keyPath; private final SecureString keyPassword; private final String certPath; @@ -55,7 +62,7 @@ class PEMKeyConfig extends KeyConfig { @Override X509ExtendedKeyManager createKeyManager(@Nullable Environment environment) { try { - PrivateKey privateKey = readPrivateKey(CertParsingUtils.resolvePath(keyPath, environment), keyPassword); + PrivateKey privateKey = readPrivateKey(keyPath, keyPassword, environment); if (privateKey == null) { throw new IllegalArgumentException("private key [" + keyPath + "] could not be loaded"); } @@ -63,12 +70,21 @@ class PEMKeyConfig extends KeyConfig { return CertParsingUtils.keyManager(certificateChain, privateKey, keyPassword.getChars()); } catch (IOException | UnrecoverableKeyException | NoSuchAlgorithmException | CertificateException | KeyStoreException e) { - throw new ElasticsearchException("failed to initialize a KeyManagerFactory", e); + throw new ElasticsearchException("failed to initialize SSL KeyManagerFactory", e); } } private Certificate[] getCertificateChain(@Nullable Environment environment) throws CertificateException, IOException { - return CertParsingUtils.readCertificates(Collections.singletonList(certPath), environment); + final Path certificate = CertParsingUtils.resolvePath(certPath, environment); + try { + return CertParsingUtils.readCertificates(Collections.singletonList(certificate)); + } catch (FileNotFoundException | NoSuchFileException fileException) { + throw missingKeyConfigFile(fileException, CERTIFICATE_FILE, certificate); + } catch (AccessDeniedException accessException) { + throw unreadableKeyConfigFile(accessException, CERTIFICATE_FILE, certificate); + } catch (AccessControlException securityException) { + throw blockedKeyConfigFile(securityException, environment, CERTIFICATE_FILE, certificate); + } } @Override @@ -87,14 +103,23 @@ class PEMKeyConfig extends KeyConfig { @Override List privateKeys(@Nullable Environment environment) { try { - return Collections.singletonList(readPrivateKey(CertParsingUtils.resolvePath(keyPath, environment), keyPassword)); + return Collections.singletonList(readPrivateKey(keyPath, keyPassword, environment)); } catch (IOException e) { throw new UncheckedIOException("failed to read key", e); } } - private static PrivateKey readPrivateKey(Path keyPath, SecureString keyPassword) throws IOException { - return PemUtils.readPrivateKey(keyPath, keyPassword::getChars); + private static PrivateKey readPrivateKey(String keyPath, SecureString keyPassword, Environment environment) throws IOException { + final Path key = CertParsingUtils.resolvePath(keyPath, environment); + try { + return PemUtils.readPrivateKey(key, keyPassword::getChars); + } catch (FileNotFoundException | NoSuchFileException fileException) { + throw missingKeyConfigFile(fileException, KEY_FILE, key); + } catch (AccessDeniedException accessException) { + throw unreadableKeyConfigFile(accessException, KEY_FILE, key); + } catch (AccessControlException securityException) { + throw blockedKeyConfigFile(securityException, environment, KEY_FILE, key); + } } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PEMTrustConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PEMTrustConfig.java index 1fd163c9915..146b8e0065e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PEMTrustConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PEMTrustConfig.java @@ -14,7 +14,10 @@ import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo; import javax.net.ssl.X509ExtendedTrustManager; import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.security.AccessControlException; import java.security.cert.Certificate; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; @@ -29,10 +32,13 @@ import java.util.Objects; */ class PEMTrustConfig extends TrustConfig { + private static final String CA_FILE = "certificate_authorities"; + private final List caPaths; /** * Create a new trust configuration that is built from the certificate files + * * @param caPaths the paths to the certificate files to trust */ PEMTrustConfig(List caPaths) { @@ -44,8 +50,17 @@ class PEMTrustConfig extends TrustConfig { try { Certificate[] certificates = CertParsingUtils.readCertificates(caPaths, environment); return CertParsingUtils.trustManager(certificates); + } catch (NoSuchFileException noSuchFileException) { + final Path missingPath = CertParsingUtils.resolvePath(noSuchFileException.getFile(), environment); + throw missingTrustConfigFile(noSuchFileException, CA_FILE, missingPath); + } catch (AccessDeniedException accessDeniedException) { + final Path missingPath = CertParsingUtils.resolvePath(accessDeniedException.getFile(), environment); + throw unreadableTrustConfigFile(accessDeniedException, CA_FILE, missingPath); + } catch (AccessControlException accessControlException) { + final List paths = CertParsingUtils.resolvePaths(caPaths, environment); + throw blockedTrustConfigFile(accessControlException, environment, CA_FILE, paths); } catch (Exception e) { - throw new ElasticsearchException("failed to initialize a TrustManagerFactory", e); + throw new ElasticsearchException("failed to initialize SSL TrustManager", e); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java index 1e67e15a33c..13bd7e95798 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/PemUtils.java @@ -16,6 +16,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.GeneralSecurityException; +import java.security.KeyException; import java.security.KeyFactory; import java.security.KeyPairGenerator; import java.security.MessageDigest; @@ -72,7 +73,7 @@ public class PemUtils { * @param passwordSupplier A password supplier for the potentially encrypted (password protected) key * @return a private key from the contents of the file */ - public static PrivateKey readPrivateKey(Path keyPath, Supplier passwordSupplier) { + public static PrivateKey readPrivateKey(Path keyPath, Supplier passwordSupplier) throws IOException { try (BufferedReader bReader = Files.newBufferedReader(keyPath, StandardCharsets.UTF_8)) { String line = bReader.readLine(); while (null != line && line.startsWith(HEADER) == false){ @@ -103,7 +104,7 @@ public class PemUtils { throw new IllegalStateException("Error parsing Private Key from: " + keyPath.toString() + ". File did not contain a " + "supported key format"); } - } catch (IOException | GeneralSecurityException e) { + } catch (GeneralSecurityException e) { throw new IllegalStateException("Error parsing Private Key from: " + keyPath.toString(), e); } } @@ -176,7 +177,7 @@ public class PemUtils { line = bReader.readLine(); } if (null == line || PKCS8_FOOTER.equals(line.trim()) == false) { - throw new IOException("Malformed PEM file, PEM footer is invalid or missing"); + throw new KeyException("Malformed PEM file, PEM footer is invalid or missing"); } byte[] keyBytes = Base64.getDecoder().decode(sb.toString()); String keyAlgo = getKeyAlgorithmIdentifier(keyBytes); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java index 3611b6663a3..539205e251f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -420,27 +421,33 @@ public class SSLService { sslSettingsMap.putAll(getRealmsSSLSettings(settings)); sslSettingsMap.putAll(getMonitoringExporterSettings(settings)); - sslSettingsMap.forEach((key, sslSettings) -> { - final SSLConfiguration configuration = new SSLConfiguration(sslSettings); - storeSslConfiguration(key, configuration); - sslContextHolders.computeIfAbsent(configuration, this::createSslContext); - }); + sslSettingsMap.forEach((key, sslSettings) -> loadConfiguration(key, sslSettings, sslContextHolders)); final Settings transportSSLSettings = settings.getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX); - final SSLConfiguration transportSSLConfiguration = new SSLConfiguration(transportSSLSettings); + final SSLConfiguration transportSSLConfiguration = + loadConfiguration(XPackSettings.TRANSPORT_SSL_PREFIX, transportSSLSettings, sslContextHolders); this.transportSSLConfiguration.set(transportSSLConfiguration); - storeSslConfiguration(XPackSettings.TRANSPORT_SSL_PREFIX, transportSSLConfiguration); Map profileSettings = getTransportProfileSSLSettings(settings); - sslContextHolders.computeIfAbsent(transportSSLConfiguration, this::createSslContext); - profileSettings.forEach((key, profileSetting) -> { - final SSLConfiguration configuration = new SSLConfiguration(profileSetting); - storeSslConfiguration(key, configuration); - sslContextHolders.computeIfAbsent(configuration, this::createSslContext); - }); + profileSettings.forEach((key, profileSetting) -> loadConfiguration(key, profileSetting, sslContextHolders)); return Collections.unmodifiableMap(sslContextHolders); } + private SSLConfiguration loadConfiguration(String key, Settings settings, Map contextHolders) { + if (key.endsWith(".")) { + // Drop trailing '.' so that any exception messages are consistent + key = key.substring(0, key.length() - 1); + } + try { + final SSLConfiguration configuration = new SSLConfiguration(settings); + storeSslConfiguration(key, configuration); + contextHolders.computeIfAbsent(configuration, this::createSslContext); + return configuration; + } catch (Exception e) { + throw new ElasticsearchSecurityException("failed to load SSL configuration [{}]", e, key); + } + } + private void storeSslConfiguration(String key, SSLConfiguration configuration) { if (key.endsWith(".")) { key = key.substring(0, key.length() - 1); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfig.java index 3337465994c..fc234b517d2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfig.java @@ -14,17 +14,18 @@ import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo; import javax.net.ssl.X509ExtendedKeyManager; import javax.net.ssl.X509ExtendedTrustManager; +import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.security.AccessControlException; import java.security.GeneralSecurityException; import java.security.Key; import java.security.KeyStore; import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; -import java.security.UnrecoverableKeyException; import java.security.cert.Certificate; -import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collection; @@ -38,6 +39,8 @@ import java.util.Objects; */ class StoreKeyConfig extends KeyConfig { + private static final String KEYSTORE_FILE = "keystore"; + final String keyStorePath; final String keyStoreType; final SecureString keyStorePassword; @@ -68,28 +71,42 @@ class StoreKeyConfig extends KeyConfig { @Override X509ExtendedKeyManager createKeyManager(@Nullable Environment environment) { + Path ksPath = keyStorePath == null ? null : CertParsingUtils.resolvePath(keyStorePath, environment); try { - KeyStore ks = getStore(environment, keyStorePath, keyStoreType, keyStorePassword); + KeyStore ks = getStore(ksPath, keyStoreType, keyStorePassword); checkKeyStore(ks); return CertParsingUtils.keyManager(ks, keyPassword.getChars(), keyStoreAlgorithm); - } catch (IOException | CertificateException | NoSuchAlgorithmException | UnrecoverableKeyException | KeyStoreException e) { - throw new ElasticsearchException("failed to initialize a KeyManagerFactory", e); + } catch (FileNotFoundException | NoSuchFileException e) { + throw missingKeyConfigFile(e, KEYSTORE_FILE, ksPath); + } catch (AccessDeniedException e) { + throw unreadableKeyConfigFile(e, KEYSTORE_FILE, ksPath); + } catch (AccessControlException e) { + throw blockedKeyConfigFile(e, environment, KEYSTORE_FILE, ksPath); + } catch (IOException | GeneralSecurityException e) { + throw new ElasticsearchException("failed to initialize SSL KeyManager", e); } } @Override X509ExtendedTrustManager createTrustManager(@Nullable Environment environment) { + final Path ksPath = CertParsingUtils.resolvePath(keyStorePath, environment); try { - KeyStore ks = getStore(environment, keyStorePath, keyStoreType, keyStorePassword); + KeyStore ks = getStore(ksPath, keyStoreType, keyStorePassword); return CertParsingUtils.trustManager(ks, trustStoreAlgorithm); - } catch (IOException | CertificateException | NoSuchAlgorithmException | KeyStoreException e) { - throw new ElasticsearchException("failed to initialize a TrustManagerFactory", e); + } catch (FileNotFoundException | NoSuchFileException e) { + throw missingTrustConfigFile(e, KEYSTORE_FILE, ksPath); + } catch (AccessDeniedException e) { + throw missingTrustConfigFile(e, KEYSTORE_FILE, ksPath); + } catch (AccessControlException e) { + throw blockedTrustConfigFile(e, environment, KEYSTORE_FILE, Collections.singletonList(ksPath)); + } catch (IOException | GeneralSecurityException e) { + throw new ElasticsearchException("failed to initialize SSL TrustManager", e); } } @Override Collection certificates(Environment environment) throws GeneralSecurityException, IOException { - final KeyStore trustStore = getStore(environment, keyStorePath, keyStoreType, keyStorePassword); + final KeyStore trustStore = getStore(CertParsingUtils.resolvePath(keyStorePath, environment), keyStoreType, keyStorePassword); final List certificates = new ArrayList<>(); final Enumeration aliases = trustStore.aliases(); while (aliases.hasMoreElements()) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/StoreTrustConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/StoreTrustConfig.java index d4848f98339..6564035389d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/StoreTrustConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/StoreTrustConfig.java @@ -13,8 +13,12 @@ import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo; import javax.net.ssl.X509ExtendedTrustManager; +import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.security.AccessControlException; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.cert.Certificate; @@ -31,6 +35,8 @@ import java.util.Objects; */ class StoreTrustConfig extends TrustConfig { + private static final String TRUSTSTORE_FILE = "truststore"; + final String trustStorePath; final String trustStoreType; final SecureString trustStorePassword; @@ -54,11 +60,18 @@ class StoreTrustConfig extends TrustConfig { @Override X509ExtendedTrustManager createTrustManager(@Nullable Environment environment) { + final Path storePath = CertParsingUtils.resolvePath(trustStorePath, environment); try { - KeyStore trustStore = getStore(environment, trustStorePath, trustStoreType, trustStorePassword); + KeyStore trustStore = getStore(storePath, trustStoreType, trustStorePassword); return CertParsingUtils.trustManager(trustStore, trustStoreAlgorithm); + } catch (FileNotFoundException | NoSuchFileException e) { + throw missingTrustConfigFile(e, TRUSTSTORE_FILE, storePath); + } catch (AccessDeniedException e) { + throw unreadableTrustConfigFile(e, TRUSTSTORE_FILE, storePath); + } catch (AccessControlException e) { + throw blockedTrustConfigFile(e, environment, TRUSTSTORE_FILE, Collections.singletonList(storePath)); } catch (Exception e) { - throw new ElasticsearchException("failed to initialize a TrustManagerFactory", e); + throw new ElasticsearchException("failed to initialize SSL TrustManager", e); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/TrustConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/TrustConfig.java index a9bc737c943..4d0f6a6c0b0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/TrustConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/TrustConfig.java @@ -12,11 +12,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo; import javax.net.ssl.X509ExtendedTrustManager; - import java.io.IOException; import java.io.InputStream; +import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.Path; +import java.security.AccessControlException; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; @@ -65,12 +66,20 @@ abstract class TrustConfig { */ public abstract int hashCode(); + /** + * @deprecated Use {@link #getStore(Path, String, SecureString)} instead + */ + @Deprecated + KeyStore getStore(@Nullable Environment environment, @Nullable String storePath, String storeType, SecureString storePassword) + throws GeneralSecurityException, IOException { + return getStore(CertParsingUtils.resolvePath(storePath, environment), storeType, storePassword); + } + /** * Loads and returns the appropriate {@link KeyStore} for the given configuration. The KeyStore can be backed by a file * in any format that the Security Provider might support, or a cryptographic software or hardware token in the case * of a PKCS#11 Provider. * - * @param environment the environment to resolve files against or null in the case of running in a transport client * @param storePath the path to the {@link KeyStore} to load, or null if a PKCS11 token is configured as the keystore/truststore * of the JVM * @param storeType the type of the {@link KeyStore} @@ -81,10 +90,9 @@ abstract class TrustConfig { * @throws NoSuchAlgorithmException if the algorithm used to check the integrity of the keystore cannot be found * @throws IOException if there is an I/O issue with the KeyStore data or the password is incorrect */ - KeyStore getStore(@Nullable Environment environment, @Nullable String storePath, String storeType, SecureString storePassword) - throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException { + KeyStore getStore(@Nullable Path storePath, String storeType, SecureString storePassword) throws IOException, GeneralSecurityException { if (null != storePath) { - try (InputStream in = Files.newInputStream(CertParsingUtils.resolvePath(storePath, environment))) { + try (InputStream in = Files.newInputStream(storePath)) { KeyStore ks = KeyStore.getInstance(storeType); ks.load(in, storePassword.getChars()); return ks; @@ -97,6 +105,44 @@ abstract class TrustConfig { throw new IllegalArgumentException("keystore.path or truststore.path can only be empty when using a PKCS#11 token"); } + /** + * generate a new exception caused by a missing file, that is required for this trust config + */ + protected ElasticsearchException missingTrustConfigFile(IOException cause, String fileType, Path path) { + return new ElasticsearchException( + "failed to initialize SSL TrustManager - " + fileType + " file [{}] does not exist", cause, path.toAbsolutePath()); + } + + /** + * generate a new exception caused by an unreadable file (i.e. file-system access denied), that is required for this trust config + */ + protected ElasticsearchException unreadableTrustConfigFile(AccessDeniedException cause, String fileType, Path path) { + return new ElasticsearchException( + "failed to initialize SSL TrustManager - not permitted to read " + fileType + " file [{}]", cause, path.toAbsolutePath()); + } + + /** + * generate a new exception caused by a blocked file (i.e. security-manager access denied), that is required for this trust config + * @param paths A list of possible files. Depending on the context, it may not be possible to know exactly which file caused the + * exception, so this method accepts multiple paths. + */ + protected ElasticsearchException blockedTrustConfigFile(AccessControlException cause, Environment environment, + String fileType, List paths) { + if (paths.size() == 1) { + return new ElasticsearchException( + "failed to initialize SSL TrustManager - access to read {} file [{}] is blocked;" + + " SSL resources should be placed in the [{}] directory", + cause, fileType, paths.get(0).toAbsolutePath(), environment.configFile()); + } else { + final String pathString = paths.stream().map(Path::toAbsolutePath).map(Path::toString).collect(Collectors.joining(", ")); + return new ElasticsearchException( + "failed to initialize SSL TrustManager - access to read one or more of the {} files [{}] is blocked;" + + " SSL resources should be placed in the [{}] directory", + cause, fileType, pathString, environment.configFile()); + } + } + + /** * A trust configuration that is a combination of a trust configuration with the default JDK trust configuration. This trust * configuration returns a trust manager verifies certificates against both the default JDK trusted configurations and the specific diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java index 2a9041575df..b5aed4eae2f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java @@ -5,7 +5,10 @@ */ package org.elasticsearch.test; +import org.hamcrest.BaseMatcher; +import org.hamcrest.CoreMatchers; import org.hamcrest.CustomMatcher; +import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.Matchers; @@ -26,6 +29,39 @@ public class TestMatchers extends Matchers { }; } + public static Matcher throwableWithMessage(String message) { + return throwableWithMessage(CoreMatchers.equalTo(message)); + } + + public static Matcher throwableWithMessage(Matcher messageMatcher) { + return new BaseMatcher() { + @Override + public void describeTo(Description description) { + description.appendText("a throwable with message of ").appendDescriptionOf(messageMatcher); + } + + @Override + public boolean matches(Object actual) { + if (actual instanceof Throwable) { + final Throwable throwable = (Throwable) actual; + return messageMatcher.matches(throwable.getMessage()); + } else { + return false; + } + } + + @Override + public void describeMismatch(Object item, Description description) { + super.describeMismatch(item, description); + if (item instanceof Throwable) { + Throwable e = (Throwable) item; + final StackTraceElement at = e.getStackTrace()[0]; + description.appendText(" at ").appendText(at.toString()); + } + } + }; + } + public static Matcher> predicateMatches(T value) { return new CustomMatcher>("Matches " + value) { @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertParsingUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertParsingUtilsTests.java index 49ce0fdb80e..bc68b864c35 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertParsingUtilsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertParsingUtilsTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.ssl; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -86,7 +87,7 @@ public class CertParsingUtilsTests extends ESTestCase { assertEquals("EC", cert.getPublicKey().getAlgorithm()); } - private void verifyPrime256v1ECKey(Path keyPath) { + private void verifyPrime256v1ECKey(Path keyPath) throws IOException { PrivateKey privateKey = PemUtils.readPrivateKey(keyPath, () -> null); assertEquals("EC", privateKey.getAlgorithm()); assertThat(privateKey, instanceOf(ECPrivateKey.class)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java index e22750e930a..1bb7159db49 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java @@ -69,6 +69,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import static org.elasticsearch.test.TestMatchers.throwableWithMessage; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.sameInstance; @@ -353,7 +354,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase { latch.await(); assertNotNull(exceptionRef.get()); - assertThat(exceptionRef.get().getMessage(), containsString("failed to initialize a KeyManagerFactory")); + assertThat(exceptionRef.get(), throwableWithMessage(containsString("failed to initialize SSL KeyManager"))); assertThat(sslService.sslContextHolder(config).sslContext(), sameInstance(context)); } @@ -451,7 +452,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase { latch.await(); assertNotNull(exceptionRef.get()); - assertThat(exceptionRef.get().getMessage(), containsString("failed to initialize a TrustManagerFactory")); + assertThat(exceptionRef.get(), throwableWithMessage(containsString("failed to initialize SSL TrustManager"))); assertThat(sslService.sslContextHolder(config).sslContext(), sameInstance(context)); } @@ -497,7 +498,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase { latch.await(); assertNotNull(exceptionRef.get()); - assertThat(exceptionRef.get().getMessage(), containsString("failed to initialize a TrustManagerFactory")); + assertThat(exceptionRef.get(), throwableWithMessage(containsString("failed to initialize SSL TrustManager"))); assertThat(sslService.sslContextHolder(config).sslContext(), sameInstance(context)); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java index ae48de463e8..ce043b4597d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java @@ -60,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import static org.elasticsearch.test.TestMatchers.throwableWithMessage; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -194,7 +195,8 @@ public class SSLServiceTests extends ESTestCase { sslService.createSSLEngine(configuration, null, -1); fail("expected an exception"); } catch (ElasticsearchException e) { - assertThat(e.getMessage(), containsString("failed to initialize a KeyManagerFactory")); + assertThat(e, throwableWithMessage("failed to load SSL configuration [xpack.security.transport.ssl]")); + assertThat(e.getCause(), throwableWithMessage(containsString("failed to initialize SSL KeyManager"))); } } @@ -326,7 +328,8 @@ public class SSLServiceTests extends ESTestCase { .build(); ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> new SSLService(settings, env)); - assertThat(e.getMessage(), is("failed to initialize a TrustManagerFactory")); + assertThat(e, throwableWithMessage("failed to load SSL configuration [xpack.security.transport.ssl]")); + assertThat(e.getCause(), throwableWithMessage(containsString("failed to initialize SSL TrustManager"))); } public void testThatKeystorePasswordIsRequired() throws Exception { @@ -336,7 +339,8 @@ public class SSLServiceTests extends ESTestCase { .build(); ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> new SSLService(settings, env)); - assertThat(e.getMessage(), is("failed to create trust manager")); + assertThat(e, throwableWithMessage("failed to load SSL configuration [xpack.security.transport.ssl]")); + assertThat(e.getCause(), throwableWithMessage("failed to create trust manager")); } public void testCiphersAndInvalidCiphersWork() throws Exception { @@ -369,9 +373,10 @@ public class SSLServiceTests extends ESTestCase { .setSecureSettings(secureSettings) .putList("xpack.security.transport.ssl.cipher_suites", new String[] { "foo", "bar" }) .build(); - IllegalArgumentException e = - expectThrows(IllegalArgumentException.class, () -> new SSLService(settings, env)); - assertThat(e.getMessage(), is("none of the ciphers [foo, bar] are supported by this JVM")); + ElasticsearchException e = + expectThrows(ElasticsearchException.class, () -> new SSLService(settings, env)); + assertThat(e, throwableWithMessage("failed to load SSL configuration [xpack.security.transport.ssl]")); + assertThat(e.getCause(), throwableWithMessage("none of the ciphers [foo, bar] are supported by this JVM")); } public void testThatSSLEngineHasCipherSuitesOrderSet() throws Exception { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfigTests.java index a7d6088bc7a..f909f90b2ee 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfigTests.java @@ -17,6 +17,7 @@ import javax.net.ssl.X509ExtendedKeyManager; import java.security.PrivateKey; +import static org.elasticsearch.test.TestMatchers.throwableWithMessage; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -46,7 +47,7 @@ public class StoreKeyConfigTests extends ESTestCase { KeyManagerFactory.getDefaultAlgorithm(), TrustManagerFactory.getDefaultAlgorithm()); ElasticsearchException ee = expectThrows(ElasticsearchException.class, () -> keyConfigPkcs11.createKeyManager(TestEnvironment.newEnvironment(settings))); - assertThat(ee.getMessage(), containsString("failed to initialize a KeyManagerFactory")); + assertThat(ee, throwableWithMessage(containsString("failed to initialize SSL KeyManager"))); assertThat(ee.getCause().getMessage(), containsString("PKCS11 not found")); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageTests.java new file mode 100644 index 00000000000..bd9d78b99f7 --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageTests.java @@ -0,0 +1,322 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ssl; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ssl.SSLService; +import org.junit.Before; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessDeniedException; +import java.nio.file.CopyOption; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.security.AccessControlException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +import static org.elasticsearch.test.SecuritySettingsSource.addSecureSettings; +import static org.elasticsearch.test.TestMatchers.throwableWithMessage; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.instanceOf; + +/** + * This is a suite of tests to ensure that meaningful error messages are generated for defined SSL configuration problems. + */ +public class SSLErrorMessageTests extends ESTestCase { + + private Environment env; + private Map paths; + + @Before + public void setup() throws Exception { + env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); + paths = new HashMap<>(); + + requirePath("ca1.p12"); + requirePath("ca1.jks"); + requirePath("ca1.crt"); + + requirePath("cert1a.p12"); + requirePath("cert1a.jks"); + requirePath("cert1a.crt"); + requirePath("cert1a.key"); + } + + public void testMessageForMissingKeystore() { + checkMissingKeyManagerResource("keystore", "keystore.path", null); + } + + public void testMessageForMissingPemCertificate() { + checkMissingKeyManagerResource("certificate", "certificate", withKey("cert1a.key")); + } + + public void testMessageForMissingPemKey() { + checkMissingKeyManagerResource("key", "key", withCertificate("cert1a.crt")); + } + + public void testMessageForMissingTruststore() { + checkMissingTrustManagerResource("truststore", "truststore.path"); + } + + public void testMessageForMissingCertificateAuthorities() { + checkMissingTrustManagerResource("certificate_authorities", "certificate_authorities"); + } + + public void testMessageForKeystoreWithoutReadAccess() throws Exception { + checkUnreadableKeyManagerResource("cert1a.p12", "keystore", "keystore.path", null); + } + + public void testMessageForPemCertificateWithoutReadAccess() throws Exception { + checkUnreadableKeyManagerResource("cert1a.crt", "certificate", "certificate", withKey("cert1a.key")); + } + + public void testMessageForPemKeyWithoutReadAccess() throws Exception { + checkUnreadableKeyManagerResource("cert1a.key", "key", "key", withCertificate("cert1a.crt")); + } + + public void testMessageForTruststoreWithoutReadAccess() throws Exception { + checkUnreadableTrustManagerResource("cert1a.p12", "truststore", "truststore.path"); + } + + public void testMessageForCertificateAuthoritiesWithoutReadAccess() throws Exception { + checkUnreadableTrustManagerResource("ca1.crt", "certificate_authorities", "certificate_authorities"); + } + + public void testMessageForKeyStoreOutsideConfigDir() throws Exception { + checkBlockedKeyManagerResource("keystore", "keystore.path", null); + } + + public void testMessageForPemCertificateOutsideConfigDir() throws Exception { + checkBlockedKeyManagerResource("certificate", "certificate", withKey("cert1a.key")); + } + + public void testMessageForPemKeyOutsideConfigDir() throws Exception { + checkBlockedKeyManagerResource("key", "key", withCertificate("cert1a.crt")); + } + + public void testMessageForTrustStoreOutsideConfigDir() throws Exception { + checkBlockedTrustManagerResource("truststore", "truststore.path"); + } + + public void testMessageForCertificateAuthoritiesOutsideConfigDir() throws Exception { + checkBlockedTrustManagerResource("certificate_authorities", "certificate_authorities"); + } + + private void checkMissingKeyManagerResource(String fileType, String configKey, @Nullable Settings.Builder additionalSettings) { + checkMissingResource("KeyManager", fileType, configKey, + (prefix, builder) -> buildKeyConfigSettings(additionalSettings, prefix, builder)); + } + + private void buildKeyConfigSettings(@Nullable Settings.Builder additionalSettings, String prefix, Settings.Builder builder) { + configureWorkingTruststore(prefix, builder); + if (additionalSettings != null) { + builder.put(additionalSettings.normalizePrefix(prefix + ".").build()); + } + } + + private void checkMissingTrustManagerResource(String fileType, String configKey) { + checkMissingResource("TrustManager", fileType, configKey, this::configureWorkingKeystore); + } + + private void checkUnreadableKeyManagerResource(String fromResource, String fileType, String configKey, + @Nullable Settings.Builder additionalSettings) throws Exception { + checkUnreadableResource("KeyManager", fromResource, fileType, configKey, + (prefix, builder) -> buildKeyConfigSettings(additionalSettings, prefix, builder)); + } + + private void checkUnreadableTrustManagerResource(String fromResource, String fileType, String configKey) throws Exception { + checkUnreadableResource("TrustManager", fromResource, fileType, configKey, this::configureWorkingKeystore); + } + + private void checkBlockedKeyManagerResource(String fileType, String configKey, Settings.Builder additionalSettings) throws Exception { + checkBlockedResource("KeyManager", fileType, configKey, + (prefix, builder) -> buildKeyConfigSettings(additionalSettings, prefix, builder)); + } + + private void checkBlockedTrustManagerResource(String fileType, String configKey) throws Exception { + checkBlockedResource("TrustManager", fileType, configKey, this::configureWorkingKeystore); + } + + private void checkMissingResource(String sslManagerType, String fileType, String configKey, + BiConsumer configure) { + final String prefix = randomSslPrefix(); + final Settings.Builder settings = Settings.builder(); + configure.accept(prefix, settings); + + final String fileName = missingFile(); + final String key = prefix + "." + configKey; + settings.put(key, fileName); + + Throwable exception = expectFailure(settings); + assertThat(exception, throwableWithMessage("failed to load SSL configuration [" + prefix + "]")); + assertThat(exception, instanceOf(ElasticsearchSecurityException.class)); + + exception = exception.getCause(); + assertThat(exception, throwableWithMessage( + "failed to initialize SSL " + sslManagerType + " - " + fileType + " file [" + fileName + "] does not exist")); + assertThat(exception, instanceOf(ElasticsearchException.class)); + + exception = exception.getCause(); + assertThat(exception, instanceOf(NoSuchFileException.class)); + assertThat(exception, throwableWithMessage(fileName)); + } + + private void checkUnreadableResource(String sslManagerType, String fromResource, String fileType, String configKey, + BiConsumer configure) throws Exception { + final String prefix = randomSslPrefix(); + final Settings.Builder settings = Settings.builder(); + configure.accept(prefix, settings); + + final String fileName = unreadableFile(fromResource); + final String key = prefix + "." + configKey; + settings.put(key, fileName); + + Throwable exception = expectFailure(settings); + assertThat(exception, throwableWithMessage("failed to load SSL configuration [" + prefix + "]")); + assertThat(exception, instanceOf(ElasticsearchSecurityException.class)); + + exception = exception.getCause(); + assertThat(exception, throwableWithMessage( + "failed to initialize SSL " + sslManagerType + " - not permitted to read " + fileType + " file [" + fileName + "]")); + assertThat(exception, instanceOf(ElasticsearchException.class)); + + exception = exception.getCause(); + assertThat(exception, instanceOf(AccessDeniedException.class)); + assertThat(exception, throwableWithMessage(fileName)); + } + + private void checkBlockedResource(String sslManagerType, String fileType, String configKey, + BiConsumer configure) throws Exception { + final String prefix = randomSslPrefix(); + final Settings.Builder settings = Settings.builder(); + configure.accept(prefix, settings); + + final String fileName = blockedFile(); + final String key = prefix + "." + configKey; + settings.put(key, fileName); + + Throwable exception = expectFailure(settings); + assertThat(exception, throwableWithMessage("failed to load SSL configuration [" + prefix + "]")); + assertThat(exception, instanceOf(ElasticsearchSecurityException.class)); + + exception = exception.getCause(); + assertThat(exception, throwableWithMessage( + "failed to initialize SSL " + sslManagerType + " - access to read " + fileType + " file [" + fileName + + "] is blocked; SSL resources should be placed in the [" + env.configFile() + "] directory")); + assertThat(exception, instanceOf(ElasticsearchException.class)); + + exception = exception.getCause(); + assertThat(exception, instanceOf(AccessControlException.class)); + assertThat(exception, throwableWithMessage(containsString(fileName))); + } + + private String missingFile() { + return resource("cert1a.p12").replace("cert1a.p12", "file.dne"); + } + + private String unreadableFile(String fromResource) throws IOException { + assumeFalse("This behaviour uses POSIX file permissions", Constants.WINDOWS); + final Path fromPath = this.paths.get(fromResource); + if (fromPath == null) { + throw new IllegalArgumentException("Test SSL resource " + fromResource + " has not been loaded"); + } + return copy(fromPath, createTempFile(fromResource, "-no-read"), PosixFilePermissions.fromString("---------")); + } + + private String blockedFile() throws IOException { + return "/this/path/is/outside/the/config/directory/file.error"; + } + + /** + * This more-or-less replicates the functionality of {@link Files#copy(Path, Path, CopyOption...)} but doing it this way allows us to + * set the file permissions when creating the file (which helps with security manager issues) + */ + private String copy(Path fromPath, Path toPath, Set permissions) throws IOException { + Files.deleteIfExists(toPath); + final FileAttribute> fileAttributes = PosixFilePermissions.asFileAttribute(permissions); + final EnumSet options = EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); + try (SeekableByteChannel channel = Files.newByteChannel(toPath, options, fileAttributes); + OutputStream out = Channels.newOutputStream(channel)) { + Files.copy(fromPath, out); + } + return toPath.toString(); + } + + private Settings.Builder withKey(String fileName) { + assertThat(fileName, endsWith(".key")); + return Settings.builder().put("key", resource(fileName)); + } + + private Settings.Builder withCertificate(String fileName) { + assertThat(fileName, endsWith(".crt")); + return Settings.builder().put("certificate", resource(fileName)); + } + + private Settings.Builder configureWorkingTruststore(String prefix, Settings.Builder settings) { + settings.put(prefix + ".truststore.path", resource("cert1a.p12")); + addSecureSettings(settings, secure -> secure.setString(prefix + ".truststore.secure_password", "cert1a-p12-password")); + return settings; + } + + private Settings.Builder configureWorkingKeystore(String prefix, Settings.Builder settings) { + settings.put(prefix + ".keystore.path", resource("cert1a.p12")); + addSecureSettings(settings, secure -> secure.setString(prefix + ".keystore.secure_password", "cert1a-p12-password")); + return settings; + } + + private ElasticsearchException expectFailure(Settings.Builder settings) { + return expectThrows(ElasticsearchException.class, () -> new SSLService(settings.build(), env)); + } + + private String resource(String fileName) { + final Path path = this.paths.get(fileName); + if (path == null) { + throw new IllegalArgumentException("Test SSL resource " + fileName + " has not been loaded"); + } + return path.toString(); + } + + private void requirePath(String fileName) throws FileNotFoundException { + final Path path = getDataPath("/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/" + fileName); + if (Files.exists(path)) { + paths.put(fileName, path); + } else { + throw new FileNotFoundException("File " + path + " does not exist"); + } + } + + private String randomSslPrefix() { + return randomFrom( + "xpack.security.transport.ssl", + "xpack.security.http.ssl", + "xpack.http.ssl", + "xpack.security.authc.realms.ldap.ldap1.ssl", + "xpack.security.authc.realms.saml.saml1.ssl", + "xpack.monitoring.exporters.http.ssl" + ); + } +} diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/README.txt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/README.txt new file mode 100644 index 00000000000..bc788d59c18 --- /dev/null +++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/README.txt @@ -0,0 +1,106 @@ +# +# NOTE: This readme is also an executable shell script. +# Run it with bash ./README.txt +# + +# +# Make sure we can call certutil +# +[ -n "$ES_HOME" ] || { printf '%s: $ES_HOME is not set\n' "$0" ; exit 1; } +[ -d "$ES_HOME" ] || { printf '%s: $ES_HOME is not a directory\n' "$0" ; exit 1; } + +function certutil() { "$ES_HOME/bin/elasticsearch-certutil" "$@"; } + +# +# Helper functions to generate files & convert file types +# +function new-p12-ca() { + local P12File="$1" + local P12Pass="$2" + local CaDn="$3" + + certutil ca --ca-dn="$CaDn" --days=5000 --out ${PWD}/$P12File --pass="$P12Pass" +} + +function new-p12-cert() { + local CertFile="$1" + local CertPass="$2" + local CertName="$3" + local CaFile="$4" + local CaPass="$5" + shift 5 + + certutil cert --ca="${PWD}/$CaFile" --ca-pass="$CaPass" --days=5000 --out ${PWD}/$CertFile --pass="$CertPass" --name="$CertName" "$@" +} + +function p12-to-jks() { + local P12File="$1" + local P12Pass="$2" + local JksFile="$3" + local JksPass="$4" + + keytool -importkeystore -srckeystore "${PWD}/$P12File" -srcstorepass "$P12Pass" \ + -destkeystore "${PWD}/$JksFile" -deststoretype JKS -deststorepass "$JksPass" +} + +function p12-export-cert() { + local P12File="$1" + local P12Pass="$2" + local P12Name="$3" + local PemFile="$4" + + keytool -exportcert -keystore "${PWD}/$P12File" -storepass "$P12Pass" -alias "$P12Name" \ + -rfc -file "${PWD}/$PemFile" +} + +function p12-export-pair() { + local P12File="$1" + local P12Pass="$2" + local P12Name="$3" + local CrtFile="$4" + local KeyFile="$5" + + local TmpFile="${PWD}/$(basename $P12File .p12).tmp.p12" + + # OpenSSL doesn't have a way to export a single entry + # Keytool doesn't have a way to export keys + # So we use keytool to export the whole entry to a temporary PKCS#12 and then use openssl to export that to PEM + + keytool -importkeystore -srckeystore "${PWD}/$P12File" -srcstorepass "$P12Pass" -srcalias "$P12Name" \ + -destkeystore "$TmpFile" -deststorepass "tmp_password" + + # This produces an unencrypted PKCS#1 key. Use other commands to convert it if needed + # The sed is to skip "BagAttributes" which we don't need + openssl pkcs12 -in "$TmpFile" -nodes -nocerts -passin "pass:tmp_password" | sed -n -e'/^-----/,/^-----/p' > $KeyFile + openssl pkcs12 -in "$TmpFile" -clcerts -nokeys -passin "pass:tmp_password" | sed -n -e'/^-----/,/^-----/p' > $CrtFile + + rm $TmpFile +} + + +# +# Create a CA in PKCS#12 +# +new-p12-ca ca1.p12 "ca1-p12-password" 'CN=Certificate Authority 1,OU=ssl-error-message-test,DC=elastic,DC=co' + +# Make a JKS version of the CA +p12-to-jks ca1.p12 "ca1-p12-password" ca1.jks "ca1-jks-password" + +# Make a PEM version of the CA cert +p12-export-cert ca1.p12 "ca1-p12-password" "ca" ca1.crt + +# +# Create a Cert/Key Pair in PKCS#12 +# - "cert1a" is signed by "ca1" +# - "cert1a.p12" is password protected, and can act as a keystore or truststore +# +new-p12-cert cert1a.p12 "cert1a-p12-password" "cert1a" "ca1.p12" "ca1-p12-password" + +# Convert to JKS +# - "cert1a.jks" is password protected, and can act as a keystore or truststore +p12-to-jks cert1a.p12 "cert1a-p12-password" cert1a.jks "cert1a-jks-password" + +# Convert to PEM +# - "cert1a.key" is an (unprotected) PKCS#1 key +p12-export-pair cert1a.p12 "cert1a-p12-password" "cert1a" cert1a.crt cert1a.key + diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.crt new file mode 100644 index 00000000000..c06bce0d097 --- /dev/null +++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.crt @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDwTCCAqmgAwIBAgIUK67pJtem4zfhIP4SpjoukdLlwnIwDQYJKoZIhvcNAQEL +BQAwcDESMBAGCgmSJomT8ixkARkWAmNvMRcwFQYKCZImiZPyLGQBGRYHZWxhc3Rp +YzEfMB0GA1UECxMWc3NsLWVycm9yLW1lc3NhZ2UtdGVzdDEgMB4GA1UEAxMXQ2Vy +dGlmaWNhdGUgQXV0aG9yaXR5IDEwHhcNMTkwNzE4MDc1NDMyWhcNMzMwMzI2MDc1 +NDMyWjBwMRIwEAYKCZImiZPyLGQBGRYCY28xFzAVBgoJkiaJk/IsZAEZFgdlbGFz +dGljMR8wHQYDVQQLExZzc2wtZXJyb3ItbWVzc2FnZS10ZXN0MSAwHgYDVQQDExdD +ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAK/yaI+3JTakaGHrUBlVhvAP7Jeejkw0XevcRr4H96pO6fEZuq8Kkf73 +/J/wZkzTqE6AGxVYGJV7brzZj8QwXPnkI4fCOLYLeBBX4333lb+X20RGmEjLeTKK +XkOA4FlpbDaecQwbUArBPF3sc2AXW62ZWEosUTR67wr3tFn3uX1RnX5OM0+7qI0Z +gcEm8ohqlig3NC7EmkuP/50CA2OnuuD8b57u0cdgM7uFXTzIASUCz5RU7SSYiI0q +HYOPza+CfUgevReRDc9rzIIVcxMmbab6gABysKGbUdlSTsqGqZMbEprFTv7zt7u1 +lqHeVEjhL7F9769XGaqFeGfa9b1odokCAwEAAaNTMFEwHQYDVR0OBBYEFKHpV7oS +AMlY64s6FbHLPWYtoClTMB8GA1UdIwQYMBaAFKHpV7oSAMlY64s6FbHLPWYtoClT +MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAg9Rq5QIiFpmmIN +yMMeSVCeqZJ1l03WO8YPNcMVGzvjsm12UCHK2ppCahOerGOchnX6tdsjj2aKIL0n +L8++LSDgxbsDA8kxkSldjcY8sJZhTMzvfUkabmWvr99UpVRDKT3EXkMmSaSIaA+U +1xEJAz8dPPFTXyxCNwEePyvy32I3JR6e/8UnUTVNlev8sOdwGvidqHq/PJvR4/qD +B5bn39gsQ6OgUaT6ye6Zp3iUx5uNipGwr2SzoK6ERJ2rwEC1/mQ321rDkSSdnHLT +E+hVL/qrGkg5cl0otPDFxIkOL33/lyyOW7cHdHEUkFsl9Osi6O0HFw1z4DoR7zT3 +Ngckq2E= +-----END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.jks new file mode 100644 index 0000000000000000000000000000000000000000..e772ede11dcf28a1548b18442c1bb6b886f86e36 GIT binary patch literal 2313 zcmc&#X*84#8=hxom@$m8kFk_pF?hzlCdQUBC`Go&jHr>!AcOPbKlzw+Y1l~1iB00H=sV=yD1ee z>{(TSkVKH?i|)?{K2B=7lc<-)wg^VZhnFqzARRe!cTQckPL<3fD zhRgalMcNq$D+|vzocB zdqtL&&j7WFE_FA%T)*o0Nt7hPj=!0IfP}y zaXYnySgx5`rU*7$mSxQhw@PGL&IMw-3`fK)YCh1Kkdl7CtYnL6O120q9gLJzoXMA@ zI*XURkTs%ZCj9($jaYt(x_tmolbu1NB`pT)!+s{nyL<#i*y{yO__>Jp z-7YA}KVDDf&ab?Mix*rb*uPv}O?{3-$d@LiOQ@_`nvfJIt52^TiG3)@&r09_>PJe` zzO}2YFe{lqh+X8cple9Dh|Ig}jX(_+&jE^iSm;g;!);W=m7sf=%VBe8k?HlM(yZ9u zE_JXYn&{udwc>cNWexu+`kKe`kmefbS?B^Mh*(zzAT0U7B8stP&n1Pigj~n=Ud++; zxI#BP!jAHcCg97F4P}L!@R5u7rYy$X6LwbPt&!Px1bv*FSqFHIERkP_@ilBDCxjVEH5UIqC<{{7bW(2bBnYk!4Y2+O<7AiaBk-|Lr*9A`_Kyz!~{SU|@76-z7r6 zO~8RV#jm4WQ+u2BxNp=<)ArAw-%4HegS78gCok!)sm}7Bi)XND@0&l4PyZU<`k^Ip zVxswE31^I993#RmvO=Pvwx1WHhb7)p`!hv(y%B9sbR!05V68XX@b_`Gx1vAD+O{N8 zB{Lgu5|DHWR}bhZkvWAkKBhj&`>lC-Nyi#QsF3kcW%aYCIrPn1PKs|tz*R4!0%U+i?S!1GCHNJ#L>EJJ)Q=bg)oX-L3 zA3m)}Hj#POCbp>Ot-*FBgiSnzmB$vnHbPlvr3;4k2`G@VbIQAvy>Qi6@2MgeTv$C{ zFQgl^&$89`mQtIEBP(FlbjBNU_H+8(5q@`%(}=3u74^v9N0G}#T?aIPKwv!}1NIbT zKyQ`7pa2vqqFOhrG*&&WHzl(zRBfo9@ru<&1JT?(D)Bb)95ev%azVfg1R+p>8^x2U zl$f=GCj&S!s81jP3yS_%4?&@N(}Vqe2zxnngG+nW1 zEiF(>Q};jR|6w8qAop)scAp500bn2?13*C;00609@k?t})UEXMo_D|xlNi~#yze_Wo&b{N|$_(jMkE(Bt>1sb1t8M@-D3#bl>L9rz3CN!Ybe9#U;(_p2Vohlm1lQk{~`w2h>wzk8gA@tlPZ;7hAleb~qcwX+Ft$ z6zN!eg`{QQR-1y0eyX&Ra5i_Jp0;}Tjl*d>#ZZ_}&Ev_fz>>w0=U!TEaURBffFg7_ z+i5`|KOsd$IwozXo)KxYhl$NV51$)kh|-0X0;)HoAhd?^LdSPUNdxh>vLuCW^pLh! zTiaT4%Rf2UOldSmvg@63x8uUjykBGbg(gB_00gKw20H#WUz#5-1{W!xb$%=a>35k= zG8AomVdAS+w*MHoch`DpdAK+z7WKagn12Yu0I&n+0%5!1-?Hy+HJQntHW;j*kg?w*rPsJ2*ZasIJMxgkc}ji#gj1Chalc8oCsFB8Wr80jdrXiA zW-4vGeC!n7To2e|s=6|9T2FCL$xe^5qmEt9{8qzX7bMn;Yr{K?3rD6mV-UH2O}xbu zE6N-zH~SY0YQnOg7p7d#Xs9ROFRP2SD8AEquw|R9H}3i@L!r2cHYz-GTx0W&gpC2s zV_)-9Pj@1}M&wQ&KGm%i5ga6v?xy&CUT$Uqfko3N4F%cS99@LM9dEmp;shPUhsv|^ O75BG5EO{_vq2q6%weev9 literal 0 HcmV?d00001 diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.p12 b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/ssl/SSLServiceErrorMessageTests/ca1.p12 new file mode 100644 index 0000000000000000000000000000000000000000..08250419d2b8ed0fcafc77e302726b3b5b8b6be3 GIT binary patch literal 2647 zcmY+GXHXM}633GedXW+cB-Dgn1EIIj1*91TAqq#85_(6a7?fTD0*V46MMR|cD!qtE zN02HYNRgs|TrhIHH}Boeec0KV-+y;M{$^1Gx@#aH1&Tn&Kt(GaV;J+35l9WpC(x3? z1lm7Q1lnU1fzsh$5hWH(pftO{#usHkMgPAo209=pp8!5X5x_rDVpMeh#}~)pU`R~} z++*32rF&KpX8z#CcE`B+Ngap+&|?86fCq0<_e~Z3F2M|sQkwR!!5Qp54@yJ+1ip2+ zHrHauEx++1-n9R;_{}SeXiHU|mcHFKvoK``npog_H;Ng1t;hjKov#4TrmYv0OJln$ zLAae|u=@Zo&!hRh3`EW56sl7JMa&R4ZKou^vAtL@D|X9|ROgn$@H9`R#m+CN(ySs# zj*nX_sIy7~iN!r~XGb-Dz2!Utw5cE>^ivxB`)AZOI5}j2eou9i7uWa5DPlCc?+DKR z{)-zT+78>77^2v^Ep`@{Kw|yC;q6(URpE!tH+NiJ84^ssD>}QS4g+!fPc0Cb| z5z30D(W^T#;h*`gxGeS*+(4$c=>KBH50e8vVTNP-Cq?g!EMSh|Ry&0-14136W(TsP zU|vWu?66KW@JWbp`)=^Fhk8fvl1Rizy@GYBkkVWC2M0+Z6`F zD;;ZX@uYGUp`hGxK21VV4Q-mF{{8i`+a7{eYd}xDwHC*sY`FqDyS1al@~dzgEJs@q zuz!RWot373w}=efzLWC0wPmxOLP@{wXg%bucX(m&rNC!#Uv7)F!}I->9{-Nz$-NXn zN`L9t7~ML58jvS1dU)Q^>j&!?z1Eyu>LAf~-{wYbZj?s4G<~XY3bFqEhSRx5D&vT9 zcRT-s#@R{F2BXE39v8X_uFY@@vzSKe);xEhlP#4<*USZTt<}ohLRIO!%{Fz^-T^r4#COWH+)bAk zWVNT<$vdp@@ZMZ7$Z4_d{{y{8@EH;{4-+40dqf97`$u#=NbFJPyi`>X$ny)uQ??%Q zK&^#_$wo2o*c+d+-Eq6DVfU_L#YYo*^{&gaIpT9REcH%w)#`+u*i(;1BB%Y>j&|Vh z*iDU&I{JE91@l1jN_zpDpAWOWGmA%rPo87Wj9v#V@YrL+PSCjSlcpwRT?K~(^;A`tU5_dXH~d?Kh*`mQipb>& z(Hx!J06q6L){N<)F`m0i*J{Dz9b8o*5#)@6`LIAtpHb@CcNk?B3`fc)eRp|C%+509Fpmd|1cG z03)er@;On^HR(b^_%&dJl8WhS8P0a$7kbo?;RW#+$sZf>wWG+e`Q|L#6f7O`}dd7nP=6rHbVoN@HdhXOlo7c<~mzumd>N#J?nG56`c8V}b@ zxWCT;h_|1v>v2=(@$PPPJT{BSNl%~AzGn=N!AA~xC<-O8)m77K3$79d*}d{dOu5Lf z01d$j)V#mc{jJK;5l!#AQ}j6R!PDMkPP!hdC%}b^9bvQ4l?l$Ik&#UOBg$?%mS(lL z_Sg`zI!;1quiE;L{)#U0jw9{|JyZW><$RJAa<(s= z8d{)d?adm$@<_jvAb1mhJjKPuV9tROG_mV43q34m+3xQL6>sLHgxt7NIoGK&-sd~9 zqML`w9un(xTaVu9m269wJs>@hESCA(hr;fv4(zSY*$pWEaFrxdPGwyyHK&& zWXjU6Q08(5wY~2IFNnE~cay}vmt4APfz6WNZdY^5yWEAVS3rGBy(2c$gd1FOW?uHO zRpn7bqdC0=!x9Kow1QD0m~`(C~dcUOa%`*Ns} z%JQG%RMoV+1~@vFqv{4z4m$IZsH@EOiGHMpz1wMe%@7IDtvgHk=}@C#)y2V}_&sKm z35w6c1E4v;6JA7KGA|Oyaz>O0vIz!NP1_3l{N7v^%{;?Ay@e+o|B^13d=-93iwt3` z#pDK=jhS5C4c~Jv!QbGxQhF4`Z9_STb78)yWJ}F?9N@l0qlH9)<@R6k`%CDC^0lCFB5d(HjF$gerFEldA zz`dQLc_Lf;Q$+)OWi9<4-{CB8ma5zJdo%>0lj7_|jIv~t?X8MdCxo_YS5-!l9}JfC zfv<92QH`3YfmfJ_8<)-|?+Ndn6%8`Hk#OIKLbX)K)vpz!-K!TO6J-3c_scSR57k!q zw10e6`{*m`^hCVAs{T}pwd?5@s?SQJijsFtr+4p&?zbYivUn6#beR~UtwT6`pF_vF zt#-1FvuxAZXGvVxxPnR;#&2q7nCZ)4jBL_|HeT)+9&r*btIy2U9JOb9iNEr(PyW%{ z*B=^xTF7uTxs{$5aj9MI$T3c2`v~kg<9Z2DP30CjL204{P_$H(5=;~z7$xB1f19($ vz{$pTV%SoA+>a}3>oy-gfguR!!+HGCCu2*&<)voWv^^tE%r>DDZ`m{Ver;9o~wN*M%J3o}_3>7}!l*v~A z`q-48?R#cfVa790aefH}&K(qadrh=6o$u<#q*VWW> z&8f|=8W%%Su6ryk-^d=M~abjLpwYvd3FG zr=Qein%Y_D3{zG-mFc|%Ik89H?u&hAw)lfo+S&?F`^k<66kv)M9@E1aK^yD^cfta! zA#}jk@=c%4YFr`*9!Wl>_L7$!K4Nc9u3G8=H=d+;0ghzp*G?`1J|uifp3J0vmQ9YH z3yr7Zr00?51c!u{RK^~(|NohNhyqqcbkrF#W5AfNoVlgi!= zDx6IFiP4P{;Kjd>3(Cd8AIP|Cno~EU^bi@^sWnRn`G(FZ!SudV?o;Xmte?_iH68C6 zvx{I(mY2LiSWJ9~GPO<_73G7OXv?k4(MUpWfUEIh&1+2+&%rfK2}@Igj-E|R=5L%I zVa?c6%Lp|G_#%8s#ldP&&=x0Dh{&{J+dr`MTZql%QlQFM+!m-$M5o;E@}EM|72amUkDfS&Wr(CjXbN?;A?sj|cJJzjK$5V1f$jyui{kM0=aMk^LK=UlkksbuSFl_mpibLiCQe#iQ5Q927tkRBgv-49c^PeirZ~;}rV1HN~!ARqJeU0N5l)mE?wzYg6*=@Q=;c&x0_f zO&p?p`iVp8w+%-MF1m>hKBn7nEP7ez%&(MMzN((j+@PB^DH>XcN2clB7me}b*N=j$ zW~}D|-@1-(CFu+S_s+`nk)UO1EOQsdXsdJ!AZAsr=Y$!-UvV({iP|gH`=4v19lXEZ zI{M1jkZaB~_q<0{#>7PT1LOXam(~M2)p*gH(|K<8NAnOzyf+3uAQVti;m z4lbq3*vl+FAdm+NKnG#=pm%d{b6t~5xW0n)1d57)JOVLrwD6zCIPrcsQn0TFMhY## z0d|1!z(hzSKNUQY7)Vs{$CF4nZ@fw{o)nBhpe28q0EWR$cM7(zm#+sd7>_Us3HAvj z`UamzV9=6qK8!9}8>55P*3?kfbcXY(tE1J`w0`?U3t|Lzg8Z?@Z$%$X2JZV~+>1 zOa=-7cE-6uWFQdGp8v?wj7ON)lhv!_9e=kuQMX8RAX@L`%&NeG>V!R^v z88-6!Cj|2J(<%At!{Id_s6g!VF^TtEYpDm4FYp;YZPZQcPA%tx8g&$e<#bqy%YS*I zZDaK{=#6t2p-TUrghO55x46EHASu1%a#Z{ATxH+t{L>dI*1x_>pJyZt#=#|vOJgM6 zrbP_Q%Pk*Kino$mVczrp7~A^w0Gjk}hI9>&MBI(w$gfOFhg**-6P4ergHy@JIbJbw zmiTtp8~XJI2n+-OZ&{%&e*7H?euxM}h^s|)x^u-{P(gU>Td!e@!{|{_b+pt@UL|B8 zd(k2>LPZS6`aOUyr}-p3;p&%$UMf^YJ2clX6JQV!8t})2?`)KvgaZM`2H#y9BYnUt7MgJ*p$duUqu;F@U z|3n-w@d_(Y^~Ca}X-#AhnsjEpTkrdvR2R6}t&EYSv2jU;lgNrx_R0oqF~6U4iY~EF z@8I{U>|YDFtf^DgK`_STLj+O#D#4^!}+9;$JfOk5g0oGc`5me;}g2lT2IjNv)IBD)zz6+lV?` zUK046Re04>!*!m1sELzRZp~O0t*3FPZ?m>;EO}WD+_8=k7k1j4aUtNzNNOAU+#XltmpSXDfSEdv*S~gBrc}*(8!Tk$wqiOU+oI=U=)R10Yrv<=KuLvwm^8 z?7H}#+;;2jFLe#InMLpIv6HCkNG8ouv^*}%dvvYQ=Ul?ClI_s8e~|oNq5iF8fgh5A zKsE%52JgK7vKxPR-eq{`p3OdK-`vxDovo6GZ3@e-g=AU1)$b6{Y!w#Q|9Id3oQ?F0 z(cA+BSYf3{{^gL(+A*0_uS7(nJgT=z1u@ay00wtq(iL4(It;2ZahBc8$ird*cp7co zzQo>C(Xh?URPJzbypI57NRSI`Bw?^@=Z-v}4U{xeUKu~FEh|~bZkM;!w91&@s+tKB zTQ9g1{?s7v^~cSNoS8G@!${LxRNLasuEm_&VU*{2DT(P-G|w99y-NoRDxMkFZhLBv zIk%?oFUThjz-Ap#n-yYMU81YPgQfPi1b$Q`I}3T$rH(WBtWcVZ?CSi!*#%BGA8A5Q zkg37a;@n@c(@pGj7$A?|yNZMV`u^qI{a+6Ee|`V!`~Syx4uFO&B)QxJ$jUMch?B%kZ^z?A=3QFpCAnq4pCZ4q*0`# zQ?B=(8|VJjZ=GheJpz0$_P5d6A1q?7=X+RT z##t|Kqjiz+NZhM(AxcbS@Yx+l)JG5523zh2I!+FKu3v%7Gs}Y1H!b=RA?Op`{dIVS58X$jS@AeMW>%3xP5a7d0c`^A zI)X0A5pEO>!EfX@?Iq80U9HxE7`M7d*TPv{ml%rNGUz{7A=O9ug$#EhvR2 zHw~~=K^Vgl3rA0mRd>-}=B04ODv*1ZmhZ+p{U>5&1ghtSdd|m7Zc*Dc_uuxs?OH-T zsHd5}b+C*El_sb>jP8Q#<`1)SiVVKn-fo!rtJ%Xz98i}iD0;4qi%6t zuTnlNcO6uQQkw2-0n|RzFgwt2#Pl{|?|1w>n&am}LkTWW`R=vMJNPk<2K{9{P6=<6 zmg4l0x?y$41?nwztcvuvmXBvEQkRIqfX=nf;No)xHWR;nN))Y`n1Lhu_U&8Tfmk}K>WF;8?^CH$@A!Nh3N2~L4jCX$vc_%;4xhS@7+IJeT*p6ElwJ`RZ}cW1$~6>s%vm`vU#scF z1&&IGjMJX+(0oQs^Ta<5^HDEdu#XVNsC9iF<}|9g05qo2dBJa)I73P)Uh6R+en31a z!g=oPs2gMY*2B$L$|AV}V6T{iWCtJS{Y~=%MQ}P(Z{p)0* zfV7E`1E7H&-HeV&;Lfu3UQj6Skw{g2brF$!Xsns)lW65VdRykxi?pLl)4PByvfQuI zLqKePj~OAWWh!fx&fki`Si#uW3#u|x{ujl^1N7^DM2eEt8uos~FBQnr*j8?9DeC4M zY+xZKuUO`e9+fyQMMSzawr9?w?g_}qX`nBwT+>5-bkkY@ce9FJ^>`_Z$)}ika^4;Z zUpD~_RzMdCL2CJCN3D6W5twbz>U7-CY*i+)F>JJa{YvWZb=47#)=f*Xsz5fzePdk} z#Rm3m+q%TTgCE33S+8%CbS_ewK`j~+aeLMn(Qh-ZF3YAk3O%VJmm@AJV+e3P%f!B~ zau=PZ)LOajaQ)CPAtqWQt90=x8Q;eqkgTxWo5^I^?9M3;D_JS_<54_p-h?UdLPZ_* zBa*~`7Q&C*r~<2^jxwY)gO9Q3LK4TOCz<-(#f@9Pxe6#(mfvMQvRYi-4hnN^*|m;n z&C+J%r+5LUC{7c=n6pei@ks^tZCnXE6!*jOleT7+rM@codc|ss=4_}RUk!e+!<`kS z*1T3#2Us;Oc?YR{n_G+){MbB?EG=_;XQz)jPGY$KA+gGa$dbc>}Nu#2PZaDP(zyE0iB5g`PO{=cduq!J=y zA_3R~900xme}E9c7RLI&kXncem(s|^!+}LeOk6}%L|jxtOh^a@LlbEIw}cNaK@-US zL2wWd@Mj7CF9H6aVZr}2EahY)!FVJP8q9H;S)k#PcOxWW`Y*#qqwz1(9Kw}MEI;kN zszWHM;UC?oO(X@&Clu?@aR|axF$-)UwH^S>(#8vB3U(tboT*$q%*T;zQ_ZI zbq-1CRD+G2LdAN!(5K#wT2-3uvL(j&?5j_#3(#pSk;HMA*QM71wk;5&yL>Sk2)lYARUj%3nA*b-IGOd5z)>v9a4Xi)5+@15 zc+;%ONS~Eaou*yi$`CP=Z1AJJku&144WD^Z*!}dIqQwAJB^x5dYDV$r!bKs64YuM2 z-YTDpXOba+_vVH@{YCXB!^8bEK%uvDEOmizSQ*yzPdaKC;}K7(ehaTN&nZpvO*;Q> z*MB}@4#HoR;UiDMdK%2VH+vR%anvNL_F2Q{^SN-RK5|UK)iXJt>YFpB(rDp}SeXvN zSquk_I`nefM2~`{HkcRrxhV(y6lN}G`1WtjX12{((p#W2SB3c2iCfgcs7S>}qQpwF z07QT`EOGmq)rZkmRz%%Q#nlu5_r`L*zvQi^k4Uo6GS*G-VLg-cOhJC7%mH&Yj=;%Ovlm)AZQCKci7v4N1)OU3OqB)9kG%eMD<)Zky*~7*qBp- zXQnB5#de6jt4EtpApw4JCNG!#ZIGrLjRv=2PBJr#O#Y+FKhp#YzfV9ww*vY5dH6=te9-xbTF7 zyP8(Fl{kzCSBWuA+$^5O;^XH>DzMZ8dL=1CmM7JBT;$ACwA z_+YHUrBMN_=L@hA$1;2N)`j_GE~bl-uDa6<7%^cS#PTD(7zn3}IK>Af4Ql$7iKN#Y zsMF54xHu3&QZj>MU1{~nqMJRG8Mx#e=5IuecB+xB=SSFCODG*&=aJlb-VpoL88fx! zhw%2!^T%NGzNp);3kqD@zaCs&LorKWDiBsw=)9aZ_`~q4y1rBnl}Ae>vcED9+$N6t z69c__+A0TCcD#tczcarEB#wp@cWM+nf~PFYUU7F7^ERoPp7RSPcyhD!9LO&AsuSSR zGCU~Ly1}n_OMrPuO?If4Ayrwz6HiNz|`g*f=Tz-&{`I^K@rdui>{L@?Lj(+SJ!Kg%c zgyr|~-bYF(JO%3aW?T#Xv0Dn^-^FHA1YVEBZ$48yIe#(uHN`-zIP{Si-De#wsJg2- zb{pAUdyvg^Zr_!UNN6qkRIOG)p9Ve%3sIPX@y{Kh-UqOG~nnE1&fyMdRH7-pF>MbSEvnL8}lxm6KKi4 z(5_UsY4$eQOm>vBNp@AsBJ-8@PfA2sl}6jz>)Mq$){C;jhz(8-x9v~;>a+(N8XGwW z3A^-ri8XJ@`Dqg=S#2Ax!sXlWC^c$KEL(T|J?6unOh91USZEb|ZN@pyVPw1MP&b>h5aH{P2(R@pE;6g$ zZd-nc4qGF}tJ=J$jna_7>OnO3k5J=Hi{+8LIhTfynr;KlEoX%b8~qyT3$WSz%4D5v zQWF3Gk;(0z{_`O}$In*G_|=Oh+Q7KA3&-ZX!p#f)hdsz(4Y$M7T;W>{-XTr`FxXl* zZAG(kuBy$c!yW>yRrHB70*00<-q&(@V%mtm(2-fk&Akbyq&dZxoc2N}YD%ma%)QOu z#sRugdW>N(Qz`=z!0T7X9(qH$Kg#XHGNml;gQ5#fhqgzO3oUQ!CL&_xkuhvf