Merge pull request #5042 from eclipse/jetty-9.4.x-5019-SslReload

Issue #5019 - hot-reload SSL certificates if keystore file changed

jetty-documentation/src/main/asciidoc/configuring/connectors/configuring-ssl.adoc

@@ -989,3 +989,15 @@ As a reminder, when configuring your includes/excludes, *excludes always win*.
 
 Dumps can be configured as part of the `jetty.xml` configuration for your server.
 Please see the documentation on the link:#jetty-dump-tool[Jetty Dump Tool] for more information.
+
+
+==== SslContextFactory KeyStore Reload
+
+Jetty can be configured to monitor the directory of the KeyStore file specified in the SslContextFactory, and reload the
+SslContextFactory if any changes are detected to the KeyStore file.
+
+If changes need to be done to other file such as the TrustStore file, this must be done before the change to the Keystore
+file which will then trigger the `SslContextFactory` reload.
+
+With the Jetty distribution this feature can be used by simply activating the `ssl-reload` startup module.
+For embedded usage the `KeyStoreScanner` should be created given the `SslContextFactory` and added as a bean on the Server.

jetty-server/src/main/config/etc/jetty-ssl-context-reload.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0"?><!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_9_3.dtd">
+
+<Configure id="Server" class="org.eclipse.jetty.server.Server">
+  <Call name="addBean">
+    <Arg>
+      <New id="keyStoreScanner" class="org.eclipse.jetty.util.ssl.KeyStoreScanner">
+        <Arg><Ref refid="sslContextFactory"/></Arg>
+        <Set name="scanInterval"><Property name="jetty.sslContext.reload.scanInterval" default="1"/></Set>
+      </New>
+    </Arg>
+  </Call>
+</Configure>

jetty-server/src/main/config/modules/ssl-reload.mod

@@ -0,0 +1,18 @@
+# DO NOT EDIT - See: https://www.eclipse.org/jetty/documentation/current/startup-modules.html
+
+[description]
+Enables the SSL keystore to be reloaded after any changes are detected on the file system.
+
+[tags]
+connector
+ssl
+
+[depend]
+ssl
+
+[xml]
+etc/jetty-ssl-context-reload.xml
+
+[ini-template]
+# Monitored directory scan period (seconds)
+# jetty.sslContext.reload.scanInterval=1

jetty-util/src/main/java/org/eclipse/jetty/util/ssl/KeyStoreScanner.java

@@ -0,0 +1,133 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.util.ssl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.function.Consumer;
+
+import org.eclipse.jetty.util.Scanner;
+import org.eclipse.jetty.util.annotation.ManagedAttribute;
+import org.eclipse.jetty.util.annotation.ManagedOperation;
+import org.eclipse.jetty.util.component.ContainerLifeCycle;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+
+/**
+ * <p>The {@link KeyStoreScanner} is used to monitor the KeyStore file used by the {@link SslContextFactory}.
+ * It will reload the {@link SslContextFactory} if it detects that the KeyStore file has been modified.</p>
+ * <p>If the TrustStore file needs to be changed, then this should be done before touching the KeyStore file,
+ * the {@link SslContextFactory#reload(Consumer)} will only occur after the KeyStore file has been modified.</p>
+ */
+public class KeyStoreScanner extends ContainerLifeCycle implements Scanner.DiscreteListener
+{
+    private static final Logger LOG = Log.getLogger(KeyStoreScanner.class);
+
+    private final SslContextFactory sslContextFactory;
+    private final File keystoreFile;
+    private final Scanner _scanner;
+
+    public KeyStoreScanner(SslContextFactory sslContextFactory)
+    {
+        this.sslContextFactory = sslContextFactory;
+        try
+        {
+            keystoreFile = sslContextFactory.getKeyStoreResource().getFile();
+            if (keystoreFile == null || !keystoreFile.exists())
+                throw new IllegalArgumentException("keystore file does not exist");
+            if (keystoreFile.isDirectory())
+                throw new IllegalArgumentException("expected keystore file not directory");
+        }
+        catch (IOException e)
+        {
+            throw new IllegalArgumentException("could not obtain keystore file", e);
+        }
+
+        File parentFile = keystoreFile.getParentFile();
+        if (!parentFile.exists() || !parentFile.isDirectory())
+            throw new IllegalArgumentException("error obtaining keystore dir");
+
+        _scanner = new Scanner();
+        _scanner.setScanDirs(Collections.singletonList(parentFile));
+        _scanner.setScanInterval(1);
+        _scanner.setReportDirs(false);
+        _scanner.setReportExistingFilesOnStartup(false);
+        _scanner.setScanDepth(1);
+        _scanner.addListener(this);
+        addBean(_scanner);
+    }
+
+    @Override
+    public void fileAdded(String filename)
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("added {}", filename);
+
+        if (keystoreFile.toPath().toString().equals(filename))
+            reload();
+    }
+
+    @Override
+    public void fileChanged(String filename)
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("changed {}", filename);
+
+        if (keystoreFile.toPath().toString().equals(filename))
+            reload();
+    }
+
+    @Override
+    public void fileRemoved(String filename)
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("removed {}", filename);
+
+        if (keystoreFile.toPath().toString().equals(filename))
+            reload();
+    }
+
+    @ManagedOperation(value = "Reload the SSL Keystore", impact = "ACTION")
+    public void reload()
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("reloading keystore file {}", keystoreFile);
+
+        try
+        {
+            sslContextFactory.reload(scf -> {});
+        }
+        catch (Throwable t)
+        {
+            LOG.warn("Keystore Reload Failed", t);
+        }
+    }
+
+    @ManagedAttribute("scanning interval to detect changes which need reloaded")
+    public int getScanInterval()
+    {
+        return _scanner.getScanInterval();
+    }
+
+    public void setScanInterval(int scanInterval)
+    {
+        _scanner.setScanInterval(scanInterval);
+    }
+}

jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java

@@ -1131,6 +1131,9 @@ public class SslContextFactory extends AbstractLifeCycle implements Dumpable
 
         synchronized (this)
         {
+            if (_factory == null)
+                throw new IllegalStateException("SslContextFactory reload failed");
+
             return _factory._context;
         }
     }
@@ -1532,6 +1535,9 @@ public class SslContextFactory extends AbstractLifeCycle implements Dumpable
 
         synchronized (this)
         {
+            if (_factory == null)
+                throw new IllegalStateException("SslContextFactory reload failed");
+
             return _factory._keyStore;
         }
     }
@@ -1553,6 +1559,9 @@ public class SslContextFactory extends AbstractLifeCycle implements Dumpable
 
         synchronized (this)
         {
+            if (_factory == null)
+                throw new IllegalStateException("SslContextFactory reload failed");
+
             return _factory._trustStore;
         }
     }

tests/test-integration/src/test/java/org/eclipse/jetty/test/KeyStoreScannerTest.java

@@ -0,0 +1,290 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.test;
+
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.util.Calendar;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
+import org.eclipse.jetty.toolchain.test.jupiter.WorkDir;
+import org.eclipse.jetty.toolchain.test.jupiter.WorkDirExtension;
+import org.eclipse.jetty.util.log.StacklessLogging;
+import org.eclipse.jetty.util.ssl.KeyStoreScanner;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+@ExtendWith(WorkDirExtension.class)
+public class KeyStoreScannerTest
+{
+    private static final int scanInterval = 1;
+    public WorkDir testdir;
+    private Server server;
+    private Path keystoreDir;
+
+    @BeforeEach
+    public void before()
+    {
+        keystoreDir = testdir.getEmptyPathDir();
+    }
+
+    @FunctionalInterface
+    public interface Configuration
+    {
+        void configure(SslContextFactory sslContextFactory) throws Exception;
+    }
+
+    public void start() throws Exception
+    {
+        start(sslContextFactory ->
+        {
+            String keystorePath = useKeystore("oldKeystore").toString();
+            sslContextFactory.setKeyStorePath(keystorePath);
+            sslContextFactory.setKeyStorePassword("storepwd");
+            sslContextFactory.setKeyManagerPassword("keypwd");
+        });
+    }
+
+    public void start(Configuration configuration) throws Exception
+    {
+        SslContextFactory sslContextFactory = new SslContextFactory.Server();
+        configuration.configure(sslContextFactory);
+
+        server = new Server();
+        SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
+        HttpConfiguration httpsConfig = new HttpConfiguration();
+        httpsConfig.addCustomizer(new SecureRequestCustomizer());
+        HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpsConfig);
+        ServerConnector connector = new ServerConnector(server, sslConnectionFactory, httpConnectionFactory);
+        server.addConnector(connector);
+
+        // Configure Keystore Reload.
+        KeyStoreScanner keystoreScanner = new KeyStoreScanner(sslContextFactory);
+        keystoreScanner.setScanInterval(scanInterval);
+        server.addBean(keystoreScanner);
+
+        server.start();
+    }
+
+    @AfterEach
+    public void stop() throws Exception
+    {
+        server.stop();
+    }
+
+    @Test
+    public void testKeystoreHotReload() throws Exception
+    {
+        start();
+
+        // Check the original certificate expiry.
+        X509Certificate cert1 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert1), is(2015));
+
+        // Switch to use newKeystore which has a later expiry date.
+        useKeystore("newKeystore");
+        Thread.sleep(Duration.ofSeconds(scanInterval * 2).toMillis());
+
+        // The scanner should have detected the updated keystore, expiry should be renewed.
+        X509Certificate cert2 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert2), is(2020));
+    }
+
+    @Test
+    public void testReloadWithBadKeystore() throws Exception
+    {
+        start();
+
+        // Check the original certificate expiry.
+        X509Certificate cert1 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert1), is(2015));
+
+        // Switch to use badKeystore which has the incorrect passwords.
+        try (StacklessLogging ignored = new StacklessLogging(KeyStoreScanner.class))
+        {
+            useKeystore("badKeystore");
+            Thread.sleep(Duration.ofSeconds(scanInterval * 2).toMillis());
+        }
+
+        // The good keystore is removed, now the bad keystore now causes an exception.
+        assertThrows(Throwable.class, () -> getCertificateFromServer());
+    }
+
+    @Test
+    public void testKeystoreRemoval() throws Exception
+    {
+        start();
+
+        // Check the original certificate expiry.
+        X509Certificate cert1 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert1), is(2015));
+
+        // Delete the keystore.
+        try (StacklessLogging ignored = new StacklessLogging(KeyStoreScanner.class))
+        {
+            useKeystore(null);
+            Thread.sleep(Duration.ofSeconds(scanInterval * 2).toMillis());
+        }
+
+        // The good keystore is removed, having no keystore causes an exception.
+        assertThrows(Throwable.class, () -> getCertificateFromServer());
+
+        // Switch to use keystore2 which has a later expiry date.
+        useKeystore("newKeystore");
+        Thread.sleep(Duration.ofSeconds(scanInterval * 2).toMillis());
+        X509Certificate cert2 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert2), is(2020));
+    }
+
+    @Test
+    public void testReloadChangingSymbolicLink() throws Exception
+    {
+        Path keystorePath = keystoreDir.resolve("symlinkKeystore");
+        start(sslContextFactory ->
+        {
+            Files.createSymbolicLink(keystorePath, useKeystore("oldKeystore"));
+            sslContextFactory.setKeyStorePath(keystorePath.toString());
+            sslContextFactory.setKeyStorePassword("storepwd");
+            sslContextFactory.setKeyManagerPassword("keypwd");
+        });
+
+        // Check the original certificate expiry.
+        X509Certificate cert1 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert1), is(2015));
+
+        // Change the symlink to point to the newKeystore file location which has a later expiry date.
+        Files.delete(keystorePath);
+        Files.createSymbolicLink(keystorePath, useKeystore("newKeystore"));
+        Thread.sleep(Duration.ofSeconds(scanInterval * 2).toMillis());
+
+        // The scanner should have detected the updated keystore, expiry should be renewed.
+        X509Certificate cert2 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert2), is(2020));
+    }
+
+    @Test
+    public void testReloadChangingTargetOfSymbolicLink() throws Exception
+    {
+        start(sslContextFactory ->
+        {
+            Path keystorePath = keystoreDir.resolve("symlinkKeystore");
+            Files.createSymbolicLink(keystorePath, useKeystore("oldKeystore"));
+            sslContextFactory.setKeyStorePath(keystorePath.toString());
+            sslContextFactory.setKeyStorePassword("storepwd");
+            sslContextFactory.setKeyManagerPassword("keypwd");
+        });
+
+        // Check the original certificate expiry.
+        X509Certificate cert1 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert1), is(2015));
+
+        // Change the target file of the symlink to the newKeystore which has a later expiry date.
+        useKeystore("newKeystore");
+        Thread.sleep(Duration.ofSeconds(scanInterval * 2).toMillis());
+
+        // The scanner should have detected the updated keystore, expiry should be renewed.
+        X509Certificate cert2 = getCertificateFromServer();
+        assertThat(getExpiryYear(cert2), is(2020));
+    }
+
+    public Path useKeystore(String keystore) throws Exception
+    {
+        Path keystorePath = keystoreDir.resolve("keystore");
+        if (Files.exists(keystorePath))
+            Files.delete(keystorePath);
+
+        if (keystore == null)
+            return null;
+
+        Files.copy(MavenTestingUtils.getTestResourceFile(keystore).toPath(), keystorePath);
+        keystorePath.toFile().deleteOnExit();
+
+        if (!Files.exists(keystorePath))
+            throw new IllegalStateException("keystore file was not created");
+
+        return keystorePath.toAbsolutePath();
+    }
+
+    public static int getExpiryYear(X509Certificate cert)
+    {
+        Calendar instance = Calendar.getInstance();
+        instance.setTime(cert.getNotAfter());
+        return instance.get(Calendar.YEAR);
+    }
+
+    public X509Certificate getCertificateFromServer() throws Exception
+    {
+        URL serverUrl = server.getURI().toURL();
+        SSLContext ctx = SSLContext.getInstance("TLS");
+        ctx.init(new KeyManager[0], new TrustManager[] {new DefaultTrustManager()}, new SecureRandom());
+        SSLContext.setDefault(ctx);
+
+        HttpsURLConnection connection = (HttpsURLConnection)serverUrl.openConnection();
+        connection.setHostnameVerifier((a, b) -> true);
+        connection.connect();
+        Certificate[] certs = connection.getServerCertificates();
+        connection.disconnect();
+
+        assertThat(certs.length, is(1));
+        return (X509Certificate)certs[0];
+    }
+
+    private static class DefaultTrustManager implements X509TrustManager
+    {
+        @Override
+        public void checkClientTrusted(X509Certificate[] arg0, String arg1)
+        {
+        }
+
+        @Override
+        public void checkServerTrusted(X509Certificate[] arg0, String arg1)
+        {
+        }
+
+        @Override
+        public X509Certificate[] getAcceptedIssuers()
+        {
+            return null;
+        }
+    }
+}

tests/test-integration/src/test/resources/jetty-logging.properties

@@ -2,3 +2,4 @@ org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
 #org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.Slf4jLog
 #org.eclipse.jetty.LEVEL=DEBUG
 #org.eclipse.jetty.websocket.LEVEL=DEBUG
+#org.eclipse.jetty.util.ssl.KeyStoreScanner.LEVEL=DEBUG