Ensure we never read from a closed MockSecureSettings object (#25322)

If secure settings are closed after the node has been constructed
no key-store access is permitted. We should also try to be as close as possible
to the real behavior if we mock secure settings. This change also adds
the same behavior as bootstrap has to InternalTestCluster to ensure we fail
if we try to read from secure settings after the node has been constructed.

core/src/main/java/org/elasticsearch/common/settings/SecureSettings.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.common.settings;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.io.InputStream;
 import java.security.GeneralSecurityException;
 import java.util.Set;
@@ -40,4 +41,7 @@ public interface SecureSettings extends Closeable {
 
     /** Return a file setting. The {@link InputStream} should be closed once it is used. */
     InputStream getFile(String setting) throws GeneralSecurityException;
+
+    @Override
+    void close() throws IOException;
 }

test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java

@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * A mock implementation of secure settings for tests to use.
@@ -35,6 +36,7 @@ public class MockSecureSettings implements SecureSettings {
     private Map<String, SecureString> secureStrings = new HashMap<>();
     private Map<String, byte[]> files = new HashMap<>();
     private Set<String> settingNames = new HashSet<>();
+    private final AtomicBoolean closed = new AtomicBoolean(false);
 
     @Override
     public boolean isLoaded() {
@@ -48,24 +50,36 @@ public class MockSecureSettings implements SecureSettings {
 
     @Override
     public SecureString getString(String setting) {
+        ensureOpen();
         return secureStrings.get(setting);
     }
 
     @Override
     public InputStream getFile(String setting) {
+        ensureOpen();
         return new ByteArrayInputStream(files.get(setting));
     }
 
     public void setString(String setting, String value) {
+        ensureOpen();
         secureStrings.put(setting, new SecureString(value.toCharArray()));
         settingNames.add(setting);
     }
 
     public void setFile(String setting, byte[] value) {
+        ensureOpen();
         files.put(setting, value);
         settingNames.add(setting);
     }
 
     @Override
-    public void close() throws IOException {}
+    public void close() throws IOException {
+        closed.set(true);
+    }
+
+    private void ensureOpen() {
+        if (closed.get()) {
+            throw new IllegalStateException("secure settings are already closed");
+        }
+    }
 }

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -53,6 +53,7 @@ import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -102,6 +103,7 @@ import org.junit.Assert;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.net.InetSocketAddress;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -605,7 +607,13 @@ public final class InternalTestCluster extends TestCluster {
         } else if (!usingSingleNodeDiscovery && finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) {
             throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured");
         }
+        SecureSettings secureSettings = finalSettings.getSecureSettings();
         MockNode node = new MockNode(finalSettings.build(), plugins);
+        try {
+            IOUtils.close(secureSettings);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
         return new NodeAndClient(name, node, nodeId);
     }