[7.x] Secure password for monitoring HTTP exporter (#51775)

Adds a secure and reloadable SECURE_AUTH_PASSWORD setting to allow keystore entries in the form "xpack.monitoring.exporters.*.auth.secure_password" to securely supply passwords for monitoring HTTP exporters. Also deprecates the insecure `AUTH_PASSWORD` setting.

docs/reference/settings/monitoring-settings.asciidoc

@@ -29,11 +29,11 @@ For more information, see <<monitor-elasticsearch-cluster>>.
 ==== General Monitoring Settings
 
 `xpack.monitoring.enabled`::
-Set to `true` (default) to enable {es} {monitoring} for {es} on the node. 
+Set to `true` (default) to enable {es} {monitoring} for {es} on the node.
 +
 --
-NOTE: To enable data collection, you must also set `xpack.monitoring.collection.enabled` 
+NOTE: To enable data collection, you must also set `xpack.monitoring.collection.enabled`
-to `true`. Its default value is `false`. 
+to `true`. Its default value is `false`.
 --
 
 [float]
@@ -51,7 +51,7 @@ this setting is `false` (default), {es} monitoring data is not collected and
 all monitoring data from other sources such as {kib}, Beats, and Logstash is
 ignored.
 
-`xpack.monitoring.collection.interval` (<<cluster-update-settings,Dynamic>>):: 
+`xpack.monitoring.collection.interval` (<<cluster-update-settings,Dynamic>>)::
 
 Setting to `-1` to disable data collection is no longer supported beginning with
 7.0.0. deprecated[6.3.0, Use `xpack.monitoring.collection.enabled` set to `false` instead.]
@@ -198,11 +198,17 @@ xpack.monitoring.exporters:
 
 `auth.username`::
 
-The username is required if a `auth.password` is supplied.
+The username is required if `auth.secure_password` or `auth.password` is supplied.
+
+`auth.secure_password` (<<secure-settings,Secure>>, <<reloadable-secure-settings,reloadable>>)::
+
+The password for the `auth.username`. Takes precedence over `auth.password` if it is also specified.
 
 `auth.password`::
 
-The password for the `auth.username`.
+The password for the `auth.username`. If `auth.secure_password` is also specified, this setting is ignored.
+
+deprecated[7.7.0, Use `auth.secure_password` instead.]
 
 `connection.timeout`::
 

docs/reference/setup/secure-settings.asciidoc

@@ -59,3 +59,4 @@ There are reloadable secure settings for:
 * {plugins}/discovery-ec2-usage.html#_configuring_ec2_discovery[The EC2 discovery plugin]
 * {plugins}/repository-gcs-client.html[The GCS repository plugin]
 * {plugins}/repository-s3-client.html[The S3 repository plugin]
+* <<monitoring-settings>>

x-pack/plugin/core/src/test/java/org/elasticsearch/test/http/MockWebServer.java

@@ -244,6 +244,13 @@ public class MockWebServer implements Closeable {
         return requests.poll();
     }
 
+    /**
+     * Removes all requests from the queue.
+     */
+    public void clearRequests() {
+        requests.clear();
+    }
+
     /**
      * A utility method to peek into the requests and find out if #MockWebServer.takeRequests will not throw an out of bound exception
      * @return true if more requests are available, false otherwise

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java

@@ -26,6 +26,7 @@ import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptService;
@@ -73,7 +74,7 @@ import static org.elasticsearch.common.settings.Setting.boolSetting;
  * - node clients: all modules are bound
  * - transport clients: only action/transport actions are bound
  */
-public class Monitoring extends Plugin implements ActionPlugin {
+public class Monitoring extends Plugin implements ActionPlugin, ReloadablePlugin {
 
     /**
      * The ability to automatically cleanup ".watcher_history*" indices while also cleaning up Monitoring indices.
@@ -86,6 +87,8 @@ public class Monitoring extends Plugin implements ActionPlugin {
     private final boolean enabled;
     private final boolean transportClientMode;
 
+    private Exporters exporters;
+
     public Monitoring(Settings settings) {
         this.settings = settings;
         this.transportClientMode = XPackPlugin.transportClientMode(settings);
@@ -134,8 +137,7 @@ public class Monitoring extends Plugin implements ActionPlugin {
         Map<String, Exporter.Factory> exporterFactories = new HashMap<>();
         exporterFactories.put(HttpExporter.TYPE, config -> new HttpExporter(config, dynamicSSLService, threadPool.getThreadContext()));
         exporterFactories.put(LocalExporter.TYPE, config -> new LocalExporter(config, client, cleanerService));
-        final Exporters exporters = new Exporters(settings, exporterFactories, clusterService, getLicenseState(),
+        exporters = new Exporters(settings, exporterFactories, clusterService, getLicenseState(), threadPool.getThreadContext());
-            threadPool.getThreadContext());
 
         Set<Collector> collectors = new HashSet<>();
         collectors.add(new IndexStatsCollector(clusterService, getLicenseState(), client));
@@ -196,4 +198,13 @@ public class Monitoring extends Plugin implements ActionPlugin {
         final String exportersKey = "xpack.monitoring.exporters.";
         return Collections.unmodifiableList(Arrays.asList(exportersKey + "*.auth.*", exportersKey + "*.ssl.*"));
     }
+
+    @Override
+    public void reload(Settings settings) throws Exception {
+        final List<String> changedExporters = HttpExporter.loadSettings(settings);
+        for (String changedExporter : changedExporters) {
+            final Settings settingsForChangedExporter = settings.filter(x -> x.startsWith("xpack.monitoring.exporters." + changedExporter));
+            exporters.setExportersSetting(settingsForChangedExporter);
+        }
+    }
 }

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringService.java

@@ -22,6 +22,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
 import org.elasticsearch.xpack.monitoring.exporter.Exporters;
+import org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter;
 
 import java.io.Closeable;
 import java.util.ArrayList;
@@ -104,6 +105,8 @@ public class MonitoringService extends AbstractLifecycleComponent {
             .addSettingsUpdateConsumer(ELASTICSEARCH_COLLECTION_ENABLED, this::setElasticsearchCollectionEnabled);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(ENABLED, this::setMonitoringActive);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(INTERVAL, this::setInterval);
+
+        HttpExporter.loadSettings(settings);
     }
 
     void setElasticsearchCollectionEnabled(final boolean enabled) {

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java

@@ -35,6 +35,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 
@@ -58,15 +59,17 @@ public class Exporters extends AbstractLifecycleComponent {
         this.clusterService = Objects.requireNonNull(clusterService);
         this.licenseState = Objects.requireNonNull(licenseState);
 
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(this::setExportersSetting, getSettings());
+        final List<Setting.AffixSetting<?>> dynamicSettings =
+            getSettings().stream().filter(Setting::isDynamic).collect(Collectors.toList());
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(this::setExportersSetting, dynamicSettings);
         HttpExporter.registerSettingValidators(clusterService);
-        // this ensures, that logging is happening by adding an empty consumer per affix setting
+        // this ensures that logging is happening by adding an empty consumer per affix setting
-        for (Setting.AffixSetting<?> affixSetting : getSettings()) {
+        for (Setting.AffixSetting<?> affixSetting : dynamicSettings) {
             clusterService.getClusterSettings().addAffixUpdateConsumer(affixSetting, (s, o) -> {}, (s, o) -> {});
         }
     }
 
-    private void setExportersSetting(Settings exportersSetting) {
+    public void setExportersSetting(Settings exportersSetting) {
         if (this.lifecycle.started()) {
             Map<String, Exporter> updated = initExporters(exportersSetting);
             closeExporters(logger, this.exporters.getAndSet(updated));

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java

@@ -26,6 +26,8 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -52,6 +54,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -205,23 +208,20 @@ public class HttpExporter extends Exporter {
                                 final String namespace =
                                     HttpExporter.AUTH_USERNAME_SETTING.getNamespace(
                                         HttpExporter.AUTH_USERNAME_SETTING.getConcreteSetting(key));
-                                final String password =
-                                    (String) settings.get(AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(namespace));
 
                                 // password must be specified along with username for any auth
                                 if (Strings.isNullOrEmpty(username) == false) {
-                                    if (Strings.isNullOrEmpty(password)) {
-                                        throw new SettingsException(
-                                            "[" + AUTH_USERNAME_SETTING.getConcreteSettingForNamespace(namespace).getKey() + "] is set " +
-                                            "but [" + AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(namespace).getKey() + "] is " +
-                                            "missing");
-                                    }
                                     final String type =
                                         (String) settings.get(Exporter.TYPE_SETTING.getConcreteSettingForNamespace(namespace));
                                     if ("http".equals(type) == false) {
                                         throw new SettingsException("username for [" + key + "] is set but type is [" + type + "]");
                                     }
                                 }
+
+                                // it would be ideal to validate that just one of either AUTH_PASSWORD_SETTING or
+                                // AUTH_SECURE_PASSWORD_SETTING were present here, but that is not currently possible with the settings
+                                // validation framework.
+                                // https://github.com/elastic/elasticsearch/issues/51332
                             }
 
                             @Override
@@ -231,8 +231,8 @@ public class HttpExporter extends Exporter {
                                         HttpExporter.AUTH_USERNAME_SETTING.getConcreteSetting(key));
 
                                 final List<Setting<?>> settings = Arrays.asList(
-                                    Exporter.TYPE_SETTING.getConcreteSettingForNamespace(namespace),
+                                    Exporter.TYPE_SETTING.getConcreteSettingForNamespace(namespace));
-                                    HttpExporter.AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(namespace));
+
                                 return settings.iterator();
                             }
 
@@ -284,8 +284,18 @@ public class HttpExporter extends Exporter {
                         },
                         Property.Dynamic,
                         Property.NodeScope,
-                        Property.Filtered),
+                        Property.Filtered,
+                        Property.Deprecated),
                     TYPE_DEPENDENCY);
+    /**
+     * Secure password for basic auth.
+     */
+    public static final Setting.AffixSetting<SecureString> AUTH_SECURE_PASSWORD_SETTING =
+        Setting.affixKeySetting(
+            "xpack.monitoring.exporters.",
+            "auth.secure_password",
+            key -> SecureSetting.secureString(key, null),
+            TYPE_DEPENDENCY);
     /**
      * The SSL settings.
      *
@@ -400,6 +410,7 @@ public class HttpExporter extends Exporter {
      */
     private final AtomicBoolean clusterAlertsAllowed = new AtomicBoolean(false);
 
+    private static final ConcurrentHashMap<String, SecureString> SECURE_AUTH_PASSWORDS = new ConcurrentHashMap<>();
     private final ThreadContext threadContext;
     private final DateFormatter dateTimeFormatter;
 
@@ -697,6 +708,25 @@ public class HttpExporter extends Exporter {
         builder.setRequestConfigCallback(new TimeoutRequestConfigCallback(connectTimeout, socketTimeout));
     }
 
+
+    /**
+     * Caches secure settings for use when dynamically configuring HTTP exporters
+     * @param settings settings used for configuring HTTP exporter
+     * @return names of HTTP exporters whose secure settings changed, if any
+     */
+    public static List<String> loadSettings(Settings settings) {
+        final List<String> changedExporters = new ArrayList<>();
+        for (final String namespace : AUTH_SECURE_PASSWORD_SETTING.getNamespaces(settings)) {
+            final Setting<SecureString> s = AUTH_SECURE_PASSWORD_SETTING.getConcreteSettingForNamespace(namespace);
+            final SecureString securePassword = s.get(settings);
+            final SecureString existingPassword = SECURE_AUTH_PASSWORDS.put(namespace, securePassword);
+            if (securePassword.equals(existingPassword) == false) {
+                changedExporters.add(namespace);
+            }
+        }
+        return changedExporters;
+    }
+
     /**
      * Creates the optional {@link CredentialsProvider} with the username/password to use with <em>all</em> requests for user
      * authentication.
@@ -708,7 +738,19 @@ public class HttpExporter extends Exporter {
     @Nullable
     private static CredentialsProvider createCredentialsProvider(final Config config) {
         final String username = AUTH_USERNAME_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
-        final String password = AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
+
+        final String deprecatedPassword = AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
+        final SecureString securePassword = SECURE_AUTH_PASSWORDS.get(config.name());
+        final String password;
+        if (securePassword != null) {
+            password = securePassword.toString();
+            if (Strings.isNullOrEmpty(deprecatedPassword) == false) {
+                logger.warn("exporter [{}] specified both auth.secure_password and auth.password.  using auth.secure_password and " +
+                    "ignoring auth.password", config.name());
+            }
+        } else {
+            password = deprecatedPassword;
+        }
 
         final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
         credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
@@ -873,9 +915,19 @@ public class HttpExporter extends Exporter {
         }
     }
 
-    public static List<Setting.AffixSetting<?>> getSettings() {
+    public static List<Setting.AffixSetting<?>> getDynamicSettings() {
         return Arrays.asList(HOST_SETTING, TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING, AUTH_PASSWORD_SETTING, AUTH_USERNAME_SETTING,
                 BULK_TIMEOUT_SETTING, CONNECTION_READ_TIMEOUT_SETTING, CONNECTION_TIMEOUT_SETTING, PIPELINE_CHECK_TIMEOUT_SETTING,
                 PROXY_BASE_PATH_SETTING, SNIFF_ENABLED_SETTING, TEMPLATE_CHECK_TIMEOUT_SETTING, SSL_SETTING, HEADERS_SETTING);
     }
+
+    public static List<Setting.AffixSetting<?>> getSecureSettings() {
+        return Collections.singletonList(AUTH_SECURE_PASSWORD_SETTING);
+    }
+
+    public static List<Setting.AffixSetting<?>> getSettings() {
+        List<Setting.AffixSetting<?>> allSettings = new ArrayList<>(getDynamicSettings());
+        allSettings.addAll(getSecureSettings());
+        return allSettings;
+    }
 }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java

@@ -17,11 +17,13 @@ import java.nio.file.Path;
 
 public class LocalStateMonitoring extends LocalStateCompositeXPackPlugin {
 
+    final Monitoring monitoring;
+
     public LocalStateMonitoring(final Settings settings, final Path configPath) throws Exception {
         super(settings, configPath);
         LocalStateMonitoring thisVar = this;
 
-        plugins.add(new Monitoring(settings) {
+        monitoring = new Monitoring(settings) {
             @Override
             protected SSLService getSslService() {
                 return thisVar.getSslService();
@@ -36,7 +38,8 @@ public class LocalStateMonitoring extends LocalStateCompositeXPackPlugin {
             protected XPackLicenseState getLicenseState() {
                 return thisVar.getLicenseState();
             }
-        });
+        };
+        plugins.add(monitoring);
         plugins.add(new Watcher(settings) {
             @Override
             protected SSLService getSslService() {
@@ -50,4 +53,9 @@ public class LocalStateMonitoring extends LocalStateCompositeXPackPlugin {
         });
         plugins.add(new IndexLifecycle(settings));
     }
+
+    public Monitoring getMonitoring() {
+        return monitoring;
+    }
+
 }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.monitoring.exporter.http;
 
+import com.unboundid.util.Base64;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
@@ -19,6 +20,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -33,6 +35,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -42,6 +45,8 @@ import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.monitoring.LocalStateMonitoring;
+import org.elasticsearch.xpack.monitoring.MonitoringService;
 import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
 import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
@@ -93,9 +98,23 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
     private final boolean currentLicenseAllowsWatcher = true;
     private final boolean watcherAlreadyExists = randomBoolean();
     private final Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
+    private final String userName = "elasticuser";
 
     private MockWebServer webServer;
 
+    private MockSecureSettings mockSecureSettings  = new MockSecureSettings();
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+
+        Settings.Builder builder = Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(MonitoringService.INTERVAL.getKey(), MonitoringService.MIN_INTERVAL)
+            // we do this by default in core, but for monitoring this isn't needed and only adds noise.
+            .put("indices.lifecycle.history_index_enabled", false)
+            .put("index.store.mock.check_index_on_close", false);
+        return builder.build();
+    }
+
     @Before
     public void startWebServer() throws IOException {
         webServer = createMockWebServer();
@@ -113,6 +132,11 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
         return true;
     }
 
+    private Settings.Builder secureSettings(String password) {
+        mockSecureSettings.setString("xpack.monitoring.exporters._http.auth.secure_password", password);
+        return baseSettings().setSecureSettings(mockSecureSettings);
+    }
+
     private Settings.Builder baseSettings() {
         return Settings.builder()
             .put("xpack.monitoring.exporters._http.enabled", false)
@@ -121,7 +145,8 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
             .put("xpack.monitoring.exporters._http.headers.ignored", "value") // ensure that headers can be used by settings
             .put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
             .putList("xpack.monitoring.exporters._http.cluster_alerts.management.blacklist", clusterAlertBlacklist)
-            .put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates);
+            .put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
+            .put("xpack.monitoring.exporters._http.auth.username", userName);
     }
 
     public void testExport() throws Exception {
@@ -142,6 +167,42 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
         assertBulk(webServer, nbDocs);
     }
 
+    public void testSecureSetting() throws Exception {
+        final String securePassword1 = "elasticpass";
+        final String securePassword2 = "anotherpassword";
+        final String authHeaderValue = Base64.encode(userName + ":" + securePassword1);
+        final String authHeaderValue2 = Base64.encode(userName + ":" + securePassword2);
+
+        Settings settings = secureSettings(securePassword1)
+            .put("xpack.monitoring.exporters._http.auth.password", "insecurePassword") // verify this password is not used
+            .build();
+        PluginsService pluginsService = internalCluster().getInstances(PluginsService.class).iterator().next();
+        LocalStateMonitoring localStateMonitoring = pluginsService.filterPlugins(LocalStateMonitoring.class).iterator().next();
+        localStateMonitoring.getMonitoring().reload(settings);
+
+        enqueueGetClusterVersionResponse(Version.CURRENT);
+        enqueueSetupResponses(webServer,
+            templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
+            remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists);
+        enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
+
+        final int nbDocs = randomIntBetween(1, 25);
+        export(settings, newRandomMonitoringDocs(nbDocs));
+        assertEquals(webServer.takeRequest().getHeader("Authorization").replace("Basic", "").replace(" ", ""), authHeaderValue);
+        webServer.clearRequests();
+
+        settings = secureSettings(securePassword2).build();
+        localStateMonitoring.getMonitoring().reload(settings);
+        enqueueGetClusterVersionResponse(Version.CURRENT);
+        enqueueSetupResponses(webServer,
+            templatesExistsAlready, includeOldTemplates, pipelineExistsAlready,
+            remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists);
+        enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
+
+        export(settings, newRandomMonitoringDocs(nbDocs));
+        assertEquals(webServer.takeRequest().getHeader("Authorization").replace("Basic", "").replace(" ", ""), authHeaderValue2);
+    }
+
     public void testExportWithHeaders() throws Exception {
         final String headerValue = randomAlphaOfLengthBetween(3, 9);
         final String[] array = generateRandomStringArray(2, 4, false, false);

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java

@@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.TimeValue;
@@ -44,6 +45,7 @@ import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.OLD_TEMPLATE_IDS;
 import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS;
@@ -142,6 +144,25 @@ public class HttpExporterTests extends ESTestCase {
         assertThat(e, hasToString(containsString("[" + prefix + ".host] is set but type is [local]")));
     }
 
+    public void testSecurePasswordIsRejectedIfTypeIsNotHttp() {
+        final String prefix = "xpack.monitoring.exporters.example";
+        final Settings.Builder builder = Settings.builder().put(prefix + ".type", "local");
+
+        final String settingName = ".auth.secure_password";
+        final String settingValue = "securePassword";
+        MockSecureSettings mockSecureSettings  = new MockSecureSettings();
+        mockSecureSettings.setString(prefix + settingName, settingValue);
+
+        builder.setSecureSettings(mockSecureSettings);
+
+        final Settings settings = builder.build();
+        final ClusterSettings clusterSettings =
+            new ClusterSettings(settings,
+                Stream.of(HttpExporter.AUTH_SECURE_PASSWORD_SETTING, Exporter.TYPE_SETTING).collect(Collectors.toSet()));
+        final SettingsException e = expectThrows(SettingsException.class, () -> clusterSettings.validate(settings, true));
+        assertThat(e, hasToString(containsString("[" + prefix + settingName + "] is set but type is [local]")));
+    }
+
     public void testInvalidHost() {
         final String prefix = "xpack.monitoring.exporters.example";
         final String host = "https://example.com:443/";
@@ -235,28 +256,8 @@ public class HttpExporterTests extends ESTestCase {
             IllegalArgumentException.class,
             () -> HttpExporter.AUTH_PASSWORD_SETTING.getConcreteSetting(prefix + ".auth.password").get(settings));
         assertThat(e, hasToString(containsString(expected)));
-    }
+        assertWarnings("[xpack.monitoring.exporters._http.auth.password] setting was deprecated in Elasticsearch and will be removed " +
-
+            "in a future release! See the breaking changes documentation for the next major version.");
-    public void testExporterWithUsernameButNoPassword() {
-        final String expected =
-            "[xpack.monitoring.exporters._http.auth.username] is set but [xpack.monitoring.exporters._http.auth.password] is missing";
-        final String prefix = "xpack.monitoring.exporters._http";
-        final Settings settings = Settings.builder()
-            .put(prefix + ".type", HttpExporter.TYPE)
-            .put(prefix + ".host", "localhost:9200")
-            .put(prefix + ".auth.username", "_user")
-            .build();
-
-        final IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            () -> HttpExporter.AUTH_USERNAME_SETTING.getConcreteSetting(prefix + ".auth.username").get(settings));
-        assertThat(
-            e,
-            hasToString(
-                containsString("Failed to parse value for setting [xpack.monitoring.exporters._http.auth.username]")));
-
-        assertThat(e.getCause(), instanceOf(SettingsException.class));
-        assertThat(e.getCause(), hasToString(containsString(expected)));
     }
 
     public void testExporterWithUnknownBlacklistedClusterAlerts() {
@@ -333,7 +334,8 @@ public class HttpExporterTests extends ESTestCase {
                 .put("xpack.monitoring.exporters._http.host", "http://localhost:9200");
 
         // use basic auth
-        if (randomBoolean()) {
+        final boolean useBasicAuth = randomBoolean();
+        if (useBasicAuth) {
             builder.put("xpack.monitoring.exporters._http.auth.username", "_user")
                    .put("xpack.monitoring.exporters._http.auth.password", "_pass");
         }
@@ -348,6 +350,10 @@ public class HttpExporterTests extends ESTestCase {
 
         // doesn't explode
         HttpExporter.createRestClient(config, sslService, listener).close();
+        if (useBasicAuth) {
+            assertWarnings("[xpack.monitoring.exporters._http.auth.password] setting was deprecated in Elasticsearch and will be " +
+                "removed in a future release! See the breaking changes documentation for the next major version.");
+        }
     }
 
     public void testCreateSnifferDisabledByDefault() {