[Monitoring] Add Cluster Alert Blacklist (elastic/x-pack-elasticsearch#3326)

This adds a traditional blacklist that can be set [dynamically] at the exporter level to block specific cluster alerts (watches) from being created. It goes further and actually removes any watch that has been created already for the current cluster.

Original commit: elastic/x-pack-elasticsearch@1f67bb9501

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/ClusterAlertsUtil.java

@@ -10,12 +10,20 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.settings.SettingsException;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.monitoring.exporter.Exporter.CLUSTER_ALERTS_BLACKLIST_SETTING;
 
 /**
  * {@code ClusterAlertsUtil} provides static methods to easily load the JSON resources that
@@ -125,4 +133,30 @@ public class ClusterAlertsUtil {
         }
     }
 
+    /**
+     * Get any blacklisted cluster alerts by their ID.
+     *
+     * @param config The {@link Exporter}'s configuration, which is used for the {@link SettingsException}.
+     * @return Never {@code null}. Can be empty.
+     * @throws SettingsException if an unknown cluster alert ID exists in the blacklist.
+     */
+    public static List<String> getClusterAlertsBlacklist(final Exporter.Config config) {
+        final List<String> blacklist = config.settings().getAsList(CLUSTER_ALERTS_BLACKLIST_SETTING, Collections.emptyList());
+
+        // validate the blacklist only contains recognized IDs
+        if (blacklist.isEmpty() == false) {
+            final List<String> watchIds = Arrays.asList(ClusterAlertsUtil.WATCH_IDS);
+            final Set<String> unknownIds = blacklist.stream().filter(id -> watchIds.contains(id) == false).collect(Collectors.toSet());
+
+            if (unknownIds.isEmpty() == false) {
+                throw new SettingsException(
+                    "[" + Exporter.settingFQN(config, CLUSTER_ALERTS_BLACKLIST_SETTING) + "] contains unrecognized Cluster Alert IDs [" +
+                    String.join(", ", unknownIds) + "]"
+                );
+            }
+        }
+
+        return blacklist;
+    }
+
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporter.java

@@ -26,6 +26,12 @@ public abstract class Exporter implements AutoCloseable {
      * Every {@code Exporter} allows users to explicitly disable cluster alerts.
      */
     public static final String CLUSTER_ALERTS_MANAGEMENT_SETTING = "cluster_alerts.management.enabled";
+    /**
+     * Every {@code Exporter} allows users to explicitly disable specific cluster alerts.
+     * <p>
+     * When cluster alerts management is enabled, this should delete anything blacklisted here in addition to not creating it.
+     */
+    public static final String CLUSTER_ALERTS_BLACKLIST_SETTING = "cluster_alerts.management.blacklist";
     /**
      * Every {@code Exporter} allows users to use a different index time format.
      */
@@ -75,7 +81,7 @@ public abstract class Exporter implements AutoCloseable {
         return Exporters.EXPORTERS_SETTINGS.getKey() + config.name;
     }
 
-    protected static String settingFQN(final Config config, final String setting) {
+    public static String settingFQN(final Config config, final String setting) {
         return Exporters.EXPORTERS_SETTINGS.getKey() + config.name + "." + setting;
     }
 

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/ClusterAlertHttpResource.java

@@ -15,6 +15,7 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -49,8 +50,10 @@ public class ClusterAlertHttpResource extends PublishableHttpResource {
      */
     private final Supplier<String> watchId;
     /**
-     * Provides a fully formed Watch (e.g., no variables that need replaced).
+     * Provides a fully formed Watch (e.g., no variables that need replaced). If {@code null}, then we are always going to delete this
+     * Cluster Alert.
      */
+    @Nullable
     private final Supplier<String> watch;
 
     /**
@@ -58,17 +61,18 @@ public class ClusterAlertHttpResource extends PublishableHttpResource {
      *
      * @param resourceOwnerName The user-recognizable name.
      * @param watchId The name of the watch, which is lazily loaded.
-     * @param watch The watch provider.
+     * @param watch The watch provider. {@code null} indicates that we should always delete this Watch.
      */
     public ClusterAlertHttpResource(final String resourceOwnerName,
                                     final XPackLicenseState licenseState,
-                                    final Supplier<String> watchId, final Supplier<String> watch) {
+                                    final Supplier<String> watchId,
+                                    @Nullable final Supplier<String> watch) {
         // Watcher does not support master_timeout
         super(resourceOwnerName, null, CLUSTER_ALERT_VERSION_PARAMETERS);
 
         this.licenseState = Objects.requireNonNull(licenseState);
         this.watchId = Objects.requireNonNull(watchId);
-        this.watch = Objects.requireNonNull(watch);
+        this.watch = watch;
     }
 
     /**
@@ -77,7 +81,7 @@ public class ClusterAlertHttpResource extends PublishableHttpResource {
     @Override
     protected CheckResponse doCheck(final RestClient client) {
         // if we should be adding, then we need to check for existence
-        if (licenseState.isMonitoringClusterAlertsAllowed()) {
+        if (isWatchDefined() && licenseState.isMonitoringClusterAlertsAllowed()) {
             final CheckedFunction<Response, Boolean, IOException> watchChecker =
                     (response) -> shouldReplaceClusterAlert(response, XContentType.JSON.xContent(), LAST_UPDATED_VERSION);
 
@@ -105,6 +109,15 @@ public class ClusterAlertHttpResource extends PublishableHttpResource {
                            resourceOwnerName, "monitoring cluster");
     }
 
+    /**
+     * Determine if the {@link #watch} is defined. If not, then we should always delete the watch.
+     *
+     * @return {@code true} if {@link #watch} is defined (non-{@code null}). Otherwise {@code false}.
+     */
+    boolean isWatchDefined() {
+        return watch != null;
+    }
+
     /**
      * Create a {@link HttpEntity} for the {@link #watch}.
      *

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java

@@ -168,7 +168,7 @@ public class HttpExporter extends Exporter {
      * @throws SettingsException if any setting is malformed
      */
     public HttpExporter(final Config config, final SSLService sslService, final ThreadContext threadContext) {
-        this(config, sslService, threadContext, new NodeFailureListener());
+        this(config, sslService, threadContext, new NodeFailureListener(), createResources(config));
     }
 
     /**
@@ -179,8 +179,9 @@ public class HttpExporter extends Exporter {
      * @param listener The node failure listener used to notify an optional sniffer and resources
      * @throws SettingsException if any setting is malformed
      */
-    HttpExporter(final Config config, final SSLService sslService, final ThreadContext threadContext, final NodeFailureListener listener) {
-        this(config, createRestClient(config, sslService, listener), threadContext, listener);
+    HttpExporter(final Config config, final SSLService sslService, final ThreadContext threadContext, final NodeFailureListener listener,
+                 final HttpResource resource) {
+        this(config, createRestClient(config, sslService, listener), threadContext, listener, resource);
     }
 
     /**
@@ -191,21 +192,9 @@ public class HttpExporter extends Exporter {
      * @param listener The node failure listener used to notify an optional sniffer and resources
      * @throws SettingsException if any setting is malformed
      */
-    HttpExporter(final Config config, final RestClient client, final ThreadContext threadContext, final NodeFailureListener listener) {
-        this(config, client, createSniffer(config, client, listener), threadContext, listener);
-    }
-
-    /**
-     * Create an {@link HttpExporter}.
-     *
-     * @param config The HTTP Exporter's configuration
-     * @param client The REST Client used to make all requests to the remote Elasticsearch cluster
-     * @param listener The node failure listener used to notify an optional sniffer and resources
-     * @throws SettingsException if any setting is malformed
-     */
-    HttpExporter(final Config config, final RestClient client, @Nullable final Sniffer sniffer, final ThreadContext threadContext,
-                 final NodeFailureListener listener) {
-        this(config, client, sniffer, threadContext, listener, createResources(config));
+    HttpExporter(final Config config, final RestClient client, final ThreadContext threadContext, final NodeFailureListener listener,
+                 final HttpResource resource) {
+        this(config, client, createSniffer(config, client, listener), threadContext, listener, resource);
     }
 
     /**
@@ -583,12 +572,14 @@ public class HttpExporter extends Exporter {
         if (settings.getAsBoolean(CLUSTER_ALERTS_MANAGEMENT_SETTING, true)) {
             final ClusterService clusterService = config.clusterService();
             final List<HttpResource> watchResources = new ArrayList<>();
+            final List<String> blacklist = ClusterAlertsUtil.getClusterAlertsBlacklist(config);
 
             // add a resource per watch
             for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
+                final boolean blacklisted = blacklist.contains(watchId);
                 // lazily load the cluster state to fetch the cluster UUID once it's loaded
                 final Supplier<String> uniqueWatchId = () -> ClusterAlertsUtil.createUniqueWatchId(clusterService, watchId);
-                final Supplier<String> watch = () -> ClusterAlertsUtil.loadWatch(clusterService, watchId);
+                final Supplier<String> watch = blacklisted ? null : () -> ClusterAlertsUtil.loadWatch(clusterService, watchId);
 
                 watchResources.add(new ClusterAlertHttpResource(resourceOwnerName, config.licenseState(), uniqueWatchId, watch));
             }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -93,6 +93,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
     private final CleanerService cleanerService;
     private final boolean useIngest;
     private final DateTimeFormatter dateTimeFormatter;
+    private final List<String> clusterAlertBlacklist;
 
     private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
     private final AtomicBoolean installingSomething = new AtomicBoolean(false);
@@ -105,6 +106,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
         this.clusterService = config.clusterService();
         this.licenseState = config.licenseState();
         this.useIngest = config.settings().getAsBoolean(USE_INGEST_PIPELINE_SETTING, true);
+        this.clusterAlertBlacklist = ClusterAlertsUtil.getClusterAlertsBlacklist(config);
         this.cleanerService = cleanerService;
         this.dateTimeFormatter = dateTimeFormatter(config);
         clusterService.addListener(this);
@@ -436,10 +438,11 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
 
         for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
             final String uniqueWatchId = ClusterAlertsUtil.createUniqueWatchId(clusterService, watchId);
+            final boolean addWatch = canAddWatches && clusterAlertBlacklist.contains(watchId) == false;
 
             // we aren't sure if no watches exist yet, so add them
             if (indexExists) {
-                if (canAddWatches) {
+                if (addWatch) {
                     logger.trace("checking monitoring watch [{}]", uniqueWatchId);
 
                     asyncActions.add(() -> watcher.getWatch(new GetWatchRequest(uniqueWatchId),
@@ -451,7 +454,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
                     asyncActions.add(() -> watcher.deleteWatch(new DeleteWatchRequest(uniqueWatchId),
                                                                new ResponseActionListener<>("watch", uniqueWatchId, pendingResponses)));
                 }
-            } else if (canAddWatches) {
+            } else if (addWatch) {
                 asyncActions.add(() -> putWatch(watcher, watchId, uniqueWatchId, pendingResponses));
             }
         }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/ClusterAlertsUtilTests.java

@@ -8,14 +8,19 @@ package org.elasticsearch.xpack.monitoring.exporter;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
 import org.junit.Before;
 
 import static org.hamcrest.Matchers.containsString;
@@ -56,7 +61,7 @@ public class ClusterAlertsUtilTests extends ESTestCase {
         assertThat(uniqueWatchId, equalTo(clusterUuid + "_" + watchId));
     }
 
-    public void testLoadWatch() throws IOException {
+    public void testLoadWatch() {
         for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
             final String watch = ClusterAlertsUtil.loadWatch(clusterService, watchId);
 
@@ -74,4 +79,46 @@ public class ClusterAlertsUtilTests extends ESTestCase {
         expectThrows(RuntimeException.class, () -> ClusterAlertsUtil.loadWatch(clusterService, "watch-does-not-exist"));
     }
 
+    public void testGetClusterAlertsBlacklistThrowsForUnknownWatchId() {
+        final List<String> watchIds = Arrays.asList(ClusterAlertsUtil.WATCH_IDS);
+        final List<String> blacklist = randomSubsetOf(watchIds);
+
+        blacklist.add("fake1");
+
+        if (randomBoolean()) {
+            blacklist.add("fake2");
+
+            if (rarely()) {
+                blacklist.add("fake3");
+            }
+        }
+
+        final Set<String> unknownIds = blacklist.stream().filter(id -> watchIds.contains(id) == false).collect(Collectors.toSet());
+        final String unknownIdsString = String.join(", ", unknownIds);
+
+        final SettingsException exception =
+                expectThrows(SettingsException.class,
+                             () -> ClusterAlertsUtil.getClusterAlertsBlacklist(createConfigWithBlacklist("_random", blacklist)));
+
+        assertThat(exception.getMessage(),
+                   equalTo("[xpack.monitoring.exporters._random.cluster_alerts.management.blacklist] contains unrecognized Cluster " +
+                           "Alert IDs [" + unknownIdsString + "]"));
+    }
+
+    public void testGetClusterAlertsBlacklist() {
+        final List<String> blacklist = randomSubsetOf(Arrays.asList(ClusterAlertsUtil.WATCH_IDS));
+
+        assertThat(blacklist, equalTo(ClusterAlertsUtil.getClusterAlertsBlacklist(createConfigWithBlacklist("any", blacklist))));
+    }
+
+    private Exporter.Config createConfigWithBlacklist(final String name, final List<String> blacklist) {
+        final Settings settings = Settings.builder()
+                .putList(Exporter.CLUSTER_ALERTS_BLACKLIST_SETTING, blacklist)
+                .build();
+        final ClusterService clusterService = mock(ClusterService.class);
+        final XPackLicenseState licenseState = mock(XPackLicenseState.class);
+
+        return new Exporter.Config(name, "fake", Settings.EMPTY, settings, clusterService, licenseState);
+    }
+
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/ClusterAlertHttpResourceTests.java

@@ -36,6 +36,13 @@ public class ClusterAlertHttpResourceTests extends AbstractPublishableHttpResour
 
     private final ClusterAlertHttpResource resource = new ClusterAlertHttpResource(owner, licenseState, () -> watchId, () -> watchValue);
 
+    public void testIsWatchDefined() {
+        final ClusterAlertHttpResource noWatchResource = new ClusterAlertHttpResource(owner, licenseState, () -> watchId, null);
+
+        assertThat(noWatchResource.isWatchDefined(), is(false));
+        assertThat(resource.isWatchDefined(), is(true));
+    }
+
     public void testWatchToHttpEntity() throws IOException {
         final byte[] watchValueBytes = watchValue.getBytes(ContentType.APPLICATION_JSON.getCharset());
         final byte[] actualBytes = new byte[watchValueBytes.length];
@@ -91,6 +98,26 @@ public class ClusterAlertHttpResourceTests extends AbstractPublishableHttpResour
         }
     }
 
+    public void testDoCheckAsDeleteWatchExistsWhenNoWatchIsSpecified() throws IOException {
+        final ClusterAlertHttpResource noWatchResource = new ClusterAlertHttpResource(owner, licenseState, () -> watchId, null);
+        final boolean clusterAlertsAllowed = randomBoolean();
+
+        // should not matter
+        when(licenseState.isMonitoringClusterAlertsAllowed()).thenReturn(clusterAlertsAllowed);
+
+        assertCheckAsDeleteExists(noWatchResource, "/_xpack/watcher/watch", watchId);
+    }
+
+    public void testDoCheckWithExceptionAsDeleteWatchErrorWhenNoWatchIsSpecified() throws IOException {
+        final ClusterAlertHttpResource noWatchResource = new ClusterAlertHttpResource(owner, licenseState, () -> watchId, null);
+        final boolean clusterAlertsAllowed = randomBoolean();
+
+        // should not matter
+        when(licenseState.isMonitoringClusterAlertsAllowed()).thenReturn(clusterAlertsAllowed);
+
+        assertCheckAsDeleteWithException(noWatchResource, "/_xpack/watcher/watch", watchId);
+    }
+
     public void testDoCheckAsDeleteWatchExists() throws IOException {
         when(licenseState.isMonitoringClusterAlertsAllowed()).thenReturn(false);
 

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java

@@ -19,7 +19,6 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
@@ -75,6 +74,8 @@ import static org.hamcrest.Matchers.notNullValue;
                               numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0, supportsDedicatedMasters = false)
 public class HttpExporterIT extends MonitoringIntegTestCase {
 
+    private final List<String> clusterAlertBlacklist =
+            rarely() ? randomSubsetOf(Arrays.asList(ClusterAlertsUtil.WATCH_IDS)) : Collections.emptyList();
     private final boolean templatesExistsAlready = randomBoolean();
     private final boolean includeOldTemplates = randomBoolean();
     private final boolean pipelineExistsAlready = randomBoolean();
@@ -114,12 +115,16 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
                        .build();
     }
 
+    protected Settings.Builder baseSettings() {
+        return Settings.builder()
+                       .put("xpack.monitoring.exporters._http.type", "http")
+                       .put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
+                       .putList("xpack.monitoring.exporters._http.cluster_alerts.management.blacklist", clusterAlertBlacklist)
+                       .put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates);
+    }
+
     public void testExport() throws Exception {
-        final Settings settings = Settings.builder()
-                .put("xpack.monitoring.exporters._http.type", "http")
-                .put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
-                .put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
-                .build();
+        final Settings settings = baseSettings().build();
 
         enqueueGetClusterVersionResponse(Version.CURRENT);
         enqueueSetupResponses(webServer,
@@ -146,10 +151,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
         headers.put("X-Found-Cluster", new String[] { headerValue });
         headers.put("Array-Check", array);
 
-        Settings settings = Settings.builder()
-                .put("xpack.monitoring.exporters._http.type", "http")
-                .put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
-                .put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
+        final Settings settings = baseSettings()
                 .put("xpack.monitoring.exporters._http.headers.X-Cloud-Cluster", headerValue)
                 .put("xpack.monitoring.exporters._http.headers.X-Found-Cluster", headerValue)
                 .putList("xpack.monitoring.exporters._http.headers.Array-Check", array)
@@ -199,15 +201,9 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
             basePath = "/" + basePath;
         }
 
-        final Settings.Builder builder = Settings.builder()
-                .put("xpack.monitoring.exporters._http.type", "http")
-                .put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
+        final Settings.Builder builder = baseSettings()
                 .put("xpack.monitoring.exporters._http.proxy.base_path", basePath + (randomBoolean() ? "/" : ""));
 
-        if (includeOldTemplates == false) {
-            builder.put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", false);
-        }
-
         if (useHeaders) {
             builder.put("xpack.monitoring.exporters._http.headers.X-Cloud-Cluster", headerValue)
                     .put("xpack.monitoring.exporters._http.headers.X-Found-Cluster", headerValue)
@@ -231,11 +227,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
     }
 
     public void testHostChangeReChecksTemplate() throws Exception {
-        Settings settings = Settings.builder()
-                .put("xpack.monitoring.exporters._http.type", "http")
-                .put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
-                .put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
-                .build();
+        final Settings settings = baseSettings().build();
 
         enqueueGetClusterVersionResponse(Version.CURRENT);
         enqueueSetupResponses(webServer,
@@ -327,11 +319,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
     }
 
     public void testDynamicIndexFormatChange() throws Exception {
-        final Settings settings = Settings.builder()
-                .put("xpack.monitoring.exporters._http.type", "http")
-                .put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
-                .put("xpack.monitoring.exporters._http.index.template.create_legacy_templates", includeOldTemplates)
-                .build();
+        final Settings settings = baseSettings().build();
 
         enqueueGetClusterVersionResponse(Version.CURRENT);
         enqueueSetupResponses(webServer,
@@ -473,7 +461,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
                                       final boolean remoteClusterAllowsWatcher, final boolean currentLicenseAllowsWatcher,
                                       final boolean alreadyExists,
                                       @Nullable final Map<String, String[]> customHeaders,
-                                      @Nullable final String basePath) throws Exception {
+                                      @Nullable final String basePath) {
         final String pathPrefix = basePathToAssertablePrefix(basePath);
         MockRequest request;
 
@@ -486,13 +474,15 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
         assertHeaders(request, customHeaders);
 
         if (remoteClusterAllowsWatcher) {
-            for (Tuple<String, String> watch : monitoringWatches()) {
+            for (final Tuple<String, String> watch : monitoringWatches()) {
+                final String uniqueWatchId = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watch.v1());
+
                 request = webServer.takeRequest();
 
                 // GET / PUT if we are allowed to use it
-                if (currentLicenseAllowsWatcher) {
+                if (currentLicenseAllowsWatcher && clusterAlertBlacklist.contains(watch.v1()) == false) {
                     assertThat(request.getMethod(), equalTo("GET"));
-                    assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + watch.v1()));
+                    assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + uniqueWatchId));
                     assertThat(request.getUri().getQuery(), equalTo(resourceClusterAlertQueryString()));
                     assertHeaders(request, customHeaders);
 
@@ -500,7 +490,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
                         request = webServer.takeRequest();
 
                         assertThat(request.getMethod(), equalTo("PUT"));
-                        assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + watch.v1()));
+                        assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + uniqueWatchId));
                         assertThat(request.getUri().getQuery(), equalTo(resourceClusterAlertQueryString()));
                         assertThat(request.getBody(), equalTo(watch.v2()));
                         assertHeaders(request, customHeaders);
@@ -508,7 +498,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
                 // DELETE if we're not allowed to use it
                 } else {
                     assertThat(request.getMethod(), equalTo("DELETE"));
-                    assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + watch.v1()));
+                    assertThat(request.getUri().getPath(), equalTo(pathPrefix + "/_xpack/watcher/watch/" + uniqueWatchId));
                     assertThat(request.getUri().getQuery(), equalTo(resourceClusterAlertQueryString()));
                     assertHeaders(request, customHeaders);
                 }
@@ -775,59 +765,58 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
 
     private void enqueueClusterAlertResponsesDoesNotExistYet(final MockWebServer webServer)
             throws IOException {
-        for (final String watchId : monitoringWatchIds()) {
-            if (randomBoolean()) {
-                enqueueResponse(webServer, 404, "watch [" + watchId + "] does not exist");
-            } else if (randomBoolean()) {
-                final int version = ClusterAlertsUtil.LAST_UPDATED_VERSION - randomIntBetween(1, 1000000);
-
-                // it DOES exist, but it's an older version
-                enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{\"version_created\":" + version + "}}}");
+        for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
+            if (clusterAlertBlacklist.contains(watchId)) {
+                enqueueDeleteClusterAlertResponse(webServer, watchId);
             } else {
-                // no version specified
-                enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{}}}");
+                if (randomBoolean()) {
+                    enqueueResponse(webServer, 404, "watch [" + watchId + "] does not exist");
+                } else if (randomBoolean()) {
+                    final int version = ClusterAlertsUtil.LAST_UPDATED_VERSION - randomIntBetween(1, 1000000);
+
+                    // it DOES exist, but it's an older version
+                    enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{\"version_created\":" + version + "}}}");
+                } else {
+                    // no version specified
+                    enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{}}}");
+                }
+                enqueueResponse(webServer, 201, "[" + watchId + "] created");
             }
-            enqueueResponse(webServer, 201, "[" + watchId + "] created");
         }
     }
 
     private void enqueueClusterAlertResponsesExistsAlready(final MockWebServer webServer) throws IOException {
-        final int count = monitoringWatchIds().size();
-        for (int i = 0; i < count; ++i) {
-            final int existsVersion;
-
-            if (randomBoolean()) {
-                // it's a NEWER cluster alert
-                existsVersion = randomFrom(Version.CURRENT.id, ClusterAlertsUtil.LAST_UPDATED_VERSION) + randomIntBetween(1, 1000000);
+        for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
+            if (clusterAlertBlacklist.contains(watchId)) {
+                enqueueDeleteClusterAlertResponse(webServer, watchId);
             } else {
-                // we already put it
-                existsVersion = ClusterAlertsUtil.LAST_UPDATED_VERSION;
-            }
+                final int existsVersion;
 
-            enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{\"version_created\":" + existsVersion + "}}}");
+                if (randomBoolean()) {
+                    // it's a NEWER cluster alert
+                    existsVersion = randomFrom(Version.CURRENT.id, ClusterAlertsUtil.LAST_UPDATED_VERSION) + randomIntBetween(1, 1000000);
+                } else {
+                    // we already put it
+                    existsVersion = ClusterAlertsUtil.LAST_UPDATED_VERSION;
+                }
+
+                enqueueResponse(webServer, 200, "{\"metadata\":{\"xpack\":{\"version_created\":" + existsVersion + "}}}");
+            }
         }
     }
 
     private void enqueueDeleteClusterAlertResponses(final MockWebServer webServer) throws IOException {
-        for (final String watchId : monitoringWatchIds()) {
-            if (randomBoolean()) {
-                enqueueResponse(webServer, 404, "watch [" + watchId + "] did not exist");
-            } else {
-                enqueueResponse(webServer, 200, "watch [" + watchId + "] deleted");
-            }
+        for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
+            enqueueDeleteClusterAlertResponse(webServer, watchId);
         }
     }
 
-    private void writeIndex(XContentBuilder response, String index, String alias) throws IOException {
-        response.startObject(index);
-        {
-            response.startObject("aliases");
-            {
-                response.startObject(alias).endObject();
-            }
-            response.endObject();
+    private void enqueueDeleteClusterAlertResponse(final MockWebServer webServer, final String watchId) throws IOException {
+        if (randomBoolean()) {
+            enqueueResponse(webServer, 404, "watch [" + watchId + "] did not exist");
+        } else {
+            enqueueResponse(webServer, 200, "watch [" + watchId + "] deleted");
         }
-        response.endObject();
     }
 
     private void enqueueResponse(int responseCode, String body) throws IOException {

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java

@@ -29,6 +29,7 @@ import org.junit.Before;
 import org.mockito.InOrder;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -195,6 +196,34 @@ public class HttpExporterTests extends ESTestCase {
         assertThat(exception.getMessage(), equalTo("[xpack.monitoring.exporters._http.host] invalid host: [" + invalidHost + "]"));
     }
 
+    public void testExporterWithUnknownBlacklistedClusterAlerts() {
+        final SSLIOSessionStrategy sslStrategy = mock(SSLIOSessionStrategy.class);
+        when(sslService.sslIOSessionStrategy(any(Settings.class))).thenReturn(sslStrategy);
+
+        final List<String> blacklist = new ArrayList<>();
+        blacklist.add("does_not_exist");
+
+        if (randomBoolean()) {
+            // a valid ID
+            blacklist.add(randomFrom(ClusterAlertsUtil.WATCH_IDS));
+            Collections.shuffle(blacklist, random());
+        }
+
+        final Settings.Builder builder = Settings.builder()
+                .put("xpack.monitoring.exporters._http.type", HttpExporter.TYPE)
+                .put("xpack.monitoring.exporters._http.host", "http://localhost:9200")
+                .putList("xpack.monitoring.exporters._http.cluster_alerts.management.blacklist", blacklist);
+
+        final Config config = createConfig(builder.build());
+
+        final SettingsException exception =
+                expectThrows(SettingsException.class, () -> new HttpExporter(config, sslService, threadContext));
+
+        assertThat(exception.getMessage(),
+                   equalTo("[xpack.monitoring.exporters._http.cluster_alerts.management.blacklist] contains unrecognized Cluster " +
+                           "Alert IDs [does_not_exist]"));
+    }
+
     public void testExporterWithHostOnly() throws Exception {
         final SSLIOSessionStrategy sslStrategy = mock(SSLIOSessionStrategy.class);
         when(sslService.sslIOSessionStrategy(any(Settings.class))).thenReturn(sslStrategy);

plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java

@@ -185,14 +185,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
         final ClusterService clusterService = clusterService();
 
         return Arrays.stream(ClusterAlertsUtil.WATCH_IDS)
-                .map(id -> new Tuple<>(ClusterAlertsUtil.createUniqueWatchId(clusterService, id),
-                                       ClusterAlertsUtil.loadWatch(clusterService, id)))
-                .collect(Collectors.toList());
-    }
-
-    protected List<String> monitoringWatchIds() {
-        return Arrays.stream(ClusterAlertsUtil.WATCH_IDS)
-                .map(id -> ClusterAlertsUtil.createUniqueWatchId(clusterService(), id))
+                .map(id -> new Tuple<>(id, ClusterAlertsUtil.loadWatch(clusterService, id)))
                 .collect(Collectors.toList());
     }