[Monitoring] Allow Cluster Alerts to be disabled in LocalExporter (elastic/x-pack-elasticsearch#1808)

You can now disable cluster alerts in the local exporter, which you can do in the HTTP exporter already.

This helps users that mess up their watcher configuration (e.g., disabling scripts) can turn off the feature to avoid log spam.

Original commit: elastic/x-pack-elasticsearch@f2096b553d

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -525,7 +525,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
      * @return {@code true} to use Cluster Alerts.
      */
     private boolean canUseWatcher() {
-        return XPackSettings.WATCHER_ENABLED.get(config.globalSettings());
+        return XPackSettings.WATCHER_ENABLED.get(config.globalSettings()) &&
+               config.settings().getAsBoolean(CLUSTER_ALERTS_MANAGEMENT_SETTING, true);
     }
 
     @Override

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java

@@ -19,6 +19,8 @@ import org.elasticsearch.xpack.security.InternalClient;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
+import static org.elasticsearch.xpack.monitoring.exporter.Exporter.CLUSTER_ALERTS_MANAGEMENT_SETTING;
+
 /**
  * {@code LocalExporterIntegTestCase} offers a basis for integration tests for the {@link LocalExporter}.
  */
@@ -52,11 +54,21 @@ public abstract class LocalExporterIntegTestCase extends MonitoringIntegTestCase
         return ENABLE_WATCHER;
     }
 
+    /**
+     * Override to disable the creation of Watches because of the setting (regardless of Watcher being enabled).
+     *
+     * @return Always {@code true} by default.
+     */
+    protected boolean useClusterAlerts() {
+        return true;
+    }
+
     protected Settings localExporterSettings() {
         return Settings.builder()
                        .put(MonitoringSettings.INTERVAL.getKey(), "-1")
                        .put("xpack.monitoring.exporters." + exporterName + ".type", LocalExporter.TYPE)
                        .put("xpack.monitoring.exporters." + exporterName +  ".enabled", false)
+                       .put("xpack.monitoring.exporters." + exporterName +  "." + CLUSTER_ALERTS_MANAGEMENT_SETTING, useClusterAlerts())
                        .put(XPackSettings.WATCHER_ENABLED.getKey(), enableWatcher())
                        .put(NetworkModule.HTTP_ENABLED.getKey(), false)
                        .build();

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java

@@ -46,10 +46,16 @@ import static org.hamcrest.Matchers.notNullValue;
                               numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0, supportsDedicatedMasters = false)
 public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase {
 
+    private final boolean useClusterAlerts = enableWatcher() && randomBoolean();
     private final MonitoredSystem system = randomFrom(MonitoredSystem.values());
     private final MockTimestampedIndexNameResolver resolver =
             new MockTimestampedIndexNameResolver(system, Settings.EMPTY, MonitoringTemplateUtils.TEMPLATE_VERSION);
 
+    @Override
+    protected boolean useClusterAlerts() {
+        return useClusterAlerts;
+    }
+
     public void testCreateWhenResourcesNeedToBeAddedOrUpdated() throws Exception {
         // sometimes they need to be added; sometimes they need to be replaced
         if (randomBoolean()) {
@@ -257,7 +263,11 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
         assertBusy(() -> {
             assertTemplatesExist();
             assertPipelinesExist();
-            assertWatchesExist();
+            if (useClusterAlerts) {
+                assertWatchesExist();
+            } else {
+                assertWatchesNotUpdated();
+            }
         });
     }
 
@@ -288,8 +298,11 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
                 final String watchName = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId);
                 final GetWatchResponse response = watcherClient.prepareGetWatch(watchName).get();
 
-                // ensure that the watch still contains a value associated with the test rather than the real watch
-                assertThat(response.getSource().getValue("metadata." + getTestName()), notNullValue());
+                // if we didn't find the watch, then we certainly didn't update it
+                if (response.isFound()) {
+                    // ensure that the watch still contains a value associated with the test rather than the real watch
+                    assertThat(response.getSource().getValue("metadata." + getTestName()), notNullValue());
+                }
             }
         }
     }