[Monitoring] Cleaner Service should be able to cleanup .watcher-history* (elastic/x-pack-elasticsearch#2696)

This adds a dynamic setting, which defaults to `false` currently, that can be used to delete all `.watcher-history*` indices that match the same age requirements as Monitoring indices.

Original commit: elastic/x-pack-elasticsearch@8ca3bdbca3

plugin/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java

@@ -57,6 +57,7 @@ import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.common.settings.Setting.boolSetting;
 import static org.elasticsearch.common.settings.Setting.timeSetting;
 
 /**
@@ -89,6 +90,12 @@ public class Monitoring implements ActionPlugin {
                                                                           TimeValue.timeValueHours(7 * 24), // default value (7 days)
                                                                           HISTORY_DURATION_MINIMUM,         // minimum value
                                                                           Setting.Property.Dynamic, Setting.Property.NodeScope);
+    /**
+     * The ability to automatically cleanup ".watcher_history*" indices while also cleaning up Monitoring indices.
+     */
+    public static final Setting<Boolean> CLEAN_WATCHER_HISTORY = boolSetting("xpack.watcher.history.cleaner_service.enabled",
+                                                                             false,
+                                                                             Setting.Property.Dynamic, Setting.Property.NodeScope);
 
     private final Settings settings;
     private final XPackLicenseState licenseState;
@@ -172,6 +179,7 @@ public class Monitoring implements ActionPlugin {
     public List<Setting<?>> getSettings() {
         return Collections.unmodifiableList(
                 Arrays.asList(HISTORY_DURATION,
+                              CLEAN_WATCHER_HISTORY,
                               MonitoringService.INTERVAL,
                               Exporters.EXPORTERS_SETTINGS,
                               Collector.INDICES,

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -69,6 +69,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString;
+import static org.elasticsearch.xpack.monitoring.Monitoring.CLEAN_WATCHER_HISTORY;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
@@ -483,10 +484,11 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
             if (clusterState != null) {
                 final long expirationTimeMillis = expiration.getMillis();
                 final long currentTimeMillis = System.currentTimeMillis();
+                final boolean cleanUpWatcherHistory = clusterService.getClusterSettings().get(CLEAN_WATCHER_HISTORY);
 
-                // list of index patterns that we clean up; we may add watcher history in the future
+                // list of index patterns that we clean up; watcher history can be included
-                final String[] indexPatterns = new String[] { ".monitoring-*" };
+                final String[] indexPatterns =
-
+                        cleanUpWatcherHistory ? new String[] { ".monitoring-*", ".watcher-history*" } : new String[] { ".monitoring-*" };
 
                 // Get the names of the current monitoring indices
                 final Set<String> currents = MonitoredSystem.allSystems()

plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java

@@ -14,6 +14,7 @@ import org.elasticsearch.xpack.monitoring.exporter.Exporter;
 import org.elasticsearch.xpack.monitoring.exporter.Exporters;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
+import org.elasticsearch.xpack.watcher.support.WatcherIndexTemplateRegistry;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -185,12 +186,29 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
     }
 
     /**
-     * Creates a monitoring alerts index from the current version.
+     * Creates a monitoring alerts index from the specified version.
      */
     protected void createAlertsIndex(final DateTime creationDate, final String version) {
         createIndex(".monitoring-alerts-" + version, creationDate);
     }
 
+    /**
+     * Creates a watcher history index from the current version.
+     */
+    protected void createWatcherHistoryIndex(final DateTime creationDate) {
+        createWatcherHistoryIndex(creationDate, WatcherIndexTemplateRegistry.INDEX_TEMPLATE_VERSION);
+    }
+
+    /**
+     * Creates a watcher history index from the specified version.
+     */
+    protected void createWatcherHistoryIndex(final DateTime creationDate, final String version) {
+        final DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYY.MM.dd").withZoneUTC();
+        final String index = ".watcher-history-" + version + "-" + formatter.print(creationDate.getMillis());
+
+        createIndex(index, creationDate);
+    }
+
     /**
      * Creates a monitoring timestamped index using the current template version.
      */
@@ -211,21 +229,21 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
 
     protected abstract void assertIndicesCount(int count) throws Exception;
 
-    private static TimeValue years(int years) {
+    protected static TimeValue years(int years) {
         DateTime now = now();
         return TimeValue.timeValueMillis(now.getMillis() - now.minusYears(years).getMillis());
     }
 
-    private static TimeValue months(int months) {
+    protected static TimeValue months(int months) {
         DateTime now = now();
         return TimeValue.timeValueMillis(now.getMillis() - now.minusMonths(months).getMillis());
     }
 
-    private static TimeValue days(int days) {
+    protected static TimeValue days(int days) {
         return TimeValue.timeValueHours(days * 24);
     }
 
-    private static DateTime now() {
+    protected static DateTime now() {
         return new DateTime(DateTimeZone.UTC);
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/local/LocalIndicesCleanerTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.xpack.monitoring.cleaner.AbstractIndicesCleanerTestCase;
+import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
 import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter;
 import org.joda.time.DateTime;
 
@@ -23,6 +24,8 @@ import static org.hamcrest.Matchers.equalTo;
 
 public class LocalIndicesCleanerTests extends AbstractIndicesCleanerTestCase {
 
+    private final boolean cleanUpWatcherHistory = randomBoolean();
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
@@ -35,6 +38,7 @@ public class LocalIndicesCleanerTests extends AbstractIndicesCleanerTestCase {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
                 .put("xpack.monitoring.exporters._local.type", LocalExporter.TYPE)
+                .put("xpack.watcher.history.cleaner_service.enabled", cleanUpWatcherHistory)
                 .build();
     }
 
@@ -57,4 +61,31 @@ public class LocalIndicesCleanerTests extends AbstractIndicesCleanerTestCase {
             assertThat(getSettingsResponse.getIndexToSettings().size(), equalTo(count));
         });
     }
+
+    public void testHandlesWatcherHistory() throws Exception {
+        internalCluster().startNode();
+
+        // Will be deleted (if we delete them)
+        createWatcherHistoryIndex(now().minusDays(7));
+        createWatcherHistoryIndex(now().minusDays(10), "6");
+        createWatcherHistoryIndex(now().minusDays(14), "3");
+        createWatcherHistoryIndex(now().minusDays(30), "2");
+        createWatcherHistoryIndex(now().minusYears(1), "1");
+        createWatcherHistoryIndex(now().minusDays(10), String.valueOf(Integer.MAX_VALUE));
+
+        // Won't be deleted
+        createWatcherHistoryIndex(now());
+
+        assertIndicesCount(7);
+
+        CleanerService.Listener listener = getListener();
+        listener.onCleanUpIndices(days(3));
+
+        if (cleanUpWatcherHistory) {
+            assertIndicesCount(1);
+        } else {
+            assertIndicesCount(7);
+        }
+    }
+
 }