[Monitoring] Remove Legacy Monitoring Indices (elastic/x-pack-elasticsearch#2513)

This changes Monitoring's Cleaner Service to remove any legacy Monitoring index that is appropriately old.

This includes any `.marvel-*` index and also the "data" indices used by both Marvel and 5.0 - 5.4 versions of X-Pack monitoring, as well as the legacy alerts index.

Original commit: elastic/x-pack-elasticsearch@8d99f5518b

plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSettings.java

@@ -25,8 +25,6 @@ import static org.elasticsearch.common.settings.Setting.timeSetting;
 
 public class MonitoringSettings extends AbstractComponent {
 
-    public static final String LEGACY_DATA_INDEX_NAME = ".marvel-es-data";
-
     public static final String HISTORY_DURATION_SETTING_NAME = "history.duration";
     /**
      * The minimum amount of time allowed for the history duration.

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -74,6 +74,7 @@ import java.util.stream.StreamSupport;
 import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS;
+import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.loadPipeline;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.pipelineName;
 
@@ -505,11 +506,9 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
             if (clusterState != null) {
                 long expirationTime = expiration.getMillis();
 
-                // Get the list of monitoring index patterns
+                // TODO: remove .marvel-* handling in 7.0
-                String[] patterns = StreamSupport.stream(getResolvers().spliterator(), false)
+                // Get the list of monitoring index patterns (note: this will include any unaliased .marvel-* indices)
-                                                .map(MonitoringIndexNameResolver::indexPattern)
+                final String[] monitoringIndexPatterns = new String[] { ".monitoring-*", ".marvel-*" };
-                                                .distinct()
-                                                .toArray(String[]::new);
 
                 MonitoringDoc monitoringDoc = new MonitoringDoc(null, null, null, null, null,
                         System.currentTimeMillis(), (MonitoringDoc.Node) null);
@@ -519,13 +518,15 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
                                                     .map(r -> r.index(monitoringDoc))
                                                     .collect(Collectors.toSet());
 
+                // avoid deleting the current alerts index, but feel free to delete older ones
+                currents.add(".monitoring-alerts-" + TEMPLATE_VERSION);
+
                 Set<String> indices = new HashSet<>();
                 for (ObjectObjectCursor<String, IndexMetaData> index : clusterState.getMetaData().indices()) {
                     String indexName =  index.key;
 
-                    if (Regex.simpleMatch(patterns, indexName)) {
+                    if (Regex.simpleMatch(monitoringIndexPatterns, indexName)) {
-
+                        // Never delete any "current" index (e.g., today's index or the most recent version no timestamp, like alerts)
-                        // Never delete the data index or a current index
                         if (currents.contains(indexName)) {
                             continue;
                         }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java

@@ -53,14 +53,15 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
         assertIndicesCount(0);
     }
 
-    public void testIgnoreCurrentDataIndex() throws Exception {
+    public void testIgnoreCurrentAlertsIndex() throws Exception {
         internalCluster().startNode();
 
         // Will be deleted
         createTimestampedIndex(now().minusDays(10));
 
         // Won't be deleted
-        createDataIndex(now().minusDays(10));
+        createAlertsIndex(now().minusYears(1));
+
         assertIndicesCount(2);
 
         CleanerService.Listener listener = getListener();
@@ -68,20 +69,34 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
         assertIndicesCount(1);
     }
 
-    public void testIgnoreDataIndicesInOtherVersions() throws Exception {
+    public void testDoesNotIgnoreIndicesInOtherVersions() throws Exception {
         internalCluster().startNode();
 
         // Will be deleted
         createTimestampedIndex(now().minusDays(10));
+        createIndex(".marvel-es-data", now().minusYears(1));
+        createIndex(".marvel-es-data-2", now().minusYears(1));
+        createIndex(".monitoring-data-2", now().minusDays(10));
+        createAlertsIndex(now().minusYears(1), MonitoringTemplateUtils.OLD_TEMPLATE_VERSION);
+        createIndex(".marvel-es-2016.05.03");
+        createIndex(".marvel-es-2-2016.05.03");
+        createTimestampedIndex(now().minusDays(10), "0");
+        createTimestampedIndex(now().minusDays(10), "1");
+        createTimestampedIndex(now().minusYears(1), MonitoringTemplateUtils.OLD_TEMPLATE_VERSION);
+        // In the past, this index would not be deleted, but starting in 6.x the monitoring cluster
+        // will be required to be a newer template version than the production cluster, so the index
+        // pushed to it will never be "unknown" in terms of their version (relates to the
+        // _xpack/monitoring/_setup API)
+        createTimestampedIndex(now().minusDays(10), String.valueOf(Integer.MAX_VALUE));
 
         // Won't be deleted
-        createIndex(MonitoringSettings.LEGACY_DATA_INDEX_NAME, now().minusYears(1));
+        createAlertsIndex(now().minusYears(1));
-        createDataIndex(now().minusDays(10));
+
-        assertIndicesCount(3);
+        assertIndicesCount(12);
 
         CleanerService.Listener listener = getListener();
         listener.onCleanUpIndices(days(0));
-        assertIndicesCount(2);
+        assertIndicesCount(1);
     }
 
     public void testIgnoreCurrentTimestampedIndex() throws Exception {
@@ -92,6 +107,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
 
         // Won't be deleted
         createTimestampedIndex(now());
+
         assertIndicesCount(2);
 
         CleanerService.Listener listener = getListener();
@@ -99,23 +115,6 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
         assertIndicesCount(1);
     }
 
-    public void testIgnoreTimestampedIndicesInOtherVersions() throws Exception {
-        internalCluster().startNode();
-
-        // Will be deleted
-        createTimestampedIndex(now().minusDays(10));
-
-        // Won't be deleted
-        createTimestampedIndex(now().minusDays(10), "0");
-        createTimestampedIndex(now().minusDays(10), "1");
-        createTimestampedIndex(now().minusDays(10), String.valueOf(Integer.MAX_VALUE));
-        assertIndicesCount(4);
-
-        CleanerService.Listener listener = getListener();
-        listener.onCleanUpIndices(days(0));
-        assertIndicesCount(3);
-    }
-
     public void testDeleteIndices() throws Exception {
         internalCluster().startNode();
 
@@ -183,10 +182,17 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
     }
 
     /**
-     * Creates a monitoring data index from an earlier version (from when we used to have them).
+     * Creates a monitoring alerts index from the current version.
      */
-    protected void createDataIndex(DateTime creationDate) {
+    protected void createAlertsIndex(final DateTime creationDate) {
-        createIndex(".monitoring-data-2", creationDate);
+        createAlertsIndex(creationDate, MonitoringTemplateUtils.TEMPLATE_VERSION);
+    }
+
+    /**
+     * Creates a monitoring alerts index from the current version.
+     */
+    protected void createAlertsIndex(final DateTime creationDate, final String version) {
+        createIndex(".monitoring-alerts-" + version, creationDate);
     }
 
     /**