From 9c9da2e1e4b2e35338ec743928ec597df584c51f Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Fri, 15 Sep 2017 08:46:10 -0700 Subject: [PATCH] [Monitoring] Remove Legacy Monitoring Indices (elastic/x-pack-elasticsearch#2513) This changes Monitoring's Cleaner Service to remove any legacy Monitoring index that is appropriately old. This includes any `.marvel-*` index and also the "data" indices used by both Marvel and 5.0 - 5.4 versions of X-Pack monitoring, as well as the legacy alerts index. Original commit: elastic/x-pack-elasticsearch@8d99f5518b5347b335335b88871e6ee84838f45f --- .../xpack/monitoring/MonitoringSettings.java | 2 - .../exporter/local/LocalExporter.java | 17 +++--- .../AbstractIndicesCleanerTestCase.java | 60 ++++++++++--------- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSettings.java b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSettings.java index 3bb4f2557fb..ce3162d529e 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSettings.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSettings.java @@ -25,8 +25,6 @@ import static org.elasticsearch.common.settings.Setting.timeSetting; public class MonitoringSettings extends AbstractComponent { - public static final String LEGACY_DATA_INDEX_NAME = ".marvel-es-data"; - public static final String HISTORY_DURATION_SETTING_NAME = "history.duration"; /** * The minimum amount of time allowed for the history duration. diff --git a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 33922dfaaea..2749f0c06c1 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -74,6 +74,7 @@ import java.util.stream.StreamSupport; import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString; import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION; import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS; +import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION; import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.loadPipeline; import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.pipelineName; @@ -505,11 +506,9 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle if (clusterState != null) { long expirationTime = expiration.getMillis(); - // Get the list of monitoring index patterns - String[] patterns = StreamSupport.stream(getResolvers().spliterator(), false) - .map(MonitoringIndexNameResolver::indexPattern) - .distinct() - .toArray(String[]::new); + // TODO: remove .marvel-* handling in 7.0 + // Get the list of monitoring index patterns (note: this will include any unaliased .marvel-* indices) + final String[] monitoringIndexPatterns = new String[] { ".monitoring-*", ".marvel-*" }; MonitoringDoc monitoringDoc = new MonitoringDoc(null, null, null, null, null, System.currentTimeMillis(), (MonitoringDoc.Node) null); @@ -519,13 +518,15 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle .map(r -> r.index(monitoringDoc)) .collect(Collectors.toSet()); + // avoid deleting the current alerts index, but feel free to delete older ones + currents.add(".monitoring-alerts-" + TEMPLATE_VERSION); + Set indices = new HashSet<>(); for (ObjectObjectCursor index : clusterState.getMetaData().indices()) { String indexName = index.key; - if (Regex.simpleMatch(patterns, indexName)) { - - // Never delete the data index or a current index + if (Regex.simpleMatch(monitoringIndexPatterns, indexName)) { + // Never delete any "current" index (e.g., today's index or the most recent version no timestamp, like alerts) if (currents.contains(indexName)) { continue; } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java index d84f08eb844..8ffbc9e8d05 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java @@ -53,14 +53,15 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest assertIndicesCount(0); } - public void testIgnoreCurrentDataIndex() throws Exception { + public void testIgnoreCurrentAlertsIndex() throws Exception { internalCluster().startNode(); // Will be deleted createTimestampedIndex(now().minusDays(10)); // Won't be deleted - createDataIndex(now().minusDays(10)); + createAlertsIndex(now().minusYears(1)); + assertIndicesCount(2); CleanerService.Listener listener = getListener(); @@ -68,20 +69,34 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest assertIndicesCount(1); } - public void testIgnoreDataIndicesInOtherVersions() throws Exception { + public void testDoesNotIgnoreIndicesInOtherVersions() throws Exception { internalCluster().startNode(); // Will be deleted createTimestampedIndex(now().minusDays(10)); + createIndex(".marvel-es-data", now().minusYears(1)); + createIndex(".marvel-es-data-2", now().minusYears(1)); + createIndex(".monitoring-data-2", now().minusDays(10)); + createAlertsIndex(now().minusYears(1), MonitoringTemplateUtils.OLD_TEMPLATE_VERSION); + createIndex(".marvel-es-2016.05.03"); + createIndex(".marvel-es-2-2016.05.03"); + createTimestampedIndex(now().minusDays(10), "0"); + createTimestampedIndex(now().minusDays(10), "1"); + createTimestampedIndex(now().minusYears(1), MonitoringTemplateUtils.OLD_TEMPLATE_VERSION); + // In the past, this index would not be deleted, but starting in 6.x the monitoring cluster + // will be required to be a newer template version than the production cluster, so the index + // pushed to it will never be "unknown" in terms of their version (relates to the + // _xpack/monitoring/_setup API) + createTimestampedIndex(now().minusDays(10), String.valueOf(Integer.MAX_VALUE)); // Won't be deleted - createIndex(MonitoringSettings.LEGACY_DATA_INDEX_NAME, now().minusYears(1)); - createDataIndex(now().minusDays(10)); - assertIndicesCount(3); + createAlertsIndex(now().minusYears(1)); + + assertIndicesCount(12); CleanerService.Listener listener = getListener(); listener.onCleanUpIndices(days(0)); - assertIndicesCount(2); + assertIndicesCount(1); } public void testIgnoreCurrentTimestampedIndex() throws Exception { @@ -92,6 +107,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest // Won't be deleted createTimestampedIndex(now()); + assertIndicesCount(2); CleanerService.Listener listener = getListener(); @@ -99,23 +115,6 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest assertIndicesCount(1); } - public void testIgnoreTimestampedIndicesInOtherVersions() throws Exception { - internalCluster().startNode(); - - // Will be deleted - createTimestampedIndex(now().minusDays(10)); - - // Won't be deleted - createTimestampedIndex(now().minusDays(10), "0"); - createTimestampedIndex(now().minusDays(10), "1"); - createTimestampedIndex(now().minusDays(10), String.valueOf(Integer.MAX_VALUE)); - assertIndicesCount(4); - - CleanerService.Listener listener = getListener(); - listener.onCleanUpIndices(days(0)); - assertIndicesCount(3); - } - public void testDeleteIndices() throws Exception { internalCluster().startNode(); @@ -183,10 +182,17 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest } /** - * Creates a monitoring data index from an earlier version (from when we used to have them). + * Creates a monitoring alerts index from the current version. */ - protected void createDataIndex(DateTime creationDate) { - createIndex(".monitoring-data-2", creationDate); + protected void createAlertsIndex(final DateTime creationDate) { + createAlertsIndex(creationDate, MonitoringTemplateUtils.TEMPLATE_VERSION); + } + + /** + * Creates a monitoring alerts index from the current version. + */ + protected void createAlertsIndex(final DateTime creationDate, final String version) { + createIndex(".monitoring-alerts-" + version, creationDate); } /**