From edd993077b514a7a0100d4fd2e737574e84e16a0 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 14 Jan 2016 16:22:22 +0100 Subject: [PATCH] Marvel: Only clean timestamped indices with the current template version Only current timestamped indices, like .marvel-es-1-* indices should be deleted. Other indices like the ones created by pre v2.3.0 plugin versions should be kept (like .marvel-es-YYYY.MM.dd) Original commit: elastic/x-pack-elasticsearch@b2aff318753d159accdfa1aa28187aca0b17eff3 --- .../elasticsearch/marvel/MarvelSettings.java | 2 + .../agent/collector/AbstractCollector.java | 16 +++- .../cluster/ClusterStateCollector.java | 2 +- .../cluster/ClusterStatsCollector.java | 2 +- .../marvel/agent/exporter/Exporter.java | 11 ++- ....java => MonitoringIndexNameResolver.java} | 10 ++- .../agent/exporter/local/LocalBulk.java | 6 +- .../agent/exporter/local/LocalExporter.java | 12 ++- .../AbstractIndicesCleanerTestCase.java | 81 +++++++++++++------ .../local/LocalIndicesCleanerTests.java | 4 +- 10 files changed, 104 insertions(+), 42 deletions(-) rename elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/{IndexNameResolver.java => MonitoringIndexNameResolver.java} (58%) diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java index 8b5e59f2840..e534aa37d71 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java @@ -24,6 +24,8 @@ public class MarvelSettings extends AbstractComponent { public static final String MONITORING_INDICES_PREFIX = ".monitoring-es-"; public static final String MONITORING_DATA_INDEX_PREFIX = ".monitoring-es-data-"; + public static final String LEGACY_DATA_INDEX_NAME = ".marvel-es-data"; + public static final String HISTORY_DURATION_SETTING_NAME = "history.duration"; public static final TimeValue MAX_LICENSE_GRACE_PERIOD = TimeValue.timeValueHours(7 * 24); diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java index 31c7b2fc44d..1feeabd5a34 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.MarvelSettings; -import org.elasticsearch.marvel.agent.exporter.IndexNameResolver; +import org.elasticsearch.marvel.agent.exporter.MonitoringIndexNameResolver; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.license.MarvelLicensee; @@ -27,7 +27,7 @@ public abstract class AbstractCollector extends AbstractLifecycleComponent protected final ClusterService clusterService; protected final MarvelSettings marvelSettings; protected final MarvelLicensee licensee; - protected final IndexNameResolver dataIndexNameResolver; + private final MonitoringIndexNameResolver dataIndexNameResolver; @Inject public AbstractCollector(Settings settings, String name, ClusterService clusterService, @@ -116,14 +116,19 @@ public abstract class AbstractCollector extends AbstractLifecycleComponent return clusterService.state().metaData().clusterUUID(); } + protected DiscoveryNode localNode() { return clusterService.localNode(); } + public String resolveDataIndexName(long timestamp) { + return dataIndexNameResolver.resolve(timestamp); + } + /** * Resolves monitoring's data index name */ - public class DataIndexNameResolver implements IndexNameResolver { + public class DataIndexNameResolver implements MonitoringIndexNameResolver { private final String index; @@ -141,5 +146,10 @@ public abstract class AbstractCollector extends AbstractLifecycleComponent public String resolve(long timestamp) { return index; } + + @Override + public String indexPattern() { + return index; + } } } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java index cbba4f28ebc..bdfc5282fbb 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java @@ -86,7 +86,7 @@ public class ClusterStateCollector extends AbstractCollector state = new AtomicReference<>(); - public LocalBulk(String name, ESLogger logger, Client client, IndexNameResolver indexNameResolver, RendererRegistry renderers) { + public LocalBulk(String name, ESLogger logger, Client client, MonitoringIndexNameResolver indexNameResolver, RendererRegistry renderers) { super(name); this.logger = logger; this.client = client; diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java index a320f7fc9a7..a27ec16c440 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java @@ -24,11 +24,11 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.agent.exporter.ExportBulk; import org.elasticsearch.marvel.agent.exporter.Exporter; import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.renderer.RendererRegistry; -import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.cleaner.CleanerService; import org.elasticsearch.shield.InternalClient; import org.joda.time.DateTime; @@ -268,11 +268,15 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle long expirationTime = expiration.getMillis(); Set indices = new HashSet<>(); + // Only clean up indices that match the timestamped index pattern + String pattern = indexNameResolver().indexPattern(); for (ObjectObjectCursor index : clusterState.getMetaData().indices()) { String indexName = index.key; - if (Regex.simpleMatch(MarvelSettings.MONITORING_INDICES_PREFIX + "*", indexName)) { - // Never delete the data indices - if (indexName.startsWith(MarvelSettings.MONITORING_DATA_INDEX_PREFIX)) { + + if (Regex.simpleMatch(pattern, indexName)) { + // Should not happen, but just in case... we never delete the data indices + if (indexName.startsWith(MarvelSettings.MONITORING_DATA_INDEX_PREFIX) + || indexName.equalsIgnoreCase(MarvelSettings.LEGACY_DATA_INDEX_NAME)) { continue; } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/AbstractIndicesCleanerTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/AbstractIndicesCleanerTestCase.java index 73d2a8eda4e..5d0034b1e40 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/AbstractIndicesCleanerTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/AbstractIndicesCleanerTestCase.java @@ -10,15 +10,16 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.license.core.License; import org.elasticsearch.license.plugin.core.LicenseState; import org.elasticsearch.license.plugin.core.Licensee; +import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.agent.exporter.Exporter; import org.elasticsearch.marvel.agent.exporter.Exporters; -import org.elasticsearch.marvel.agent.exporter.IndexNameResolver; -import org.elasticsearch.marvel.MarvelSettings; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.license.MarvelLicensee; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; import java.util.Locale; @@ -49,7 +50,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase public void testDeleteIndex() throws Exception { internalCluster().startNode(); - createIndex(MarvelSettings.MONITORING_INDICES_PREFIX + "test", now().minusDays(10)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10)); assertIndicesCount(1); CleanerService.Listener listener = getListener(); @@ -57,10 +58,10 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase assertIndicesCount(0); } - public void testIgnoreDataIndex() throws Exception { + public void testIgnoreCurrentDataIndex() throws Exception { internalCluster().startNode(); - createIndex(MarvelSettings.MONITORING_DATA_INDEX_PREFIX + "test", now().minusDays(10)); + createDataIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10)); assertIndicesCount(1); CleanerService.Listener listener = getListener(); @@ -68,24 +69,41 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase assertIndicesCount(1); } + public void testIgnoreDataIndicesInOtherVersions() throws Exception { + internalCluster().startNode(); + + createIndex(MarvelSettings.LEGACY_DATA_INDEX_NAME, now().minusYears(1)); + createDataIndex(0, now().minusDays(10)); + createDataIndex(Integer.MAX_VALUE, now().minusDays(20)); + assertIndicesCount(3); + + CleanerService.Listener listener = getListener(); + listener.onCleanUpIndices(days(0)); + assertIndicesCount(3); + } + public void testIgnoreCurrentTimestampedIndex() throws Exception { internalCluster().startNode(); - IndexNameResolver indexNameResolver = null; - for (Exporter exporter : internalCluster().getInstance(Exporters.class)) { - indexNameResolver = exporter.indexNameResolver(); - } - assertNotNull(indexNameResolver); - - DateTime tenDaysAgo = now().minusDays(10); - createIndex(indexNameResolver.resolve(tenDaysAgo.getMillis()), tenDaysAgo); - - DateTime today = now(); - createIndex(indexNameResolver.resolve(today.getMillis()), today); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now().minusDays(10)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now()); assertIndicesCount(2); CleanerService.Listener listener = getListener(); listener.onCleanUpIndices(days(0)); + assertIndicesCount(1); + } + + public void testIgnoreTimestampedIndicesInOtherVersions() throws Exception { + internalCluster().startNode(); + + createTimestampedIndex(0, now().minusDays(10)); + createTimestampedIndex(Integer.MAX_VALUE, now().minusDays(10)); + assertIndicesCount(2); + + CleanerService.Listener listener = getListener(); + listener.onCleanUpIndices(days(0)); + assertIndicesCount(2); } public void testDeleteIndices() throws Exception { @@ -94,11 +112,11 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase CleanerService.Listener listener = getListener(); final DateTime now = now(); - createIndex(MarvelSettings.MONITORING_INDICES_PREFIX + "one-year-ago", now.minusYears(1)); - createIndex(MarvelSettings.MONITORING_INDICES_PREFIX + "six-months-ago", now.minusMonths(6)); - createIndex(MarvelSettings.MONITORING_INDICES_PREFIX + "one-month-ago", now.minusMonths(1)); - createIndex(MarvelSettings.MONITORING_INDICES_PREFIX + "ten-days-ago", now.minusDays(10)); - createIndex(MarvelSettings.MONITORING_INDICES_PREFIX + "one-day-ago", now.minusDays(1)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusYears(1)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusMonths(6)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusMonths(1)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(10)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(1)); assertIndicesCount(5); // Clean indices that have expired two years ago @@ -134,7 +152,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase final DateTime now = now(); for (int i = 0; i < max; i++) { - createIndex(MarvelSettings.MONITORING_INDICES_PREFIX + String.valueOf(i), now.minusDays(i)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(i)); } assertIndicesCount(max); @@ -154,7 +172,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase final DateTime now = now(); for (int i = 0; i < max; i++) { - createIndex(MarvelSettings.MONITORING_INDICES_PREFIX + String.valueOf(i), now.minusDays(i)); + createTimestampedIndex(MarvelTemplateUtils.TEMPLATE_VERSION, now.minusDays(i)); } assertIndicesCount(max); @@ -194,6 +212,23 @@ public abstract class AbstractIndicesCleanerTestCase extends MarvelIntegTestCase throw new IllegalStateException("unable to find listener"); } + /** + * Creates a monitoring data index in a given version. + */ + protected void createDataIndex(int version, DateTime creationDate) { + String indexName = MarvelSettings.MONITORING_DATA_INDEX_PREFIX + String.valueOf(version); + createIndex(indexName, creationDate); + } + + /** + * Creates a monitoring timestamped index in a given version. + */ + protected void createTimestampedIndex(int version, DateTime creationDate) { + String indexName = MarvelSettings.MONITORING_INDICES_PREFIX + String.valueOf(version) + "-" + + DateTimeFormat.forPattern(Exporter.DEFAULT_INDEX_NAME_TIME_FORMAT).withZoneUTC().print(creationDate.getMillis()); + createIndex(indexName, creationDate); + } + protected abstract void createIndex(String name, DateTime creationDate); protected abstract void assertIndicesCount(int count) throws Exception; diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/local/LocalIndicesCleanerTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/local/LocalIndicesCleanerTests.java index 91d1f4564bb..7cd7f5d5a85 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/local/LocalIndicesCleanerTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/cleaner/local/LocalIndicesCleanerTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.marvel.agent.exporter.local.LocalExporter; -import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.cleaner.AbstractIndicesCleanerTestCase; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.InternalSettingsPlugin; @@ -52,8 +51,7 @@ public class LocalIndicesCleanerTests extends AbstractIndicesCleanerTestCase { @Override public void run() { try { - assertThat(client().admin().indices().prepareGetSettings(MarvelSettings.MONITORING_INDICES_PREFIX + "*") - .get().getIndexToSettings().size(), equalTo(count)); + assertThat(client().admin().indices().prepareGetSettings().get().getIndexToSettings().size(), equalTo(count)); } catch (IndexNotFoundException e) { if (shieldEnabled) { assertThat(0, equalTo(count));