honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Monitoring] Remove Legacy Monitoring Indices (elastic/x-pack-elasticsearch#2513) 

This changes Monitoring's Cleaner Service to remove any legacy Monitoring index that is appropriately old.

This includes any `.marvel-*` index and also the "data" indices used by both Marvel and 5.0 - 5.4 versions of X-Pack monitoring, as well as the legacy alerts index.

Original commit: elastic/x-pack-elasticsearch@8d99f5518b
This commit is contained in:
Chris Earle 2017-09-15 08:46:10 -07:00 committed by GitHub
parent 023bdb72b2
commit 9c9da2e1e4
3 changed files with 42 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSettings.java
View File

@ -25,8 +25,6 @@ import static org.elasticsearch.common.settings.Setting.timeSetting;
public class MonitoringSettings extends AbstractComponent {
public static final String LEGACY_DATA_INDEX_NAME = ".marvel-es-data";
public static final String HISTORY_DURATION_SETTING_NAME = "history.duration";
/**
* The minimum amount of time allowed for the history duration.

17
plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
View File

@ -74,6 +74,7 @@ import java.util.stream.StreamSupport;
import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.PIPELINE_IDS;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.loadPipeline;
import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.pipelineName;
@ -505,11 +506,9 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
if (clusterState != null) {
long expirationTime = expiration.getMillis();
// Get the list of monitoring index patterns
String[] patterns = StreamSupport.stream(getResolvers().spliterator(), false)
.map(MonitoringIndexNameResolver::indexPattern)
.distinct()
.toArray(String[]::new);
// TODO: remove .marvel-* handling in 7.0
// Get the list of monitoring index patterns (note: this will include any unaliased .marvel-* indices)
final String[] monitoringIndexPatterns = new String[] { ".monitoring-*", ".marvel-*" };
MonitoringDoc monitoringDoc = new MonitoringDoc(null, null, null, null, null,
System.currentTimeMillis(), (MonitoringDoc.Node) null);
@ -519,13 +518,15 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
.map(r -> r.index(monitoringDoc))
.collect(Collectors.toSet());
// avoid deleting the current alerts index, but feel free to delete older ones
currents.add(".monitoring-alerts-" + TEMPLATE_VERSION);
Set<String> indices = new HashSet<>();
for (ObjectObjectCursor<String, IndexMetaData> index : clusterState.getMetaData().indices()) {
String indexName = index.key;
if (Regex.simpleMatch(patterns, indexName)) {
// Never delete the data index or a current index
if (Regex.simpleMatch(monitoringIndexPatterns, indexName)) {
// Never delete any "current" index (e.g., today's index or the most recent version no timestamp, like alerts)
if (currents.contains(indexName)) {
continue;
}

60
plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java
View File

@ -53,14 +53,15 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
assertIndicesCount(0);
}
public void testIgnoreCurrentDataIndex() throws Exception {
public void testIgnoreCurrentAlertsIndex() throws Exception {
internalCluster().startNode();
// Will be deleted
createTimestampedIndex(now().minusDays(10));
// Won't be deleted
createDataIndex(now().minusDays(10));
createAlertsIndex(now().minusYears(1));
assertIndicesCount(2);
CleanerService.Listener listener = getListener();
@ -68,20 +69,34 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
assertIndicesCount(1);
}
public void testIgnoreDataIndicesInOtherVersions() throws Exception {
public void testDoesNotIgnoreIndicesInOtherVersions() throws Exception {
internalCluster().startNode();
// Will be deleted
createTimestampedIndex(now().minusDays(10));
createIndex(".marvel-es-data", now().minusYears(1));
createIndex(".marvel-es-data-2", now().minusYears(1));
createIndex(".monitoring-data-2", now().minusDays(10));
createAlertsIndex(now().minusYears(1), MonitoringTemplateUtils.OLD_TEMPLATE_VERSION);
createIndex(".marvel-es-2016.05.03");
createIndex(".marvel-es-2-2016.05.03");
createTimestampedIndex(now().minusDays(10), "0");
createTimestampedIndex(now().minusDays(10), "1");
createTimestampedIndex(now().minusYears(1), MonitoringTemplateUtils.OLD_TEMPLATE_VERSION);
// In the past, this index would not be deleted, but starting in 6.x the monitoring cluster
// will be required to be a newer template version than the production cluster, so the index
// pushed to it will never be "unknown" in terms of their version (relates to the
// _xpack/monitoring/_setup API)
createTimestampedIndex(now().minusDays(10), String.valueOf(Integer.MAX_VALUE));
// Won't be deleted
createIndex(MonitoringSettings.LEGACY_DATA_INDEX_NAME, now().minusYears(1));
createDataIndex(now().minusDays(10));
assertIndicesCount(3);
createAlertsIndex(now().minusYears(1));
assertIndicesCount(12);
CleanerService.Listener listener = getListener();
listener.onCleanUpIndices(days(0));
assertIndicesCount(2);
assertIndicesCount(1);
}
public void testIgnoreCurrentTimestampedIndex() throws Exception {
@ -92,6 +107,7 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
// Won't be deleted
createTimestampedIndex(now());
assertIndicesCount(2);
CleanerService.Listener listener = getListener();
@ -99,23 +115,6 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
assertIndicesCount(1);
}
public void testIgnoreTimestampedIndicesInOtherVersions() throws Exception {
internalCluster().startNode();
// Will be deleted
createTimestampedIndex(now().minusDays(10));
// Won't be deleted
createTimestampedIndex(now().minusDays(10), "0");
createTimestampedIndex(now().minusDays(10), "1");
createTimestampedIndex(now().minusDays(10), String.valueOf(Integer.MAX_VALUE));
assertIndicesCount(4);
CleanerService.Listener listener = getListener();
listener.onCleanUpIndices(days(0));
assertIndicesCount(3);
}
public void testDeleteIndices() throws Exception {
internalCluster().startNode();
@ -183,10 +182,17 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
}
/**
* Creates a monitoring data index from an earlier version (from when we used to have them).
* Creates a monitoring alerts index from the current version.
*/
protected void createDataIndex(DateTime creationDate) {
createIndex(".monitoring-data-2", creationDate);
protected void createAlertsIndex(final DateTime creationDate) {
createAlertsIndex(creationDate, MonitoringTemplateUtils.TEMPLATE_VERSION);
}
/**
* Creates a monitoring alerts index from the current version.
*/
protected void createAlertsIndex(final DateTime creationDate, final String version) {
createIndex(".monitoring-alerts-" + version, creationDate);
}
/**