From cc7751eb161d9894a0a0765b59c657fb2c69f200 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 11 Mar 2020 13:01:34 +0200 Subject: [PATCH] [7.x][ML] Add ILM policy to ml stats indices (#53349) (#53392) Adds a size based ILM policy to automatically rollover ml stats indices. Backport of #53349 --- ...policy.json => size_based_ilm_policy.json} | 0 .../xpack/core/ml/stats_index_template.json | 4 +- .../xpack/ml/MlIndexTemplateRegistry.java | 23 ++++++---- .../ml/MlIndexTemplateRegistryTests.java | 43 ++++++++++++++++++- 4 files changed, 60 insertions(+), 10 deletions(-) rename x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/{anomalydetection/state_index_ilm_policy.json => size_based_ilm_policy.json} (100%) diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_ilm_policy.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_ilm_policy.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json index 1c694d9d1a7..223d1a79bad 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json @@ -10,6 +10,8 @@ "auto_expand_replicas" : "0-1", "hidden": true } + ${xpack.ml.index.lifecycle.settings} }, - "mappings" : ${xpack.ml.stats.mappings} + "mappings" : ${xpack.ml.stats.mappings}, + "aliases" : {} } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java index 6c205f23bb2..83c0050035d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java @@ -56,11 +56,12 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { ROOT_RESOURCE_PATH + "inference_index_template.json", Version.CURRENT.id, VERSION_PATTERN, Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id))); - private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate(); + private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate(true); + private static final IndexTemplateConfig STATS_TEMPLATE_NO_ILM = statsTemplate(false); - private static final String ML_STATE_ILM_POLICY_NAME = "ml-state-ilm-policy"; - private static final LifecyclePolicyConfig ML_STATE_ILM_POLICY = - new LifecyclePolicyConfig(ML_STATE_ILM_POLICY_NAME, ANOMALY_DETECTION_PATH + "state_index_ilm_policy.json"); + private static final String ML_SIZE_BASED_ILM_POLICY_NAME = "ml-size-based-ilm-policy"; + private static final LifecyclePolicyConfig ML_SIZE_BASED_ILM_POLICY = + new LifecyclePolicyConfig(ML_SIZE_BASED_ILM_POLICY_NAME, ROOT_RESOURCE_PATH + "size_based_ilm_policy.json"); private static IndexTemplateConfig configTemplate() { Map variables = new HashMap<>(); @@ -81,7 +82,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { variables.put( "xpack.ml.index.lifecycle.settings", ilmEnabled - ? ",\"index.lifecycle.name\": \"" + ML_STATE_ILM_POLICY_NAME + "\"\n" + + ? ",\"index.lifecycle.name\": \"" + ML_SIZE_BASED_ILM_POLICY_NAME + "\"\n" + ",\"index.lifecycle.rollover_alias\": \"" + AnomalyDetectorsIndex.jobStateIndexWriteAlias() + "\"\n" : ""); @@ -102,10 +103,16 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { variables); } - private static IndexTemplateConfig statsTemplate() { + private static IndexTemplateConfig statsTemplate(boolean ilmEnabled) { Map variables = new HashMap<>(); variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); variables.put("xpack.ml.stats.mappings", MlStatsIndex.mapping()); + variables.put( + "xpack.ml.index.lifecycle.settings", + ilmEnabled + ? ",\"index.lifecycle.name\": \"" + ML_SIZE_BASED_ILM_POLICY_NAME + "\"\n" + + ",\"index.lifecycle.rollover_alias\": \"" + MlStatsIndex.writeAlias() + "\"\n" + : ""); return new IndexTemplateConfig(MlStatsIndex.TEMPLATE_NAME, ROOT_RESOURCE_PATH + "stats_index_template.json", @@ -126,7 +133,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { INFERENCE_TEMPLATE, META_TEMPLATE, NOTIFICATIONS_TEMPLATE, - STATS_TEMPLATE); + ilmEnabled ? STATS_TEMPLATE : STATS_TEMPLATE_NO_ILM); } @Override @@ -141,7 +148,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { @Override protected List getPolicyConfigs() { - return Collections.singletonList(ML_STATE_ILM_POLICY); + return Collections.singletonList(ML_SIZE_BASED_ILM_POLICY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java index a4bffadd429..d339c54815a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ml.MlStatsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -98,7 +99,7 @@ public class MlIndexTemplateRegistryTests extends ESTestCase { .filter(r -> r.name().equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX)) .findFirst() .orElseThrow(() -> new AssertionError("expected the ml state index template to be put")); - assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-state-ilm-policy")); + assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy")); assertThat(req.settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-state-write")); } @@ -122,6 +123,46 @@ public class MlIndexTemplateRegistryTests extends ESTestCase { assertThat(req.settings().get("index.lifecycle.rollover_alias"), is(nullValue())); } + public void testStatsTemplateWithIlm() { + MlIndexTemplateRegistry registry = + new MlIndexTemplateRegistry( + Settings.builder() + .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true) + .build(), + clusterService, threadPool, client, xContentRegistry); + + registry.clusterChanged(createClusterChangedEvent(nodes)); + + verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject()); + + PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream() + .filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME)) + .findFirst() + .orElseThrow(() -> new AssertionError("expected the ml stats index template to be put")); + assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy")); + assertThat(req.settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-stats-write")); + } + + public void testStatsTemplateWithNoIlm() { + MlIndexTemplateRegistry registry = + new MlIndexTemplateRegistry( + Settings.builder() + .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false) + .build(), + clusterService, threadPool, client, xContentRegistry); + + registry.clusterChanged(createClusterChangedEvent(nodes)); + + verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject()); + + PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream() + .filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME)) + .findFirst() + .orElseThrow(() -> new AssertionError("expected the ml stats index template to be put")); + assertThat(req.settings().get("index.lifecycle.name"), is(nullValue())); + assertThat(req.settings().get("index.lifecycle.rollover_alias"), is(nullValue())); + } + @SuppressWarnings("unchecked") private static Answer withResponse(Response response) { return invocationOnMock -> {