honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x][ML] Add ILM policy to ml stats indices (#53349) (#53392) 

Adds a size based ILM policy to automatically
rollover ml stats indices.

Backport of #53349
This commit is contained in:
Dimitris Athanasiou 2020-03-11 13:01:34 +02:00 committed by GitHub
parent 0fd0516d0d
commit cc7751eb16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 60 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_ilm_policy.json → x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/size_based_ilm_policy.json
View File

4
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_template.json
View File

@ -10,6 +10,8 @@
"auto_expand_replicas" : "0-1",
"hidden": true
}
${xpack.ml.index.lifecycle.settings}
},
"mappings" : ${xpack.ml.stats.mappings}
"mappings" : ${xpack.ml.stats.mappings},
"aliases" : {}
}

23
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java
View File

@ -56,11 +56,12 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
ROOT_RESOURCE_PATH + "inference_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate();
private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate(true);
private static final IndexTemplateConfig STATS_TEMPLATE_NO_ILM = statsTemplate(false);
private static final String ML_STATE_ILM_POLICY_NAME = "ml-state-ilm-policy";
private static final LifecyclePolicyConfig ML_STATE_ILM_POLICY =
new LifecyclePolicyConfig(ML_STATE_ILM_POLICY_NAME, ANOMALY_DETECTION_PATH + "state_index_ilm_policy.json");
private static final String ML_SIZE_BASED_ILM_POLICY_NAME = "ml-size-based-ilm-policy";
private static final LifecyclePolicyConfig ML_SIZE_BASED_ILM_POLICY =
new LifecyclePolicyConfig(ML_SIZE_BASED_ILM_POLICY_NAME, ROOT_RESOURCE_PATH + "size_based_ilm_policy.json");
private static IndexTemplateConfig configTemplate() {
Map<String, String> variables = new HashMap<>();
@ -81,7 +82,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
variables.put(
"xpack.ml.index.lifecycle.settings",
ilmEnabled
? ",\"index.lifecycle.name\": \"" + ML_STATE_ILM_POLICY_NAME + "\"\n" +
? ",\"index.lifecycle.name\": \"" + ML_SIZE_BASED_ILM_POLICY_NAME + "\"\n" +
",\"index.lifecycle.rollover_alias\": \"" + AnomalyDetectorsIndex.jobStateIndexWriteAlias() + "\"\n"
: "");
@ -102,10 +103,16 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
variables);
}
private static IndexTemplateConfig statsTemplate() {
private static IndexTemplateConfig statsTemplate(boolean ilmEnabled) {
Map<String, String> variables = new HashMap<>();
variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
variables.put("xpack.ml.stats.mappings", MlStatsIndex.mapping());
variables.put(
"xpack.ml.index.lifecycle.settings",
ilmEnabled
? ",\"index.lifecycle.name\": \"" + ML_SIZE_BASED_ILM_POLICY_NAME + "\"\n" +
",\"index.lifecycle.rollover_alias\": \"" + MlStatsIndex.writeAlias() + "\"\n"
: "");
return new IndexTemplateConfig(MlStatsIndex.TEMPLATE_NAME,
ROOT_RESOURCE_PATH + "stats_index_template.json",
@ -126,7 +133,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
INFERENCE_TEMPLATE,
META_TEMPLATE,
NOTIFICATIONS_TEMPLATE,
STATS_TEMPLATE);
ilmEnabled ? STATS_TEMPLATE : STATS_TEMPLATE_NO_ILM);
}
@Override
@ -141,7 +148,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
@Override
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
return Collections.singletonList(ML_STATE_ILM_POLICY);
return Collections.singletonList(ML_SIZE_BASED_ILM_POLICY);
}
@Override

43
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.XPackSettings;
import org.elasticsearch.xpack.core.ilm.LifecycleAction;
import org.elasticsearch.xpack.core.ilm.RolloverAction;
import org.elasticsearch.xpack.core.ml.MlStatsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.junit.Before;
import org.mockito.ArgumentCaptor;
@ -98,7 +99,7 @@ public class MlIndexTemplateRegistryTests extends ESTestCase {
.filter(r -> r.name().equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX))
.findFirst()
.orElseThrow(() -> new AssertionError("expected the ml state index template to be put"));
assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-state-ilm-policy"));
assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy"));
assertThat(req.settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-state-write"));
}
@ -122,6 +123,46 @@ public class MlIndexTemplateRegistryTests extends ESTestCase {
assertThat(req.settings().get("index.lifecycle.rollover_alias"), is(nullValue()));
}
public void testStatsTemplateWithIlm() {
MlIndexTemplateRegistry registry =
new MlIndexTemplateRegistry(
Settings.builder()
.put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true)
.build(),
clusterService, threadPool, client, xContentRegistry);
registry.clusterChanged(createClusterChangedEvent(nodes));
verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
.filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME))
.findFirst()
.orElseThrow(() -> new AssertionError("expected the ml stats index template to be put"));
assertThat(req.settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy"));
assertThat(req.settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-stats-write"));
}
public void testStatsTemplateWithNoIlm() {
MlIndexTemplateRegistry registry =
new MlIndexTemplateRegistry(
Settings.builder()
.put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false)
.build(),
clusterService, threadPool, client, xContentRegistry);
registry.clusterChanged(createClusterChangedEvent(nodes));
verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
.filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME))
.findFirst()
.orElseThrow(() -> new AssertionError("expected the ml stats index template to be put"));
assertThat(req.settings().get("index.lifecycle.name"), is(nullValue()));
assertThat(req.settings().get("index.lifecycle.rollover_alias"), is(nullValue()));
}
@SuppressWarnings("unchecked")
private static <Response> Answer<Response> withResponse(Response response) {
return invocationOnMock -> {