honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ILM] Add date setting to calculate index age (#46561) (#46697) 

* [ILM] Add date setting to calculate index age

Add the `index.lifecycle.origination_date` to allow users to configure a
custom date that'll be used to calculate the index age for the phase
transmissions (as opposed to the default index creation date).

This could be useful for users to create an index with an "older"
origination date when indexing old data.

Relates to #42449.

* [ILM] Don't override creation date on policy init

The initial approach we took was to override the lifecycle creation date
if the `index.lifecycle.origination_date` setting was set. This had the
disadvantage of the user not being able to update the `origination_date`
anymore once set.

This commit changes the way we makes use of the
`index.lifecycle.origination_date` setting by checking its value when
we calculate the index age (ie. at "read time") and, in case it's not
set, default to the index creation date.

* Make origination date setting index scope dynamic

* Document orignation date setting in ilm settings

(cherry picked from commit d5bd2bb77ee28c1978ab6679f941d7c02e389d32)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>
This commit is contained in:
Andrei Dan 2019-09-16 08:50:28 +01:00 committed by GitHub
parent c99adfb99a
commit c57cca98b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 82 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/reference/ilm/policy-definitions.asciidoc
View File

@ -60,10 +60,12 @@ phase after one day. Until then, the index is in a waiting state. After
moving into the warm phase, it will wait until 30 days have elapsed before
moving to the delete phase and deleting the index.
`min_age` is usually the time elapsed from the time the index is created. If the
index is rolled over, then `min_age` is the time elapsed from the time the index
is rolled over. The intention here is to execute following phases and actions
relative to when data was written last to a rolled over index.
`min_age` is usually the time elapsed from the time the index is created, unless
the `index.lifecycle.origination_date` index setting is configured, in which
case the `min_age` will be the time elapsed since that specified date. If the
index is rolled over, then `min_age` is the time elapsed from the time the
index is rolled over. The intention here is to execute following phases and
actions relative to when data was written last to a rolled over index.
The previous phase's actions must complete before {ilm} will check `min_age` and
transition into the next phase. By default, {ilm} checks for indices that meet

5
docs/reference/settings/ilm-settings.asciidoc
View File

@ -17,3 +17,8 @@ information about rollover, see <<using-policies-rollover>>.
`indices.lifecycle.poll_interval`::
(<<time-units, time units>>) How often {ilm} checks for indices that meet policy
criteria. Defaults to `10m`.
`index.lifecycle.origination_date`::
The timestamp that will be used to calculate the index age for its phase
transitions. This allows the users to create an index containing old data and
use the original creation date of the old data to calculate the index age.

12
server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View File

@ -283,7 +283,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, ToXContentFragmen
private final long mappingVersion;
private final long settingsVersion;
private final long aliasesVersion;
private final long[] primaryTerms;
@ -1090,25 +1090,25 @@ public class IndexMetaData implements Diffable<IndexMetaData>, ToXContentFragmen
this.mappingVersion = mappingVersion;
return this;
}
public long settingsVersion() {
return settingsVersion;
}
public Builder settingsVersion(final long settingsVersion) {
this.settingsVersion = settingsVersion;
return this;
}
public long aliasesVersion() {
return aliasesVersion;
}
public Builder aliasesVersion(final long aliasesVersion) {
this.aliasesVersion = aliasesVersion;
return this;
}
/**
* returns the primary term for the given shard.
* See {@link IndexMetaData#primaryTerm(int)} for more information.

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java
View File

@ -22,7 +22,7 @@ public final class InitializePolicyContextStep extends ClusterStateActionStep {
public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);
public InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
super(key, nextStepKey);
}

4
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java
View File

@ -17,6 +17,8 @@ public class LifecycleSettings {
public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval";
public static final String LIFECYCLE_NAME = "index.lifecycle.name";
public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete";
public static final String LIFECYCLE_ORIGINATION_DATE = "index.lifecycle.origination_date";
public static final String SLM_HISTORY_INDEX_ENABLED = "slm.history_index_enabled";
public static final String SLM_RETENTION_SCHEDULE = "slm.retention_schedule";
public static final String SLM_RETENTION_DURATION = "slm.retention_duration";
@ -28,6 +30,8 @@ public class LifecycleSettings {
Setting.Property.Dynamic, Setting.Property.IndexScope);
public static final Setting<Boolean> LIFECYCLE_INDEXING_COMPLETE_SETTING = Setting.boolSetting(LIFECYCLE_INDEXING_COMPLETE, false,
Setting.Property.Dynamic, Setting.Property.IndexScope);
public static final Setting<Long> LIFECYCLE_ORIGINATION_DATE_SETTING =
Setting.longSetting(LIFECYCLE_ORIGINATION_DATE, -1, -1, Setting.Property.Dynamic, Setting.Property.IndexScope);
public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(SLM_HISTORY_INDEX_ENABLED, true,
Setting.Property.NodeScope);

1
x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java
View File

@ -153,6 +153,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin {
return Arrays.asList(
LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING,
LifecycleSettings.LIFECYCLE_NAME_SETTING,
LifecycleSettings.LIFECYCLE_ORIGINATION_DATE_SETTING,
LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING,
RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING,
LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING,

8
x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java
View File

@ -50,6 +50,7 @@ import java.util.function.LongSupplier;
import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE;
import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_ORIGINATION_DATE;
public class IndexLifecycleRunner {
private static final Logger logger = LogManager.getLogger(IndexLifecycleRunner.class);
@ -73,11 +74,12 @@ public class IndexLifecycleRunner {
*/
boolean isReadyToTransitionToThisPhase(final String policy, final IndexMetaData indexMetaData, final String phase) {
LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
if (lifecycleState.getLifecycleDate() == null) {
logger.trace("no index creation date has been set yet");
Long originationDate = indexMetaData.getSettings().getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L);
if (lifecycleState.getLifecycleDate() == null && originationDate == -1L) {
logger.trace("no index creation or origination date has been set yet");
return true;
}
final Long lifecycleDate = lifecycleState.getLifecycleDate();
final Long lifecycleDate = originationDate != -1L ? originationDate : lifecycleState.getLifecycleDate();
assert lifecycleDate != null && lifecycleDate >= 0 : "expected index to have a lifecycle date but it did not";
final TimeValue after = stepRegistry.getIndexAgeForPhase(policy, phase);
final long now = nowSupplier.getAsLong();

5
x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
View File

@ -41,6 +41,8 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_ORIGINATION_DATE;
public class TransportExplainLifecycleAction
extends TransportClusterInfoAction<ExplainLifecycleRequest, ExplainLifecycleResponse> {
@ -107,8 +109,9 @@ public class TransportExplainLifecycleAction
// If this is requesting only errors, only include indices in the error step or which are using a nonexistent policy
if (request.onlyErrors() == false
|| (ErrorStep.NAME.equals(lifecycleState.getStep()) || indexLifecycleService.policyExists(policyName) == false)) {
Long originationDate = idxSettings.getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L);
indexResponse = IndexLifecycleExplainResponse.newManagedIndexResponse(index, policyName,
lifecycleState.getLifecycleDate(),
originationDate != -1L ? originationDate : lifecycleState.getLifecycleDate(),
lifecycleState.getPhase(),
lifecycleState.getAction(),
lifecycleState.getStep(),

47
x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
View File

@ -59,6 +59,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import static org.elasticsearch.client.Requests.clusterHealthRequest;
@ -204,7 +205,7 @@ public class IndexLifecycleInitialisationTests extends ESIntegTestCase {
public void testExplainExecution() throws Exception {
// start node
logger.info("Starting server1");
final String server_1 = internalCluster().startNode();
internalCluster().startNode();
logger.info("Creating lifecycle [test_lifecycle]");
PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
PutLifecycleAction.Response putLifecycleResponse = client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get();
@ -223,15 +224,39 @@ public class IndexLifecycleInitialisationTests extends ESIntegTestCase {
.actionGet();
assertAcked(createIndexResponse);
// using AtomicLong only to extract a value from a lambda rather than the more traditional atomic update use-case
AtomicLong originalLifecycleDate = new AtomicLong();
{
PhaseExecutionInfo expectedExecutionInfo = new PhaseExecutionInfo(lifecyclePolicy.getName(), mockPhase, 1L, actualModifiedDate);
assertBusy(() -> {
ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
IndexLifecycleExplainResponse indexResponse = explainResponse.getIndexResponses().get("test");
IndexLifecycleExplainResponse indexResponse = executeExplainRequestAndGetTestIndexResponse();
assertThat(indexResponse.getStep(), equalTo("observable_cluster_state_action"));
assertThat(indexResponse.getPhaseExecutionInfo(), equalTo(expectedExecutionInfo));
originalLifecycleDate.set(indexResponse.getLifecycleDate());
});
}
// set the origination date setting to an older value
client().admin().indices().prepareUpdateSettings("test")
.setSettings(Collections.singletonMap(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, 1000L)).get();
{
assertBusy(() -> {
IndexLifecycleExplainResponse indexResponse = executeExplainRequestAndGetTestIndexResponse();
assertThat("The configured origination date dictates the lifecycle date",
indexResponse.getLifecycleDate(), equalTo(1000L));
});
}
// set the origination date setting to null
client().admin().indices().prepareUpdateSettings("test")
.setSettings(Collections.singletonMap(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, null)).get();
{
assertBusy(() -> {
IndexLifecycleExplainResponse indexResponse = executeExplainRequestAndGetTestIndexResponse();
assertThat("Without the origination date, the index create date should dictate the lifecycle date",
indexResponse.getLifecycleDate(), equalTo(originalLifecycleDate.get()));
});
}
@ -242,10 +267,7 @@ public class IndexLifecycleInitialisationTests extends ESIntegTestCase {
{
PhaseExecutionInfo expectedExecutionInfo = new PhaseExecutionInfo(lifecyclePolicy.getName(), null, 1L, actualModifiedDate);
assertBusy(() -> {
ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
IndexLifecycleExplainResponse indexResponse = explainResponse.getIndexResponses().get("test");
IndexLifecycleExplainResponse indexResponse = executeExplainRequestAndGetTestIndexResponse();
assertThat(indexResponse.getPhase(), equalTo(TerminalPolicyStep.COMPLETED_PHASE));
assertThat(indexResponse.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
assertThat(indexResponse.getPhaseExecutionInfo(), equalTo(expectedExecutionInfo));
@ -253,6 +275,13 @@ public class IndexLifecycleInitialisationTests extends ESIntegTestCase {
}
}
private IndexLifecycleExplainResponse executeExplainRequestAndGetTestIndexResponse() throws ExecutionException, InterruptedException {
ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
return explainResponse.getIndexResponses().get("test");
}
public void testMasterDedicatedDataDedicated() throws Exception {
settings = Settings.builder().put(settings).put("index.lifecycle.test.complete", true).build();
// start master node

12
x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java
View File

@ -1213,6 +1213,18 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
now.set(Long.MAX_VALUE);
assertTrue("index should be able to transition past phase's age",
runner.isReadyToTransitionToThisPhase(policyName, indexMetaData, "phase"));
// Come back to the "present"
now.set(5L);
indexMetaData = IndexMetaData.builder(indexMetaData)
.settings(Settings.builder()
.put(indexMetaData.getSettings())
.put(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, 3L)
.build())
.putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
.build();
assertTrue("index should be able to transition due to the origination date indicating it's old enough",
runner.isReadyToTransitionToThisPhase(policyName, indexMetaData, "phase"));
}