Use custom index metadata for ILM state (#33783)

Using index settings for ILM state is fragile and exposes too much
information that doesn't need to be exposed. Using custom index metadata
is more resilient and allows more controlled access to internal
information.

As part of these changes, moves away from using defaults for ILM-related
values, in favor of using null values to clearly indicate that the value is not
present.

client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java

@@ -26,9 +26,12 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.indexlifecycle.AllocateAction;
 import org.elasticsearch.client.indexlifecycle.DeleteAction;
 import org.elasticsearch.client.indexlifecycle.DeleteLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.client.indexlifecycle.ExplainLifecycleResponse;
 import org.elasticsearch.client.indexlifecycle.ForceMergeAction;
 import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyRequest;
 import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyResponse;
+import org.elasticsearch.client.indexlifecycle.IndexLifecycleExplainResponse;
 import org.elasticsearch.client.indexlifecycle.LifecycleAction;
 import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusRequest;
 import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusResponse;
@@ -39,16 +42,13 @@ import org.elasticsearch.client.indexlifecycle.Phase;
 import org.elasticsearch.client.indexlifecycle.PhaseExecutionInfo;
 import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
 import org.elasticsearch.client.indexlifecycle.RolloverAction;
-import org.elasticsearch.client.indexlifecycle.ShrinkAction;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
-import org.elasticsearch.client.indexlifecycle.ExplainLifecycleResponse;
-import org.elasticsearch.client.indexlifecycle.IndexLifecycleExplainResponse;
 import org.elasticsearch.client.indexlifecycle.SetIndexLifecyclePolicyRequest;
 import org.elasticsearch.client.indexlifecycle.SetIndexLifecyclePolicyResponse;
+import org.elasticsearch.client.indexlifecycle.ShrinkAction;
 import org.elasticsearch.client.indexlifecycle.StartILMRequest;
 import org.elasticsearch.client.indexlifecycle.StopILMRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -167,14 +167,6 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
             .put("index.lifecycle.rollover_alias", "baz-alias").build(), "", "\"baz-alias\" : {}");
 
         createIndex("squash", Settings.EMPTY);
-        assertBusy(() -> {
-            GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("foo-01", "baz-01");
-            GetSettingsResponse settingsResponse = highLevelClient().indices().getSettings(getSettingsRequest, RequestOptions.DEFAULT);
-            assertThat(settingsResponse.getSetting("foo-01", "index.lifecycle.name"), equalTo(policy.getName()));
-            assertThat(settingsResponse.getSetting("baz-01", "index.lifecycle.name"), equalTo(policy.getName()));
-            assertThat(settingsResponse.getSetting("foo-01", "index.lifecycle.phase"), equalTo("hot"));
-            assertThat(settingsResponse.getSetting("baz-01", "index.lifecycle.phase"), equalTo("hot"));
-        });
 
         ExplainLifecycleRequest req = new ExplainLifecycleRequest();
         req.indices("foo-01", "baz-01", "squash");

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -955,6 +955,11 @@ public class IndexMetaData implements Diffable<IndexMetaData>, ToXContentFragmen
             return this;
         }
 
+        public Builder removeCustom(String type) {
+            this.customMetaData.remove(type);
+            return this;
+        }
+
         public Set<String> getInSyncAllocationIds(int shardId) {
             return inSyncAllocationIds.get(shardId);
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStep.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.index.Index;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+
+/**
+ * Copies the execution state data from one index to another, typically after a
+ * new index has been created. Useful for actions such as shrink.
+ */
+public class CopyExecutionStateStep extends ClusterStateActionStep {
+    public static final String NAME = "copy_execution_state";
+    private String shrunkIndexPrefix;
+
+    public CopyExecutionStateStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {
+        super(key, nextStepKey);
+        this.shrunkIndexPrefix = shrunkIndexPrefix;
+    }
+
+    String getShrunkIndexPrefix() {
+        return shrunkIndexPrefix;
+    }
+
+    @Override
+    public ClusterState performAction(Index index, ClusterState clusterState) {
+        IndexMetaData indexMetaData = clusterState.metaData().index(index);
+        // get source index
+        String indexName = indexMetaData.getIndex().getName();
+        // get target shrink index
+        String targetIndexName = shrunkIndexPrefix + indexName;
+        IndexMetaData targetIndexMetaData = clusterState.metaData().index(targetIndexName);
+
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+        String phase = lifecycleState.getPhase();
+        String action = lifecycleState.getAction();
+        long lifecycleDate = lifecycleState.getLifecycleDate();
+
+        LifecycleExecutionState.Builder relevantTargetCustomData = LifecycleExecutionState.builder();
+        relevantTargetCustomData.setIndexCreationDate(lifecycleDate);
+        relevantTargetCustomData.setPhase(phase);
+        relevantTargetCustomData.setAction(action);
+        relevantTargetCustomData.setStep(ShrunkenIndexCheckStep.NAME);
+
+        MetaData.Builder newMetaData = MetaData.builder(clusterState.getMetaData())
+            .put(IndexMetaData.builder(targetIndexMetaData)
+                .putCustom(ILM_CUSTOM_METADATA_KEY, relevantTargetCustomData.build().asMap()));
+
+        return ClusterState.builder(clusterState).metaData(newMetaData).build();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        CopyExecutionStateStep that = (CopyExecutionStateStep) o;
+        return Objects.equals(shrunkIndexPrefix, that.shrunkIndexPrefix);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), shrunkIndexPrefix);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleExplainResponse.java

@@ -48,14 +48,14 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
                     (boolean) a[1],
                     (String) a[2],
                     (boolean) (a[3] == null ? false: a[3]),
-                    (long) (a[4] == null ? -1L: a[4]),
+                    (Long) (a[4]),
                     (String) a[5],
                     (String) a[6],
                     (String) a[7],
                     (String) a[8],
-                    (long) (a[9] == null ? -1L: a[9]),
+                    (Long) (a[9]),
-                    (long) (a[10] == null ? -1L: a[10]),
+                    (Long) (a[10]),
-                    (long) (a[11] == null ? -1L: a[11]),
+                    (Long) (a[11]),
                     (BytesReference) a[12],
                     (PhaseExecutionInfo) a[13]));
     static {
@@ -86,36 +86,36 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     private final String action;
     private final String step;
     private final String failedStep;
-    private final long lifecycleDate;
+    private final Long lifecycleDate;
-    private final long phaseTime;
+    private final Long phaseTime;
-    private final long actionTime;
+    private final Long actionTime;
-    private final long stepTime;
+    private final Long stepTime;
     private final boolean skip;
     private final boolean managedByILM;
     private final BytesReference stepInfo;
     private final PhaseExecutionInfo phaseExecutionInfo;
 
-    public static IndexLifecycleExplainResponse newManagedIndexResponse(String index, String policyName, boolean skip, long lifecycleDate,
+    public static IndexLifecycleExplainResponse newManagedIndexResponse(String index, String policyName, boolean skip, Long lifecycleDate,
-            String phase, String action, String step, String failedStep, long phaseTime, long actionTime, long stepTime,
+            String phase, String action, String step, String failedStep, Long phaseTime, Long actionTime, Long stepTime,
             BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
         return new IndexLifecycleExplainResponse(index, true, policyName, skip, lifecycleDate, phase, action, step, failedStep, phaseTime,
                 actionTime, stepTime, stepInfo, phaseExecutionInfo);
     }
 
     public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String index) {
-        return new IndexLifecycleExplainResponse(index, false, null, false, -1L, null, null, null, null, -1L, -1L, -1L, null, null);
+        return new IndexLifecycleExplainResponse(index, false, null, false, null, null, null, null, null, null, null, null, null, null);
     }
 
-    private IndexLifecycleExplainResponse(String index, boolean managedByILM, String policyName, boolean skip, long lifecycleDate,
+    private IndexLifecycleExplainResponse(String index, boolean managedByILM, String policyName, boolean skip, Long lifecycleDate,
-                                          String phase, String action, String step, String failedStep, long phaseTime, long actionTime,
+                                          String phase, String action, String step, String failedStep, Long phaseTime, Long actionTime,
-                                          long stepTime, BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
+                                          Long stepTime, BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
         if (managedByILM) {
             if (policyName == null) {
                 throw new IllegalArgumentException("[" + POLICY_NAME_FIELD.getPreferredName() + "] cannot be null for managed index");
             }
         } else {
-            if (policyName != null || lifecycleDate >= 0 || phase != null || action != null || step != null || failedStep != null
+            if (policyName != null || lifecycleDate != null || phase != null || action != null || step != null || failedStep != null
-                    || phaseTime >= 0 || actionTime >= 0 || stepTime >= 0 || stepInfo != null || phaseExecutionInfo != null) {
+                    || phaseTime != null || actionTime != null || stepTime != null || stepInfo != null || phaseExecutionInfo != null) {
                 throw new IllegalArgumentException(
                         "Unmanaged index response must only contain fields: [" + MANAGED_BY_ILM_FIELD + ", " + INDEX_FIELD + "]");
             }
@@ -142,27 +142,27 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         if (managedByILM) {
             policyName = in.readString();
             skip = in.readBoolean();
-            lifecycleDate = in.readZLong();
+            lifecycleDate = in.readOptionalLong();
-            phase = in.readString();
+            phase = in.readOptionalString();
-            action = in.readString();
+            action = in.readOptionalString();
-            step = in.readString();
+            step = in.readOptionalString();
             failedStep = in.readOptionalString();
-            phaseTime = in.readZLong();
+            phaseTime = in.readOptionalLong();
-            actionTime = in.readZLong();
+            actionTime = in.readOptionalLong();
-            stepTime = in.readZLong();
+            stepTime = in.readOptionalLong();
             stepInfo = in.readOptionalBytesReference();
             phaseExecutionInfo = in.readOptionalWriteable(PhaseExecutionInfo::new);
         } else {
             policyName = null;
             skip = false;
-            lifecycleDate = -1L;
+            lifecycleDate = null;
             phase = null;
             action = null;
             step = null;
             failedStep = null;
-            phaseTime = -1L;
+            phaseTime = null;
-            actionTime = -1L;
+            actionTime = null;
-            stepTime = -1L;
+            stepTime = null;
             stepInfo = null;
             phaseExecutionInfo = null;
         }
@@ -175,14 +175,14 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         if (managedByILM) {
             out.writeString(policyName);
             out.writeBoolean(skip);
-            out.writeZLong(lifecycleDate);
+            out.writeOptionalLong(lifecycleDate);
-            out.writeString(phase);
+            out.writeOptionalString(phase);
-            out.writeString(action);
+            out.writeOptionalString(action);
-            out.writeString(step);
+            out.writeOptionalString(step);
             out.writeOptionalString(failedStep);
-            out.writeZLong(phaseTime);
+            out.writeOptionalLong(phaseTime);
-            out.writeZLong(actionTime);
+            out.writeOptionalLong(actionTime);
-            out.writeZLong(stepTime);
+            out.writeOptionalLong(stepTime);
             out.writeOptionalBytesReference(stepInfo);
             out.writeOptionalWriteable(phaseExecutionInfo);
         }
@@ -204,7 +204,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         return skip;
     }
 
-    public long getLifecycleDate() {
+    public Long getLifecycleDate() {
         return lifecycleDate;
     }
 
@@ -212,7 +212,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         return phase;
     }
 
-    public long getPhaseTime() {
+    public Long getPhaseTime() {
         return phaseTime;
     }
 
@@ -220,7 +220,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         return action;
     }
 
-    public long getActionTime() {
+    public Long getActionTime() {
         return actionTime;
     }
 
@@ -228,7 +228,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         return step;
     }
 
-    public long getStepTime() {
+    public Long getStepTime() {
         return stepTime;
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java

@@ -10,9 +10,10 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+
 public final class InitializePolicyContextStep extends ClusterStateActionStep {
     public static final String INITIALIZATION_PHASE = "new";
     public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
@@ -30,18 +31,19 @@ public final class InitializePolicyContextStep extends ClusterStateActionStep {
             // Index must have been since deleted, ignore it
             return clusterState;
         }
-        Settings settings = indexMetaData.getSettings();
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState
-        if (settings.hasValue(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE)) {
+            .fromIndexMetadata(indexMetaData);
+        if (lifecycleState.getLifecycleDate() != null) {
             return clusterState;
         }
 
         ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
-        IndexMetaData idxMeta = clusterState.getMetaData().index(index);
+
-        Settings.Builder indexSettings = Settings.builder().put(idxMeta.getSettings())
+        LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
-            .put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, idxMeta.getCreationDate());
+        newCustomData.setIndexCreationDate(indexMetaData.getCreationDate());
         newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData()).put(IndexMetaData
-            .builder(clusterState.getMetaData().index(index))
+            .builder(indexMetaData)
-            .settings(indexSettings)));
+            .putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())));
         return newClusterStateBuilder.build();
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleExecutionState.java

@@ -0,0 +1,319 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Contains information about the execution of a lifecycle policy for a single
+ * index, and serializes/deserializes this information to and from custom
+ * index metadata.
+ */
+public class LifecycleExecutionState {
+    public static final String ILM_CUSTOM_METADATA_KEY = "ilm";
+
+    private static final String PHASE = "phase";
+    private static final String ACTION = "action";
+    private static final String STEP = "step";
+    private static final String INDEX_CREATION_DATE = "creation_date";
+    private static final String PHASE_TIME = "phase_time";
+    private static final String ACTION_TIME = "action_time";
+    private static final String STEP_TIME = "step_time";
+    private static final String FAILED_STEP = "failed_step";
+    private static final String STEP_INFO = "step_info";
+    private static final String PHASE_DEFINITION = "phase_definition";
+
+    private final String phase;
+    private final String action;
+    private final String step;
+    private final String failedStep;
+    private final String stepInfo;
+    private final String phaseDefinition;
+    private final Long lifecycleDate;
+    private final Long phaseTime;
+    private final Long actionTime;
+    private final Long stepTime;
+
+    private LifecycleExecutionState(String phase, String action, String step, String failedStep,
+                                    String stepInfo, String phaseDefinition, Long lifecycleDate,
+                                    Long phaseTime, Long actionTime, Long stepTime) {
+        this.phase = phase;
+        this.action = action;
+        this.step = step;
+        this.failedStep = failedStep;
+        this.stepInfo = stepInfo;
+        this.phaseDefinition = phaseDefinition;
+        this.lifecycleDate = lifecycleDate;
+        this.phaseTime = phaseTime;
+        this.actionTime = actionTime;
+        this.stepTime = stepTime;
+    }
+
+    /**
+     * Retrieves the execution state from an {@link IndexMetaData} based on the
+     * custom metadata.
+     * @param indexMetaData The metadata of the index to retrieve the execution
+     *                      state from.
+     * @return The execution state of that index.
+     */
+    public static LifecycleExecutionState fromIndexMetadata(IndexMetaData indexMetaData) {
+        Map<String, String> customData = indexMetaData.getCustomData(ILM_CUSTOM_METADATA_KEY);
+        customData = customData == null ? new HashMap<>() : customData;
+        return fromCustomMetadata(customData);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static Builder builder(LifecycleExecutionState state) {
+        return new Builder()
+            .setPhase(state.phase)
+            .setAction(state.action)
+            .setStep(state.step)
+            .setFailedStep(state.failedStep)
+            .setStepInfo(state.stepInfo)
+            .setPhaseDefinition(state.phaseDefinition)
+            .setIndexCreationDate(state.lifecycleDate)
+            .setPhaseTime(state.phaseTime)
+            .setActionTime(state.actionTime)
+            .setStepTime(state.stepTime);
+    }
+
+    static LifecycleExecutionState fromCustomMetadata(Map<String, String> customData) {
+        Builder builder = builder();
+        if (customData.containsKey(PHASE)) {
+            builder.setPhase(customData.get(PHASE));
+        }
+        if (customData.containsKey(ACTION)) {
+            builder.setAction(customData.get(ACTION));
+        }
+        if (customData.containsKey(STEP)) {
+            builder.setStep(customData.get(STEP));
+        }
+        if (customData.containsKey(FAILED_STEP)) {
+            builder.setFailedStep(customData.get(FAILED_STEP));
+        }
+        if (customData.containsKey(STEP_INFO)) {
+            builder.setStepInfo(customData.get(STEP_INFO));
+        }
+        if (customData.containsKey(PHASE_DEFINITION)) {
+            builder.setPhaseDefinition(customData.get(PHASE_DEFINITION));
+        }
+        if (customData.containsKey(INDEX_CREATION_DATE)) {
+            try {
+                builder.setIndexCreationDate(Long.parseLong(customData.get(INDEX_CREATION_DATE)));
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
+                    e, INDEX_CREATION_DATE, customData.get(INDEX_CREATION_DATE));
+            }
+        }
+        if (customData.containsKey(PHASE_TIME)) {
+            try {
+                builder.setPhaseTime(Long.parseLong(customData.get(PHASE_TIME)));
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
+                    e, PHASE_TIME, customData.get(PHASE_TIME));
+            }
+        }
+        if (customData.containsKey(ACTION_TIME)) {
+            try {
+                builder.setActionTime(Long.parseLong(customData.get(ACTION_TIME)));
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
+                    e, ACTION_TIME, customData.get(ACTION_TIME));
+            }
+        }
+        if (customData.containsKey(STEP_TIME)) {
+            try {
+                builder.setStepTime(Long.parseLong(customData.get(STEP_TIME)));
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
+                    e, STEP_TIME, customData.get(STEP_TIME));
+            }
+        }
+        return builder.build();
+    }
+
+    /**
+     * Converts this object to an immutable map representation for use with
+     * {@link IndexMetaData.Builder#putCustom(String, Map)}.
+     * @return An immutable Map representation of this execution state.
+     */
+    public Map<String, String> asMap() {
+        Map<String, String> result = new HashMap<>();
+        if (phase != null) {
+            result.put(PHASE, phase);
+        }
+        if (action != null) {
+            result.put(ACTION, action);
+        }
+        if (step != null) {
+            result.put(STEP, step);
+        }
+        if (failedStep != null) {
+            result.put(FAILED_STEP, failedStep);
+        }
+        if (stepInfo != null) {
+            result.put(STEP_INFO, stepInfo);
+        }
+        if (lifecycleDate != null) {
+            result.put(INDEX_CREATION_DATE, String.valueOf(lifecycleDate));
+        }
+        if (phaseTime != null) {
+            result.put(PHASE_TIME, String.valueOf(phaseTime));
+        }
+        if (actionTime != null) {
+            result.put(ACTION_TIME, String.valueOf(actionTime));
+        }
+        if (stepTime != null) {
+            result.put(STEP_TIME, String.valueOf(stepTime));
+        }
+        if (phaseDefinition != null) {
+            result.put(PHASE_DEFINITION, String.valueOf(phaseDefinition));
+        }
+        return Collections.unmodifiableMap(result);
+    }
+
+    public String getPhase() {
+        return phase;
+    }
+
+    public String getAction() {
+        return action;
+    }
+
+    public String getStep() {
+        return step;
+    }
+
+    public String getFailedStep() {
+        return failedStep;
+    }
+
+    public String getStepInfo() {
+        return stepInfo;
+    }
+
+    public String getPhaseDefinition() {
+        return phaseDefinition;
+    }
+
+    public Long getLifecycleDate() {
+        return lifecycleDate;
+    }
+
+    public Long getPhaseTime() {
+        return phaseTime;
+    }
+
+    public Long getActionTime() {
+        return actionTime;
+    }
+
+    public Long getStepTime() {
+        return stepTime;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        LifecycleExecutionState that = (LifecycleExecutionState) o;
+        return getLifecycleDate() == that.getLifecycleDate() &&
+            getPhaseTime() == that.getPhaseTime() &&
+            getActionTime() == that.getActionTime() &&
+            getStepTime() == that.getStepTime() &&
+            Objects.equals(getPhase(), that.getPhase()) &&
+            Objects.equals(getAction(), that.getAction()) &&
+            Objects.equals(getStep(), that.getStep()) &&
+            Objects.equals(getFailedStep(), that.getFailedStep()) &&
+            Objects.equals(getStepInfo(), that.getStepInfo()) &&
+            Objects.equals(getPhaseDefinition(), that.getPhaseDefinition());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getPhase(), getAction(), getStep(), getFailedStep(), getStepInfo(), getPhaseDefinition(),
+            getLifecycleDate(), getPhaseTime(), getActionTime(), getStepTime());
+    }
+
+    public static class Builder {
+        private String phase;
+        private String action;
+        private String step;
+        private String failedStep;
+        private String stepInfo;
+        private String phaseDefinition;
+        private Long indexCreationDate;
+        private Long phaseTime;
+        private Long actionTime;
+        private Long stepTime;
+
+        public Builder setPhase(String phase) {
+            this.phase = phase;
+            return this;
+        }
+
+        public Builder setAction(String action) {
+            this.action = action;
+            return this;
+        }
+
+        public Builder setStep(String step) {
+            this.step = step;
+            return this;
+        }
+
+        public Builder setFailedStep(String failedStep) {
+            this.failedStep = failedStep;
+            return this;
+        }
+
+        public Builder setStepInfo(String stepInfo) {
+            this.stepInfo = stepInfo;
+            return this;
+        }
+
+        public Builder setPhaseDefinition(String phaseDefinition) {
+            this.phaseDefinition = phaseDefinition;
+            return this;
+        }
+
+        public Builder setIndexCreationDate(Long indexCreationDate) {
+            this.indexCreationDate = indexCreationDate;
+            return this;
+        }
+
+        public Builder setPhaseTime(Long phaseTime) {
+            this.phaseTime = phaseTime;
+            return this;
+        }
+
+        public Builder setActionTime(Long actionTime) {
+            this.actionTime = actionTime;
+            return this;
+        }
+
+        public Builder setStepTime(Long stepTime) {
+            this.stepTime = stepTime;
+            return this;
+        }
+
+        public LifecycleExecutionState build() {
+            return new LifecycleExecutionState(phase, action, step, failedStep, stepInfo, phaseDefinition, indexCreationDate,
+                phaseTime, actionTime, stepTime);
+        }
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java

@@ -14,42 +14,12 @@ import org.elasticsearch.common.unit.TimeValue;
 public class LifecycleSettings {
     public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval";
     public static final String LIFECYCLE_NAME = "index.lifecycle.name";
-    public static final String LIFECYCLE_PHASE = "index.lifecycle.phase";
-    public static final String LIFECYCLE_ACTION = "index.lifecycle.action";
-    public static final String LIFECYCLE_STEP = "index.lifecycle.step";
-    public static final String LIFECYCLE_INDEX_CREATION_DATE = "index.lifecycle.date";
-    public static final String LIFECYCLE_PHASE_TIME = "index.lifecycle.phase_time";
-    public static final String LIFECYCLE_ACTION_TIME = "index.lifecycle.action_time";
-    public static final String LIFECYCLE_STEP_TIME = "index.lifecycle.step_time";
-    public static final String LIFECYCLE_FAILED_STEP = "index.lifecycle.failed_step";
-    public static final String LIFECYCLE_STEP_INFO = "index.lifecycle.step_info";
     public static final String LIFECYCLE_SKIP = "index.lifecycle.skip";
-    public static final String LIFECYCLE_PHASE_DEFINITION = "index.lifecycle.phase_definition";
 
     public static final Setting<TimeValue> LIFECYCLE_POLL_INTERVAL_SETTING = Setting.positiveTimeSetting(LIFECYCLE_POLL_INTERVAL,
         TimeValue.timeValueMinutes(10), Setting.Property.Dynamic, Setting.Property.NodeScope);
     public static final Setting<String> LIFECYCLE_NAME_SETTING = Setting.simpleString(LIFECYCLE_NAME,
         Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<String> LIFECYCLE_PHASE_SETTING = Setting.simpleString(LIFECYCLE_PHASE,
-        Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<String> LIFECYCLE_ACTION_SETTING = Setting.simpleString(LIFECYCLE_ACTION,
-            Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<String> LIFECYCLE_STEP_SETTING = Setting.simpleString(LIFECYCLE_STEP,
-        Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<String> LIFECYCLE_FAILED_STEP_SETTING = Setting.simpleString(LIFECYCLE_FAILED_STEP,
-            Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<Long> LIFECYCLE_INDEX_CREATION_DATE_SETTING = Setting.longSetting(LIFECYCLE_INDEX_CREATION_DATE,
-        -1L, -1L, Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<Long> LIFECYCLE_PHASE_TIME_SETTING = Setting.longSetting(LIFECYCLE_PHASE_TIME,
-        -1L, -1L, Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<Long> LIFECYCLE_ACTION_TIME_SETTING = Setting.longSetting(LIFECYCLE_ACTION_TIME,
-        -1L, -1L, Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<Long> LIFECYCLE_STEP_TIME_SETTING = Setting.longSetting(LIFECYCLE_STEP_TIME,
-        -1L, -1L, Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
-    public static final Setting<String> LIFECYCLE_STEP_INFO_SETTING = Setting.simpleString(LIFECYCLE_STEP_INFO, Setting.Property.Dynamic,
-            Setting.Property.IndexScope, Setting.Property.NotCopyableOnResize, Setting.Property.InternalIndex);
     public static final Setting<Boolean> LIFECYCLE_SKIP_SETTING = Setting.boolSetting(LIFECYCLE_SKIP, false,
         Setting.Property.Dynamic, Setting.Property.IndexScope);
-    public static final Setting<String> LIFECYCLE_PHASE_DEFINITION_SETTING = Setting.simpleString(LIFECYCLE_PHASE_DEFINITION,
-        Setting.Property.Dynamic, Setting.Property.IndexScope);
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java

@@ -85,15 +85,18 @@ public class ShrinkAction implements LifecycleAction {
         StepKey allocationRoutedKey = new StepKey(phase, NAME, AllocationRoutedStep.NAME);
         StepKey shrinkKey = new StepKey(phase, NAME, ShrinkStep.NAME);
         StepKey enoughShardsKey = new StepKey(phase, NAME, ShrunkShardsAllocatedStep.NAME);
+        StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
         StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
         StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
+
         SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client);
         AllocationRoutedStep allocationStep = new AllocationRoutedStep(allocationRoutedKey, shrinkKey, false);
         ShrinkStep shrink = new ShrinkStep(shrinkKey, enoughShardsKey, client, numberOfShards, SHRUNKEN_INDEX_PREFIX);
-        ShrunkShardsAllocatedStep allocated = new ShrunkShardsAllocatedStep(enoughShardsKey, aliasKey, SHRUNKEN_INDEX_PREFIX);
+        ShrunkShardsAllocatedStep allocated = new ShrunkShardsAllocatedStep(enoughShardsKey, copyMetadataKey, SHRUNKEN_INDEX_PREFIX);
+        CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX);
         ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX);
         ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX);
-        return Arrays.asList(setSingleNodeStep, allocationStep, shrink, allocated, aliasSwapAndDelete, waitOnShrinkTakeover);
+        return Arrays.asList(setSingleNodeStep, allocationStep, shrink, allocated, copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover);
     }
 
     @Override
@@ -102,9 +105,11 @@ public class ShrinkAction implements LifecycleAction {
         StepKey allocationRoutedKey = new StepKey(phase, NAME, AllocationRoutedStep.NAME);
         StepKey shrinkKey = new StepKey(phase, NAME, ShrinkStep.NAME);
         StepKey enoughShardsKey = new StepKey(phase, NAME, ShrunkShardsAllocatedStep.NAME);
+        StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
         StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
         StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
-        return Arrays.asList(setSingleNodeKey, allocationRoutedKey, shrinkKey, enoughShardsKey, aliasKey, isShrunkIndexKey);
+        return Arrays.asList(setSingleNodeKey, allocationRoutedKey, shrinkKey, enoughShardsKey,
+            copyMetadataKey, aliasKey, isShrunkIndexKey);
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkStep.java

@@ -37,28 +37,22 @@ public class ShrinkStep extends AsyncActionStep {
 
     @Override
     public void performAction(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
-        // if operating on the shrunken index, do nothing
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
-
+        if (lifecycleState.getLifecycleDate() == null) {
-        Long lifecycleDate = LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING.get(indexMetaData.getSettings());
+            throw new IllegalStateException("source index [" + indexMetaData.getIndex().getName() +
-        if (lifecycleDate == null) {
+                "] is missing lifecycle date");
-            throw new IllegalStateException("source index[" + indexMetaData.getIndex().getName() +
-                "] is missing setting[" + LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE);
         }
+
         String lifecycle = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetaData.getSettings());
-        String phase = LifecycleSettings.LIFECYCLE_PHASE_SETTING.get(indexMetaData.getSettings());
-        String action = LifecycleSettings.LIFECYCLE_ACTION_SETTING.get(indexMetaData.getSettings());
 
         Settings relevantTargetSettings = Settings.builder()
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, indexMetaData.getNumberOfReplicas())
-            .put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, lifecycleDate)
             .put(LifecycleSettings.LIFECYCLE_NAME, lifecycle)
-            .put(LifecycleSettings.LIFECYCLE_PHASE, phase)
-            .put(LifecycleSettings.LIFECYCLE_ACTION, action)
-            .put(LifecycleSettings.LIFECYCLE_STEP, ShrunkenIndexCheckStep.NAME) // skip source-index steps
             .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", (String) null) // need to remove the single shard
                                                                                              // allocation so replicas can be allocated
             .build();
+
         String shrunkenIndexName = shrunkIndexPrefix + indexMetaData.getIndex().getName();
         ResizeRequest resizeRequest = new ResizeRequest(shrunkenIndexName, indexMetaData.getIndex().getName());
         indexMetaData.getAliases().values().spliterator().forEachRemaining(aliasMetaDataObjectCursor -> {
@@ -70,6 +64,7 @@ public class ShrinkStep extends AsyncActionStep {
             // TODO(talevy): when is this not acknowledged?
             listener.onResponse(response.isAcknowledged());
         }, listener::onFailure));
+
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java

@@ -10,9 +10,10 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+
 public class UpdateRolloverLifecycleDateStep extends ClusterStateActionStep {
     public static final String NAME = "update-rollover-lifecycle-date";
 
@@ -33,9 +34,15 @@ public class UpdateRolloverLifecycleDateStep extends ClusterStateActionStep {
         if (rolloverInfo == null) {
             throw new IllegalStateException("index [" + indexMetaData.getIndex().getName() + "] has not rolled over yet");
         }
-        Settings settings = Settings.builder().put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, rolloverInfo.getTime()).build();
+
+        LifecycleExecutionState.Builder newLifecycleState = LifecycleExecutionState
+            .builder(LifecycleExecutionState.fromIndexMetadata(indexMetaData));
+        newLifecycleState.setIndexCreationDate(rolloverInfo.getTime());
+
+        IndexMetaData.Builder newIndexMetadata = IndexMetaData.builder(indexMetaData);
+        newIndexMetadata.putCustom(ILM_CUSTOM_METADATA_KEY, newLifecycleState.build().asMap());
         return ClusterState.builder(currentState).metaData(MetaData.builder(currentState.metaData())
-                .updateSettings(settings, indexMetaData.getIndex().getName())).build();
+            .put(newIndexMetadata)).build();
     }
 
     @Override

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStepTests.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.util.Map;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionStateTests.createCustomMetadata;
+
+public class CopyExecutionStateStepTests extends AbstractStepTestCase<CopyExecutionStateStep> {
+    @Override
+    protected CopyExecutionStateStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        String shrunkIndexPrefix = randomAlphaOfLength(10);
+        return new CopyExecutionStateStep(stepKey, nextStepKey, shrunkIndexPrefix);
+    }
+
+    @Override
+    protected CopyExecutionStateStep mutateInstance(CopyExecutionStateStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        String shrunkIndexPrefix = instance.getShrunkIndexPrefix();
+
+        switch (between(0, 2)) {
+            case 0:
+                key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 1:
+                nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 2:
+                shrunkIndexPrefix += randomAlphaOfLength(5);
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new CopyExecutionStateStep(key, nextKey, shrunkIndexPrefix);
+    }
+
+    @Override
+    protected CopyExecutionStateStep copyInstance(CopyExecutionStateStep instance) {
+        return new CopyExecutionStateStep(instance.getKey(), instance.getNextStepKey(), instance.getShrunkIndexPrefix());
+    }
+
+    public void testPerformAction() {
+        CopyExecutionStateStep step = createRandomInstance();
+        String indexName = randomAlphaOfLengthBetween(5, 20);
+        Map<String, String> customMetadata = createCustomMetadata();
+
+        IndexMetaData originalIndexMetaData = IndexMetaData.builder(indexName)
+            .settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1,5))
+            .numberOfReplicas(randomIntBetween(1,5))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, customMetadata)
+            .build();
+        IndexMetaData shrunkIndexMetaData = IndexMetaData.builder(step.getShrunkIndexPrefix() + indexName)
+            .settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1,5))
+            .numberOfReplicas(randomIntBetween(1,5))
+            .build();
+        ClusterState originalClusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(MetaData.builder()
+                .put(originalIndexMetaData, false)
+                .put(shrunkIndexMetaData, false))
+            .build();
+
+        ClusterState newClusterState = step.performAction(originalIndexMetaData.getIndex(), originalClusterState);
+
+        LifecycleExecutionState oldIndexData = LifecycleExecutionState.fromIndexMetadata(originalIndexMetaData);
+        LifecycleExecutionState newIndexData = LifecycleExecutionState
+            .fromIndexMetadata(newClusterState.metaData().index(step.getShrunkIndexPrefix() + indexName));
+
+        assertEquals(oldIndexData.getLifecycleDate(), newIndexData.getLifecycleDate());
+        assertEquals(oldIndexData.getPhase(), newIndexData.getPhase());
+        assertEquals(oldIndexData.getAction(), newIndexData.getAction());
+        assertEquals(ShrunkenIndexCheckStep.NAME, newIndexData.getStep());
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/IndexExplainResponseTests.java

@@ -71,10 +71,10 @@ public class IndexExplainResponseTests extends AbstractSerializingTestCase<Index
         String action = instance.getAction();
         String step = instance.getStep();
         String failedStep = instance.getFailedStep();
-        long policyTime = instance.getLifecycleDate();
+        Long policyTime = instance.getLifecycleDate();
-        long phaseTime = instance.getPhaseTime();
+        Long phaseTime = instance.getPhaseTime();
-        long actionTime = instance.getActionTime();
+        Long actionTime = instance.getActionTime();
-        long stepTime = instance.getStepTime();
+        Long stepTime = instance.getStepTime();
         boolean managed = instance.managedByILM();
         boolean skip = instance.skip();
         BytesReference stepInfo = instance.getStepInfo();

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStepTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.hamcrest.Matchers.equalTo;
 
 public class InitializePolicyContextStepTests extends AbstractStepTestCase<InitializePolicyContextStep> {
@@ -69,8 +70,11 @@ public class InitializePolicyContextStepTests extends AbstractStepTestCase<Initi
 
     public void testDoNothing() {
         long creationDate = randomNonNegativeLong();
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setIndexCreationDate(creationDate);
         IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
-            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, creationDate))
+            .settings(settings(Version.CURRENT))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
             .creationDate(creationDate)
             .numberOfShards(1).numberOfReplicas(0).build();
         MetaData metaData = MetaData.builder()
@@ -85,7 +89,6 @@ public class InitializePolicyContextStepTests extends AbstractStepTestCase<Initi
     }
 
     private long getIndexLifecycleDate(Index index, ClusterState clusterState) {
-        return clusterState.metaData().index(index).getSettings()
+        return LifecycleExecutionState.fromIndexMetadata(clusterState.getMetaData().index(index)).getLifecycleDate();
-            .getAsLong(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, -1L);
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleExecutionStateTests.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class LifecycleExecutionStateTests extends ESTestCase {
+
+    public void testConversion() {
+        Map<String, String> customMetadata = createCustomMetadata();
+        LifecycleExecutionState parsed = LifecycleExecutionState.fromCustomMetadata(customMetadata);
+        assertEquals(customMetadata, parsed.asMap());
+    }
+
+    public void testEmptyValuesAreNotSerialized() {
+        LifecycleExecutionState empty = LifecycleExecutionState.builder().build();
+        assertEquals(new HashMap<String, String>().entrySet(), empty.asMap().entrySet());
+
+        Map<String, String> originalMap = createCustomMetadata();
+        LifecycleExecutionState originalState = LifecycleExecutionState.fromCustomMetadata(originalMap);
+        LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder(originalState);
+        newState.setPhase(null);
+        assertFalse(newState.build().asMap().containsKey("phase"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setAction(null);
+        assertFalse(newState.build().asMap().containsKey("action"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setStep(null);
+        assertFalse(newState.build().asMap().containsKey("step"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setFailedStep(null);
+        assertFalse(newState.build().asMap().containsKey("failed_step"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setPhaseDefinition(null);
+        assertFalse(newState.build().asMap().containsKey("phase_definition"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setStepInfo(null);
+        assertFalse(newState.build().asMap().containsKey("step_info"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setPhaseTime(null);
+        assertFalse(newState.build().asMap().containsKey("phase_time"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setActionTime(null);
+        assertFalse(newState.build().asMap().containsKey("action_time"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setIndexCreationDate(null);
+        assertFalse(newState.build().asMap().containsKey("creation_date"));
+    }
+
+    public void testEqualsAndHashcode() {
+        LifecycleExecutionState original = LifecycleExecutionState.fromCustomMetadata(createCustomMetadata());
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(
+            original,
+            toCopy -> LifecycleExecutionState.builder(toCopy).build(),
+            LifecycleExecutionStateTests::mutate);
+    }
+
+    private static LifecycleExecutionState mutate(LifecycleExecutionState toMutate) {
+        LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder(toMutate);
+        boolean changed = false;
+        if (randomBoolean()) {
+            newState.setPhase(randomValueOtherThan(toMutate.getPhase(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setAction(randomValueOtherThan(toMutate.getAction(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setStep(randomValueOtherThan(toMutate.getStep(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setPhaseDefinition(randomValueOtherThan(toMutate.getPhaseDefinition(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setFailedStep(randomValueOtherThan(toMutate.getFailedStep(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setStepInfo(randomValueOtherThan(toMutate.getStepInfo(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setPhaseTime(randomValueOtherThan(toMutate.getPhaseTime(), ESTestCase::randomLong));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setActionTime(randomValueOtherThan(toMutate.getActionTime(), ESTestCase::randomLong));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setStepTime(randomValueOtherThan(toMutate.getStepTime(), ESTestCase::randomLong));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setIndexCreationDate(randomValueOtherThan(toMutate.getLifecycleDate(), ESTestCase::randomLong));
+            changed = true;
+        }
+
+        if (changed == false) {
+            return LifecycleExecutionState.builder().build();
+        }
+
+        return newState.build();
+    }
+
+    static Map<String, String> createCustomMetadata() {
+        String phase = randomAlphaOfLengthBetween(5,20);
+        String action = randomAlphaOfLengthBetween(5,20);
+        String step = randomAlphaOfLengthBetween(5,20);
+        String failedStep = randomAlphaOfLengthBetween(5,20);
+        String stepInfo = randomAlphaOfLengthBetween(15,50);
+        String phaseDefinition = randomAlphaOfLengthBetween(15,50);
+        long indexCreationDate = randomLong();
+        long phaseTime = randomLong();
+        long actionTime = randomLong();
+        long stepTime = randomLong();
+
+        Map<String, String> customMetadata = new HashMap<>();
+        customMetadata.put("phase", phase);
+        customMetadata.put("action", action);
+        customMetadata.put("step", step);
+        customMetadata.put("failed_step", failedStep);
+        customMetadata.put("step_info", stepInfo);
+        customMetadata.put("phase_definition", phaseDefinition);
+        customMetadata.put("creation_date", String.valueOf(indexCreationDate));
+        customMetadata.put("phase_time", String.valueOf(phaseTime));
+        customMetadata.put("action_time", String.valueOf(actionTime));
+        customMetadata.put("step_time", String.valueOf(stepTime));
+        return customMetadata;
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java

@@ -51,13 +51,14 @@ public class ShrinkActionTests extends AbstractActionTestCase<ShrinkAction> {
         StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
             randomAlphaOfLengthBetween(1, 10));
         List<Step> steps = action.toSteps(null, phase, nextStepKey);
-        assertThat(steps.size(), equalTo(6));
+        assertThat(steps.size(), equalTo(7));
         StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
         StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, AllocationRoutedStep.NAME);
         StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
         StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
-        StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
+        StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
-        StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);
+        StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
+        StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);
         assertTrue(steps.get(0) instanceof SetSingleNodeAllocateStep);
         assertThat(steps.get(0).getKey(), equalTo(expectedFirstKey));
         assertThat(steps.get(0).getNextStepKey(), equalTo(expectedSecondKey));
@@ -74,14 +75,18 @@ public class ShrinkActionTests extends AbstractActionTestCase<ShrinkAction> {
         assertThat(steps.get(3).getKey(), equalTo(expectedFourthKey));
         assertThat(steps.get(3).getNextStepKey(), equalTo(expectedFifthKey));
         assertThat(((ShrunkShardsAllocatedStep) steps.get(3)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
-        assertTrue(steps.get(4) instanceof ShrinkSetAliasStep);
+        assertTrue(steps.get(4) instanceof CopyExecutionStateStep);
         assertThat(steps.get(4).getKey(), equalTo(expectedFifthKey));
         assertThat(steps.get(4).getNextStepKey(), equalTo(expectedSixthKey));
-        assertThat(((ShrinkSetAliasStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
+        assertThat(((CopyExecutionStateStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
-        assertTrue(steps.get(5) instanceof ShrunkenIndexCheckStep);
+        assertTrue(steps.get(5) instanceof ShrinkSetAliasStep);
         assertThat(steps.get(5).getKey(), equalTo(expectedSixthKey));
-        assertThat(steps.get(5).getNextStepKey(), equalTo(nextStepKey));
+        assertThat(steps.get(5).getNextStepKey(), equalTo(expectedSeventhKey));
-        assertThat(((ShrunkenIndexCheckStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
+        assertThat(((ShrinkSetAliasStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
+        assertTrue(steps.get(6) instanceof ShrunkenIndexCheckStep);
+        assertThat(steps.get(6).getKey(), equalTo(expectedSeventhKey));
+        assertThat(steps.get(6).getNextStepKey(), equalTo(nextStepKey));
+        assertThat(((ShrunkenIndexCheckStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
     }
 
     @Override

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkStepTests.java

@@ -27,6 +27,7 @@ import org.mockito.stubbing.Answer;
 
 import java.util.Collections;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.hamcrest.Matchers.equalTo;
 
 public class ShrinkStepTests extends AbstractStepTestCase<ShrinkStep> {
@@ -82,16 +83,17 @@ public class ShrinkStepTests extends AbstractStepTestCase<ShrinkStep> {
 
     public void testPerformAction() throws Exception {
         String lifecycleName = randomAlphaOfLength(5);
-        long creationDate = randomNonNegativeLong();
         ShrinkStep step = createRandomInstance();
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(step.getKey().getPhase());
+        lifecycleState.setAction(step.getKey().getAction());
+        lifecycleState.setStep(step.getKey().getName());
+        lifecycleState.setIndexCreationDate(randomNonNegativeLong());
         IndexMetaData sourceIndexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
             .settings(settings(Version.CURRENT)
-                .put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, creationDate)
                 .put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)
-                .put(LifecycleSettings.LIFECYCLE_PHASE, step.getKey().getPhase())
-                .put(LifecycleSettings.LIFECYCLE_ACTION, step.getKey().getAction())
-                .put(LifecycleSettings.LIFECYCLE_STEP, ShrunkenIndexCheckStep.NAME)
             )
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5))
             .putAlias(AliasMetaData.builder("my_alias"))
             .build();
@@ -114,11 +116,7 @@ public class ShrinkStepTests extends AbstractStepTestCase<ShrinkStep> {
                 assertThat(request.getTargetIndexRequest().settings(), equalTo(Settings.builder()
                     .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, step.getNumberOfShards())
                     .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, sourceIndexMetaData.getNumberOfReplicas())
-                    .put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, creationDate)
                     .put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)
-                    .put(LifecycleSettings.LIFECYCLE_PHASE, step.getKey().getPhase())
-                    .put(LifecycleSettings.LIFECYCLE_ACTION, step.getKey().getAction())
-                    .put(LifecycleSettings.LIFECYCLE_STEP, ShrunkenIndexCheckStep.NAME)
                     .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", (String) null)
                     .build()));
                 assertThat(request.getTargetIndexRequest().settings()
@@ -151,7 +149,10 @@ public class ShrinkStepTests extends AbstractStepTestCase<ShrinkStep> {
     }
 
     public void testPerformActionNotComplete() throws Exception {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setIndexCreationDate(randomNonNegativeLong());
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
         ShrinkStep step = createRandomInstance();
 
@@ -194,7 +195,10 @@ public class ShrinkStepTests extends AbstractStepTestCase<ShrinkStep> {
     }
 
     public void testPerformActionFailure() throws Exception {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setIndexCreationDate(randomNonNegativeLong());
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
         Exception exception = new RuntimeException();
         ShrinkStep step = createRandomInstance();

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java

@@ -67,8 +67,9 @@ public class UpdateRolloverLifecycleDateStepTests extends AbstractStepTestCase<U
 
         UpdateRolloverLifecycleDateStep step = createRandomInstance();
         ClusterState newState = step.performAction(indexMetaData.getIndex(), clusterState);
-        long actualRolloverTime = LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING
+        long actualRolloverTime = LifecycleExecutionState
-            .get(newState.metaData().index(indexMetaData.getIndex()).getSettings());
+            .fromIndexMetadata(newState.metaData().index(indexMetaData.getIndex()))
+            .getLifecycleDate();
         assertThat(actualRolloverTime, equalTo(rolloverTime));
     }
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java

@@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateActionStep;
 import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
 
 import java.io.IOException;
@@ -71,7 +72,7 @@ public class ExecuteStepsUpdateTask extends ClusterStateUpdateTask {
             return currentState;
         }
         Step registeredCurrentStep = IndexLifecycleRunner.getCurrentStep(policyStepsRegistry, policy, indexMetaData,
-            indexMetaData.getSettings());
+            LifecycleExecutionState.fromIndexMetadata(indexMetaData));
         if (currentStep.equals(registeredCurrentStep)) {
             // We can do cluster state steps all together until we
             // either get to a step that isn't a cluster state step or a

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java

@@ -123,17 +123,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin {
         return Arrays.asList(
             LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING,
             LifecycleSettings.LIFECYCLE_NAME_SETTING,
-            LifecycleSettings.LIFECYCLE_PHASE_SETTING,
-            LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING,
-            LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING,
-            LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING,
-            LifecycleSettings.LIFECYCLE_ACTION_SETTING,
-            LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING,
-            LifecycleSettings.LIFECYCLE_STEP_SETTING,
-            LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING,
-            LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING,
             LifecycleSettings.LIFECYCLE_SKIP_SETTING,
-            LifecycleSettings.LIFECYCLE_PHASE_DEFINITION_SETTING,
             RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING);
     }
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java

@@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateActionStep;
 import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep;
 import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.Phase;
@@ -42,6 +43,8 @@ import java.io.IOException;
 import java.util.List;
 import java.util.function.LongSupplier;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+
 public class IndexLifecycleRunner {
     private static final Logger logger = LogManager.getLogger(IndexLifecycleRunner.class);
     private PolicyStepsRegistry stepRegistry;
@@ -58,13 +61,13 @@ public class IndexLifecycleRunner {
      * Return true or false depending on whether the index is ready to be in {@code phase}
      */
     boolean isReadyToTransitionToThisPhase(final String policy, final IndexMetaData indexMetaData, final String phase) {
-        final Settings indexSettings = indexMetaData.getSettings();
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
-        if (indexSettings.hasValue(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE) == false) {
+        if (lifecycleState.getLifecycleDate() == null) {
             logger.trace("no index creation date has been set yet");
             return true;
         }
-        final long lifecycleDate = indexSettings.getAsLong(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, -1L);
+        final Long lifecycleDate = lifecycleState.getLifecycleDate();
-        assert lifecycleDate >= 0 : "expected index to have a lifecycle date but it did not";
+        assert lifecycleDate != null && lifecycleDate >= 0 : "expected index to have a lifecycle date but it did not";
         final TimeValue after = stepRegistry.getIndexAgeForPhase(policy, phase);
         final long now = nowSupplier.getAsLong();
         final TimeValue age = new TimeValue(now - lifecycleDate);
@@ -80,18 +83,19 @@ public class IndexLifecycleRunner {
     }
 
     public void runPolicy(String policy, IndexMetaData indexMetaData, ClusterState currentState,
-            boolean fromClusterStateChange) {
+                          boolean fromClusterStateChange) {
         Settings indexSettings = indexMetaData.getSettings();
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
         if (LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(indexSettings)) {
             logger.info("skipping policy [" + policy + "] for index [" + indexMetaData.getIndex().getName() + "]."
                 + LifecycleSettings.LIFECYCLE_SKIP + "== true");
             return;
         }
-        Step currentStep = getCurrentStep(stepRegistry, policy, indexMetaData, indexSettings);
+        Step currentStep = getCurrentStep(stepRegistry, policy, indexMetaData, lifecycleState);
         if (currentStep == null) {
             // This may happen in the case that there is invalid ilm-step index settings or the stepRegistry is out of
             // sync with the current cluster state
-            logger.warn("current step [" + getCurrentStepKey(indexSettings) + "] for index [" + indexMetaData.getIndex().getName()
+            logger.warn("current step [" + getCurrentStepKey(lifecycleState) + "] for index [" + indexMetaData.getIndex().getName()
                 + "] with policy [" + policy + "] is not recognized");
             return;
         }
@@ -154,7 +158,7 @@ public class IndexLifecycleRunner {
             }
         } else {
             throw new IllegalStateException(
-                    "Step with key [" + currentStep.getKey() + "] is not a recognised type: [" + currentStep.getClass().getName() + "]");
+                "Step with key [" + currentStep.getKey() + "] is not a recognised type: [" + currentStep.getClass().getName() + "]");
         }
     }
 
@@ -179,13 +183,12 @@ public class IndexLifecycleRunner {
      * or for the step to be unset with the phase and/or action set. All three
      * settings must be either present or missing.
      *
-     * @param indexSettings
+     * @param lifecycleState the index custom data to extract the {@link StepKey} from.
-     *            the index settings to extract the {@link StepKey} from.
      */
-    public static StepKey getCurrentStepKey(Settings indexSettings) {
+    public static StepKey getCurrentStepKey(LifecycleExecutionState lifecycleState) {
-        String currentPhase = LifecycleSettings.LIFECYCLE_PHASE_SETTING.get(indexSettings);
+        String currentPhase = lifecycleState.getPhase();
-        String currentAction = LifecycleSettings.LIFECYCLE_ACTION_SETTING.get(indexSettings);
+        String currentAction = lifecycleState.getAction();
-        String currentStep = LifecycleSettings.LIFECYCLE_STEP_SETTING.get(indexSettings);
+        String currentStep = lifecycleState.getStep();
         if (Strings.isNullOrEmpty(currentStep)) {
             assert Strings.isNullOrEmpty(currentPhase) : "Current phase is not empty: " + currentPhase;
             assert Strings.isNullOrEmpty(currentAction) : "Current action is not empty: " + currentAction;
@@ -197,8 +200,9 @@ public class IndexLifecycleRunner {
         }
     }
 
-    static Step getCurrentStep(PolicyStepsRegistry stepRegistry, String policy, IndexMetaData indexMetaData, Settings indexSettings) {
+    static Step getCurrentStep(PolicyStepsRegistry stepRegistry, String policy, IndexMetaData indexMetaData,
-        StepKey currentStepKey = getCurrentStepKey(indexSettings);
+                               LifecycleExecutionState lifecycleState) {
+        StepKey currentStepKey = getCurrentStepKey(lifecycleState);
         if (currentStepKey == null) {
             return stepRegistry.getFirstStep(policy);
         } else {
@@ -209,18 +213,13 @@ public class IndexLifecycleRunner {
     /**
      * This method is intended for handling moving to different steps from {@link TransportAction} executions.
      * For this reason, it is reasonable to throw {@link IllegalArgumentException} when state is not as expected.
-     * @param indexName
+     *
-     *          The index whose step is to change
+     * @param indexName      The index whose step is to change
-     * @param currentState
+     * @param currentState   The current {@link ClusterState}
-     *          The current {@link ClusterState}
+     * @param currentStepKey The current {@link StepKey} found for the index in the current cluster state
-     * @param currentStepKey
+     * @param nextStepKey    The next step to move the index into
-     *          The current {@link StepKey} found for the index in the current cluster state
+     * @param nowSupplier    The current-time supplier for updating when steps changed
-     * @param nextStepKey
+     * @param stepRegistry   The steps registry to check a step-key's existence in the index's current policy
-     *          The next step to move the index into
-     * @param nowSupplier
-     *          The current-time supplier for updating when steps changed
-     * @param stepRegistry
-     *          The steps registry to check a step-key's existence in the index's current policy
      * @return The updated cluster state where the index moved to <code>nextStepKey</code>
      */
     static ClusterState moveClusterStateToStep(String indexName, ClusterState currentState, StepKey currentStepKey,
@@ -235,7 +234,8 @@ public class IndexLifecycleRunner {
             throw new IllegalArgumentException("index [" + indexName + "] is not associated with an Index Lifecycle Policy");
         }
 
-        if (currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexSettings)) == false) {
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMeta);
+        if (currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(lifecycleState)) == false) {
             throw new IllegalArgumentException("index [" + indexName + "] is not on current step [" + currentStepKey + "]");
         }
 
@@ -248,19 +248,21 @@ public class IndexLifecycleRunner {
     }
 
     static ClusterState moveClusterStateToNextStep(Index index, ClusterState clusterState, StepKey currentStep, StepKey nextStep,
-            LongSupplier nowSupplier) {
+                                                   LongSupplier nowSupplier) {
         IndexMetaData idxMeta = clusterState.getMetaData().index(index);
         IndexLifecycleMetadata ilmMeta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
         LifecyclePolicyMetadata policyMetadata = ilmMeta.getPolicyMetadatas()
             .get(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()));
-        Settings.Builder indexSettings = moveIndexSettingsToNextStep(policyMetadata, idxMeta.getSettings(), currentStep,
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMeta);
-            nextStep, nowSupplier);
+        LifecycleExecutionState newLifecycleState = moveExecutionStateToNextStep(policyMetadata,
-        ClusterState.Builder newClusterStateBuilder = newClusterStateWithIndexSettings(index, clusterState, indexSettings);
+            lifecycleState, currentStep, nextStep, nowSupplier);
+        ClusterState.Builder newClusterStateBuilder = newClusterStateWithLifecycleState(index, clusterState, newLifecycleState);
+
         return newClusterStateBuilder.build();
     }
 
     static ClusterState moveClusterStateToErrorStep(Index index, ClusterState clusterState, StepKey currentStep, Exception cause,
-            LongSupplier nowSupplier) throws IOException {
+                                                    LongSupplier nowSupplier) throws IOException {
         IndexMetaData idxMeta = clusterState.getMetaData().index(index);
         IndexLifecycleMetadata ilmMeta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
         LifecyclePolicyMetadata policyMetadata = ilmMeta.getPolicyMetadatas()
@@ -269,11 +271,13 @@ public class IndexLifecycleRunner {
         causeXContentBuilder.startObject();
         ElasticsearchException.generateThrowableXContent(causeXContentBuilder, ToXContent.EMPTY_PARAMS, cause);
         causeXContentBuilder.endObject();
-        Settings.Builder indexSettings = moveIndexSettingsToNextStep(policyMetadata, idxMeta.getSettings(), currentStep,
+        LifecycleExecutionState nextStepState = moveExecutionStateToNextStep(policyMetadata,
-                new StepKey(currentStep.getPhase(), currentStep.getAction(), ErrorStep.NAME), nowSupplier)
+            LifecycleExecutionState.fromIndexMetadata(idxMeta), currentStep, new StepKey(currentStep.getPhase(),
-                        .put(LifecycleSettings.LIFECYCLE_FAILED_STEP, currentStep.getName())
+                currentStep.getAction(), ErrorStep.NAME), nowSupplier);
-                        .put(LifecycleSettings.LIFECYCLE_STEP_INFO, BytesReference.bytes(causeXContentBuilder).utf8ToString());
+        LifecycleExecutionState.Builder failedState = LifecycleExecutionState.builder(nextStepState);
-        ClusterState.Builder newClusterStateBuilder = newClusterStateWithIndexSettings(index, clusterState, indexSettings);
+        failedState.setFailedStep(currentStep.getName());
+        failedState.setStepInfo(BytesReference.bytes(causeXContentBuilder).utf8ToString());
+        ClusterState.Builder newClusterStateBuilder = newClusterStateWithLifecycleState(index, clusterState, failedState.build());
         return newClusterStateBuilder.build();
     }
 
@@ -284,8 +288,9 @@ public class IndexLifecycleRunner {
             if (indexMetaData == null) {
                 throw new IllegalArgumentException("index [" + index + "] does not exist");
             }
-            StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(indexMetaData.getSettings());
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
-            String failedStep = LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING.get(indexMetaData.getSettings());
+            StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+            String failedStep = lifecycleState.getFailedStep();
             if (currentStepKey != null && ErrorStep.NAME.equals(currentStepKey.getName())
                 && Strings.isNullOrEmpty(failedStep) == false) {
                 StepKey nextStepKey = new StepKey(currentStepKey.getPhase(), currentStepKey.getAction(), failedStep);
@@ -298,15 +303,21 @@ public class IndexLifecycleRunner {
         return newState;
     }
 
-    private static Settings.Builder moveIndexSettingsToNextStep(LifecyclePolicyMetadata policyMetadata, Settings existingSettings,
+    private static LifecycleExecutionState moveExecutionStateToNextStep(LifecyclePolicyMetadata policyMetadata,
-                                                                StepKey currentStep, StepKey nextStep, LongSupplier nowSupplier) {
+                                                                        LifecycleExecutionState existingState,
+                                                                        StepKey currentStep, StepKey nextStep,
+                                                                        LongSupplier nowSupplier) {
         long nowAsMillis = nowSupplier.getAsLong();
-        Settings.Builder newSettings = Settings.builder().put(existingSettings).put(LifecycleSettings.LIFECYCLE_PHASE, nextStep.getPhase())
+        LifecycleExecutionState.Builder updatedState = LifecycleExecutionState.builder(existingState);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, nextStep.getAction()).put(LifecycleSettings.LIFECYCLE_STEP, nextStep.getName())
+        updatedState.setPhase(nextStep.getPhase());
-                .put(LifecycleSettings.LIFECYCLE_STEP_TIME, nowAsMillis)
+        updatedState.setAction(nextStep.getAction());
-                // clear any step info or error-related settings from the current step
+        updatedState.setStep(nextStep.getName());
-                .put(LifecycleSettings.LIFECYCLE_FAILED_STEP, (String) null)
+        updatedState.setStepTime(nowAsMillis);
-                .put(LifecycleSettings.LIFECYCLE_STEP_INFO, (String) null);
+
+        // clear any step info or error-related settings from the current step
+        updatedState.setFailedStep(null);
+        updatedState.setStepInfo(null);
+
         if (currentStep.getPhase().equals(nextStep.getPhase()) == false) {
             final String newPhaseDefinition;
             final Phase nextPhase;
@@ -318,20 +329,21 @@ public class IndexLifecycleRunner {
             PhaseExecutionInfo phaseExecutionInfo = new PhaseExecutionInfo(policyMetadata.getName(), nextPhase,
                 policyMetadata.getVersion(), policyMetadata.getModifiedDate());
             newPhaseDefinition = Strings.toString(phaseExecutionInfo, false, false);
-            newSettings.put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, newPhaseDefinition);
+            updatedState.setPhaseDefinition(newPhaseDefinition);
-            newSettings.put(LifecycleSettings.LIFECYCLE_PHASE_TIME, nowAsMillis);
+            updatedState.setPhaseTime(nowAsMillis);
         }
         if (currentStep.getAction().equals(nextStep.getAction()) == false) {
-            newSettings.put(LifecycleSettings.LIFECYCLE_ACTION_TIME, nowAsMillis);
+            updatedState.setActionTime(nowAsMillis);
         }
-        return newSettings;
+        return updatedState.build();
     }
 
-    static ClusterState.Builder newClusterStateWithIndexSettings(Index index, ClusterState clusterState,
+    static ClusterState.Builder newClusterStateWithLifecycleState(Index index, ClusterState clusterState,
-            Settings.Builder newSettings) {
+                                                                  LifecycleExecutionState lifecycleState) {
         ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
         newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData())
-                .put(IndexMetaData.builder(clusterState.getMetaData().index(index)).settings(newSettings)));
+            .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
+                .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.asMap())));
         return newClusterStateBuilder;
     }
 
@@ -340,39 +352,44 @@ public class IndexLifecycleRunner {
      * built if the step info has changed, otherwise the same old <code>clusterState</code> is
      * returned
      *
-     * @param index the index to modify
+     * @param index        the index to modify
      * @param clusterState the cluster state to modify
-     * @param stepInfo the new step info to update
+     * @param stepInfo     the new step info to update
      * @return Updated cluster state with <code>stepInfo</code> if changed, otherwise the same cluster state
-     *         if no changes to step info exist
+     * if no changes to step info exist
      * @throws IOException if parsing step info fails
      */
     static ClusterState addStepInfoToClusterState(Index index, ClusterState clusterState, ToXContentObject stepInfo) throws IOException {
-        IndexMetaData idxMeta = clusterState.getMetaData().index(index);
+        IndexMetaData indexMetaData = clusterState.getMetaData().index(index);
+        if (indexMetaData == null) {
+            // This index doesn't exist anymore, we can't do anything
+            return clusterState;
+        }
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
         final String stepInfoString;
         try (XContentBuilder infoXContentBuilder = JsonXContent.contentBuilder()) {
             stepInfo.toXContent(infoXContentBuilder, ToXContent.EMPTY_PARAMS);
             stepInfoString = BytesReference.bytes(infoXContentBuilder).utf8ToString();
         }
-        if (stepInfoString.equals(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(idxMeta.getSettings()))) {
+        if (stepInfoString.equals(lifecycleState.getStepInfo())) {
             return clusterState;
         }
-        Settings.Builder indexSettings = Settings.builder().put(idxMeta.getSettings())
+        LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder(lifecycleState);
-            .put(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.getKey(), stepInfoString);
+        newState.setStepInfo(stepInfoString);
-        ClusterState.Builder newClusterStateBuilder = newClusterStateWithIndexSettings(index, clusterState, indexSettings);
+        ClusterState.Builder newClusterStateBuilder = newClusterStateWithLifecycleState(index, clusterState, newState.build());
         return newClusterStateBuilder.build();
     }
 
     private void moveToStep(Index index, String policy, StepKey currentStepKey, StepKey nextStepKey) {
         logger.debug("moveToStep[" + policy + "] [" + index.getName() + "]" + currentStepKey + " -> "
-                + nextStepKey);
+            + nextStepKey);
         clusterService.submitStateUpdateTask("ILM", new MoveToNextStepUpdateTask(index, policy, currentStepKey,
-                nextStepKey, nowSupplier));
+            nextStepKey, nowSupplier));
     }
 
     private void moveToErrorStep(Index index, String policy, StepKey currentStepKey, Exception e) {
         logger.error("policy [" + policy + "] for index [" + index.getName() + "] failed on step [" + currentStepKey
-                + "]. Moving to ERROR step.", e);
+            + "]. Moving to ERROR step.", e);
         clusterService.submitStateUpdateTask("ILM", new MoveToErrorStepUpdateTask(index, policy, currentStepKey, e, nowSupplier));
     }
 
@@ -381,7 +398,8 @@ public class IndexLifecycleRunner {
     }
 
     public static ClusterState setPolicyForIndexes(final String newPolicyName, final Index[] indices, ClusterState currentState,
-            LifecyclePolicyMetadata newPolicyMetadata, List<String> failedIndexes, LongSupplier nowSupplier) {
+                                                   LifecyclePolicyMetadata newPolicyMetadata, List<String> failedIndexes,
+                                                   LongSupplier nowSupplier) {
         MetaData.Builder newMetadata = MetaData.builder(currentState.getMetaData());
         boolean clusterStateChanged = false;
         for (Index index : indices) {
@@ -409,20 +427,22 @@ public class IndexLifecycleRunner {
 
     private static IndexMetaData.Builder setPolicyForIndex(final String newPolicyName, LifecyclePolicyMetadata newPolicyMetadata,
                                                            IndexMetaData indexMetadata, LongSupplier nowSupplier) {
-        Settings idxSettings = indexMetadata.getSettings();
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetadata);
-        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(idxSettings);
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
 
-        Settings.Builder newSettings = Settings.builder().put(idxSettings);
+        LifecycleExecutionState newState = LifecycleExecutionState.builder(lifecycleState).build();
         if (currentStepKey != null) {
             // Check if current step exists in new policy and if not move to
             // next available step
             StepKey nextValidStepKey = newPolicyMetadata.getPolicy().getNextValidStep(currentStepKey);
             if (nextValidStepKey.equals(currentStepKey) == false) {
-                newSettings = moveIndexSettingsToNextStep(newPolicyMetadata, idxSettings, currentStepKey, nextValidStepKey, nowSupplier);
+                newState = moveExecutionStateToNextStep(newPolicyMetadata, lifecycleState, currentStepKey, nextValidStepKey, nowSupplier);
             }
         }
+
+        Settings.Builder newSettings = Settings.builder().put(indexMetadata.getSettings());
         newSettings.put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), newPolicyName);
-        return IndexMetaData.builder(indexMetadata).settings(newSettings);
+        return IndexMetaData.builder(indexMetadata).settings(newSettings).putCustom(ILM_CUSTOM_METADATA_KEY, newState.asMap());
     }
 
     public static ClusterState removePolicyForIndexes(final Index[] indices, ClusterState currentState, List<String> failedIndexes) {
@@ -455,17 +475,11 @@ public class IndexLifecycleRunner {
         Settings.Builder newSettings = Settings.builder().put(idxSettings);
 
         newSettings.remove(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_PHASE_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_ACTION_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_STEP_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING.getKey());
-        newSettings.remove(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING.getKey());
         newSettings.remove(LifecycleSettings.LIFECYCLE_SKIP_SETTING.getKey());
         newSettings.remove(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.getKey());
-        return IndexMetaData.builder(indexMetadata).settings(newSettings);
+
+        return IndexMetaData.builder(indexMetadata)
+            .removeCustom(ILM_CUSTOM_METADATA_KEY)
+            .settings(newSettings);
     }
 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.indexlifecycle;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
@@ -25,6 +24,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
@@ -192,7 +192,7 @@ public class IndexLifecycleService extends AbstractComponent
             IndexMetaData idxMeta = cursor.value;
             String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings());
             if (Strings.isNullOrEmpty(policyName) == false) {
-                StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(idxMeta.getSettings());
+                StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(LifecycleExecutionState.fromIndexMetadata(idxMeta));
                 if (OperationMode.STOPPING == currentMode && stepKey != null
                         && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) {
                     logger.info("skipping policy [" + policyName + "] for index [" + idxMeta.getIndex().getName()

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTask.java

@@ -11,6 +11,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
 
@@ -56,8 +57,9 @@ public class MoveToErrorStepUpdateTask extends ClusterStateUpdateTask {
             return currentState;
         }
         Settings indexSettings = idxMeta.getSettings();
+        LifecycleExecutionState indexILMData = LifecycleExecutionState.fromIndexMetadata(idxMeta);
         if (policy.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings))
-            && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexSettings))) {
+            && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexILMData))) {
             return IndexLifecycleRunner.moveClusterStateToErrorStep(index, currentState, currentStepKey, cause, nowSupplier);
         } else {
             // either the policy has changed or the step is now

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTask.java

@@ -11,6 +11,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
 
@@ -56,8 +57,9 @@ public class MoveToNextStepUpdateTask extends ClusterStateUpdateTask {
             return currentState;
         }
         Settings indexSettings = indexMetaData.getSettings();
+        LifecycleExecutionState indexILMData = LifecycleExecutionState.fromIndexMetadata(currentState.getMetaData().index(index));
         if (policy.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings))
-            && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexSettings))) {
+            && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexILMData))) {
             return IndexLifecycleRunner.moveClusterStateToNextStep(index, currentState, currentStepKey, nextStepKey, nowSupplier);
         } else {
             // either the policy has changed or the step is now

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java

@@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.InitializePolicyContextStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
@@ -40,6 +41,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
@@ -187,10 +189,9 @@ public class PolicyStepsRegistry {
             throw new IllegalArgumentException("failed to retrieve step " + stepKey + " as index [" + index.getName() + "] has no policy");
         }
 
-
         // parse phase steps from the phase definition in the index settings
-        final String phaseJson = indexMetaData.getSettings().get(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION,
+        final String phaseJson = Optional.ofNullable(LifecycleExecutionState.fromIndexMetadata(indexMetaData).getPhaseDefinition())
-            InitializePolicyContextStep.INITIALIZATION_PHASE);
+            .orElse(InitializePolicyContextStep.INITIALIZATION_PHASE);
 
         final List<Step> phaseSteps;
         try {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTask.java

@@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
 
@@ -55,8 +56,9 @@ public class SetStepInfoUpdateTask extends ClusterStateUpdateTask {
             return currentState;
         }
         Settings indexSettings = idxMeta.getSettings();
+        LifecycleExecutionState indexILMData = LifecycleExecutionState.fromIndexMetadata(idxMeta);
         if (policy.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings))
-                && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexSettings))) {
+                && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexILMData))) {
             return IndexLifecycleRunner.addStepInfoToClusterState(index, currentState, stepInfo);
         } else {
             // either the policy has changed or the step is now

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportExplainLifecycleAction.java

@@ -24,11 +24,12 @@ import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
 import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleExplainResponse;
-import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
-import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.PhaseExecutionInfo;
 import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction;
@@ -76,10 +77,16 @@ public class TransportExplainLifecycleAction
         for (String index : concreteIndices) {
             IndexMetaData idxMetadata = state.metaData().index(index);
             Settings idxSettings = idxMetadata.getSettings();
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMetadata);
             String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxSettings);
-            String currentPhase = LifecycleSettings.LIFECYCLE_PHASE_SETTING.get(idxSettings);
+            String currentPhase = lifecycleState.getPhase();
+            String stepInfo = lifecycleState.getStepInfo();
+            BytesArray stepInfoBytes = null;
+            if (stepInfo != null) {
+                stepInfoBytes = new BytesArray(stepInfo);
+            }
             // parse existing phase steps from the phase definition in the index settings
-            String phaseDef = idxSettings.get(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION);
+            String phaseDef = lifecycleState.getPhaseDefinition();
             PhaseExecutionInfo phaseExecutionInfo = null;
             if (Strings.isNullOrEmpty(phaseDef) == false) {
                 try (XContentParser parser = JsonXContent.jsonXContent.createParser(xContentRegistry,
@@ -87,7 +94,7 @@ public class TransportExplainLifecycleAction
                     phaseExecutionInfo = PhaseExecutionInfo.parse(parser, currentPhase);
                 } catch (IOException e) {
                     listener.onFailure(new ElasticsearchParseException(
-                        "failed to parse [" + LifecycleSettings.LIFECYCLE_PHASE_DEFINITION + "] for index [" + index + "]", e));
+                        "failed to parse phase definition for index [" + index + "]", e));
                     return;
                 }
             }
@@ -95,15 +102,15 @@ public class TransportExplainLifecycleAction
             if (Strings.hasLength(policyName)) {
                 indexResponse = IndexLifecycleExplainResponse.newManagedIndexResponse(index, policyName,
                     LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(idxSettings),
-                    LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING.get(idxSettings),
+                    lifecycleState.getLifecycleDate(),
-                    LifecycleSettings.LIFECYCLE_PHASE_SETTING.get(idxSettings),
+                    lifecycleState.getPhase(),
-                    LifecycleSettings.LIFECYCLE_ACTION_SETTING.get(idxSettings),
+                    lifecycleState.getAction(),
-                    LifecycleSettings.LIFECYCLE_STEP_SETTING.get(idxSettings),
+                    lifecycleState.getStep(),
-                    LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING.get(idxSettings),
+                    lifecycleState.getFailedStep(),
-                    LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(idxSettings),
+                    lifecycleState.getPhaseTime(),
-                    LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(idxSettings),
+                    lifecycleState.getActionTime(),
-                    LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(idxSettings),
+                    lifecycleState.getStepTime(),
-                    new BytesArray(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(idxSettings)),
+                    stepInfoBytes,
                     phaseExecutionInfo);
             } else {
                 indexResponse = IndexLifecycleExplainResponse.newUnmanagedIndexResponse(index);

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java

@@ -22,6 +22,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
@@ -44,8 +45,10 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.newTestLifecyclePolicy;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class ExecuteStepsUpdateTaskTests extends ESTestCase {
@@ -110,12 +113,14 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
 
     private IndexMetaData setupIndexPolicy(String policyName) {
         // Reset the index to use the "allClusterPolicyName"
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase("new");
+        lifecycleState.setAction("init");
+        lifecycleState.setStep("init");
         IndexMetaData indexMetadata = IndexMetaData.builder(indexName)
             .settings(settings(Version.CURRENT)
-                .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
-                .put(LifecycleSettings.LIFECYCLE_PHASE, "new")
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
-                .put(LifecycleSettings.LIFECYCLE_ACTION, "init")
-                .put(LifecycleSettings.LIFECYCLE_STEP, "init"))
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
         index = indexMetadata.getIndex();
         MetaData metaData = MetaData.builder()
@@ -149,24 +154,28 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
         long now = randomNonNegativeLong();
         ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, () -> now);
         ClusterState newState = task.execute(clusterState);
-        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(newState.metaData().index(index).getSettings());
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
         assertThat(currentStepKey, equalTo(thirdStepKey));
         assertThat(firstStep.getExecuteCount(), equalTo(0L));
         assertThat(secondStep.getExecuteCount(), equalTo(1L));
-        assertThat(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getActionTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(""));
+        assertThat(lifecycleState.getStepInfo(), nullValue());
     }
 
     public void testExecuteInvalidStartStep() throws IOException {
         // Unset the index's phase/action/step to simulate starting from scratch
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.getMetaData().index(index)));
+        lifecycleState.setPhase(null);
+        lifecycleState.setAction(null);
+        lifecycleState.setStep(null);
         clusterState = ClusterState.builder(clusterState)
-            .metaData(MetaData.builder(clusterState.metaData())
+            .metaData(MetaData.builder(clusterState.getMetaData())
-                .put(IndexMetaData.builder(clusterState.metaData().index(indexName))
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
-                    .settings(Settings.builder().put(clusterState.metaData().index(indexName).getSettings())
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
-                        .put(LifecycleSettings.LIFECYCLE_PHASE, (String) null)
+
-                        .put(LifecycleSettings.LIFECYCLE_ACTION, (String) null)
-                        .put(LifecycleSettings.LIFECYCLE_STEP, (String) null).build()))).build();
         policyStepsRegistry.update(clusterState);
 
         Step invalidStep = new MockClusterStateActionStep(firstStepKey, secondStepKey);
@@ -183,13 +192,14 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
         long now = randomNonNegativeLong();
         ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, () -> now);
         ClusterState newState = task.execute(clusterState);
-        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(newState.metaData().index(index).getSettings());
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
         assertThat(currentStepKey, equalTo(secondStepKey));
         assertThat(firstStep.getExecuteCount(), equalTo(0L));
         assertThat(secondStep.getExecuteCount(), equalTo(1L));
-        assertThat(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getActionTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(""));
+        assertThat(lifecycleState.getStepInfo(), nullValue());
     }
 
     public void testExecuteIncompleteWaitStepWithInfo() throws IOException {
@@ -201,14 +211,14 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
         long now = randomNonNegativeLong();
         ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, () -> now);
         ClusterState newState = task.execute(clusterState);
-        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(newState.metaData().index(index).getSettings());
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
         assertThat(currentStepKey, equalTo(secondStepKey));
         assertThat(firstStep.getExecuteCount(), equalTo(0L));
         assertThat(secondStep.getExecuteCount(), equalTo(1L));
-        assertThat(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getActionTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newState.metaData().index(index).getSettings()),
+        assertThat(lifecycleState.getStepInfo(), equalTo(stepInfo.toString()));
-                equalTo(stepInfo.toString()));
     }
 
     public void testOnFailure() throws IOException {
@@ -225,13 +235,15 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
     }
 
     private void setStateToKey(StepKey stepKey) throws IOException {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.getMetaData().index(index)));
+        lifecycleState.setPhase(stepKey.getPhase());
+        lifecycleState.setAction(stepKey.getAction());
+        lifecycleState.setStep(stepKey.getName());
         clusterState = ClusterState.builder(clusterState)
-            .metaData(MetaData.builder(clusterState.metaData())
+            .metaData(MetaData.builder(clusterState.getMetaData())
-                .put(IndexMetaData.builder(clusterState.metaData().index(indexName))
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
-                    .settings(Settings.builder().put(clusterState.metaData().index(indexName).getSettings())
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
-                        .put(LifecycleSettings.LIFECYCLE_PHASE, stepKey.getPhase())
-                        .put(LifecycleSettings.LIFECYCLE_ACTION, stepKey.getAction())
-                        .put(LifecycleSettings.LIFECYCLE_STEP, stepKey.getName()).build()))).build();
         policyStepsRegistry.update(clusterState);
     }
 }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationIT.java

@@ -7,7 +7,7 @@ package org.elasticsearch.xpack.indexlifecycle;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -32,6 +32,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
 import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleExplainResponse;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
@@ -176,9 +177,9 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
         assertThat(indexLifecycleService.getScheduler().jobCount(), equalTo(1));
         assertNotNull(indexLifecycleService.getScheduledJob());
         assertBusy(() -> {
-            GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get();
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
-            String step = settingsResponse.getSetting("test", "index.lifecycle.step");
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
-            assertThat(step, equalTo(TerminalPolicyStep.KEY.getName()));
+            assertThat(lifecycleState.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
         });
     }
 
@@ -273,9 +274,9 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
             assertEquals(true, client().admin().indices().prepareExists("test").get().isExists());
         });
         assertBusy(() -> {
-            GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get();
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
-            String step = settingsResponse.getSetting("test", "index.lifecycle.step");
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
-            assertThat(step, equalTo(TerminalPolicyStep.KEY.getName()));
+            assertThat(lifecycleState.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
         });
     }
 
@@ -313,9 +314,9 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
 
         // check step in progress in lifecycle
         assertBusy(() -> {
-            GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get();
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
-            String step = settingsResponse.getSetting("test", "index.lifecycle.step");
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
-            assertThat(step, equalTo(ObservableClusterStateWaitStep.NAME));
+            assertThat(lifecycleState.getStep(), equalTo(ObservableClusterStateWaitStep.NAME));
         });
 
         if (randomBoolean()) {
@@ -335,19 +336,20 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
 
         // check that index lifecycle picked back up where it
         assertBusy(() -> {
-            GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get();
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
-            String step = settingsResponse.getSetting("test", "index.lifecycle.step");
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
-            assertThat(step, equalTo(ObservableClusterStateWaitStep.NAME));
+            assertThat(lifecycleState.getStep(), equalTo(ObservableClusterStateWaitStep.NAME));
         });
 
+        logger.info("new master is operation");
         // complete the step
-        client().admin().indices().prepareUpdateSettings("test")
+        AcknowledgedResponse repsonse = client().admin().indices().prepareUpdateSettings("test")
             .setSettings(Collections.singletonMap("index.lifecycle.test.complete", true)).get();
 
         assertBusy(() -> {
-            GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get();
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
-            String step = settingsResponse.getSetting("test", "index.lifecycle.step");
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
-            assertThat(step, equalTo(TerminalPolicyStep.KEY.getName()));
+            assertThat(lifecycleState.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
         });
     }
 

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java

@@ -33,6 +33,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep;
 import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
@@ -62,6 +63,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.newTestLifecyclePolicy;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
@@ -110,10 +112,12 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
         ClusterService clusterService = mock(ClusterService.class);
         IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
-        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT)
+        LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_PHASE, stepKey.getPhase())
+        newState.setPhase(stepKey.getPhase());
-                .put(LifecycleSettings.LIFECYCLE_ACTION, stepKey.getAction())
+        newState.setAction(stepKey.getAction());
-                .put(LifecycleSettings.LIFECYCLE_STEP, ErrorStep.NAME))
+        newState.setStep(ErrorStep.NAME);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, newState.build().asMap())
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
 
         runner.runPolicy(policyName, indexMetaData, null, false);
@@ -375,19 +379,18 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
     }
 
     public void testGetCurrentStepKey() {
-        Settings indexSettings = Settings.EMPTY;
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-        StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(indexSettings);
+        StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState.build());
         assertNull(stepKey);
 
         String phase = randomAlphaOfLength(20);
         String action = randomAlphaOfLength(20);
         String step = randomAlphaOfLength(20);
-        Settings indexSettings2 = Settings.builder()
+        LifecycleExecutionState.Builder lifecycleState2 = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_PHASE, phase)
+        lifecycleState2.setPhase(phase);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, action)
+        lifecycleState2.setAction(action);
-                .put(LifecycleSettings.LIFECYCLE_STEP, step)
+        lifecycleState2.setStep(step);
-                .build();
+        stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState2.build());
-        stepKey = IndexLifecycleRunner.getCurrentStepKey(indexSettings2);
         assertNotNull(stepKey);
         assertEquals(phase, stepKey.getPhase());
         assertEquals(action, stepKey.getAction());
@@ -395,46 +398,42 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
 
         phase = randomAlphaOfLength(20);
         action = randomAlphaOfLength(20);
-        step = randomBoolean() ? null : "";
+        step = null;
-        Settings indexSettings3 = Settings.builder()
+        LifecycleExecutionState.Builder lifecycleState3 = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_PHASE, phase)
+        lifecycleState3.setPhase(phase);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, action)
+        lifecycleState3.setAction(action);
-                .put(LifecycleSettings.LIFECYCLE_STEP, step)
+        lifecycleState3.setStep(step);
-                .build();
+        AssertionError error3 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(lifecycleState3.build()));
-        AssertionError error3 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(indexSettings3));
         assertEquals("Current phase is not empty: " + phase, error3.getMessage());
 
-        phase = randomBoolean() ? null : "";
+        phase = null;
         action = randomAlphaOfLength(20);
-        step = randomBoolean() ? null : "";
+        step = null;
-        Settings indexSettings4 = Settings.builder()
+        LifecycleExecutionState.Builder lifecycleState4 = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_PHASE, phase)
+        lifecycleState4.setPhase(phase);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, action)
+        lifecycleState4.setAction(action);
-                .put(LifecycleSettings.LIFECYCLE_STEP, step)
+        lifecycleState4.setStep(step);
-                .build();
+        AssertionError error4 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(lifecycleState4.build()));
-        AssertionError error4 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(indexSettings4));
         assertEquals("Current action is not empty: " + action, error4.getMessage());
 
-        phase = randomBoolean() ? null : "";
+        phase = null;
         action = randomAlphaOfLength(20);
         step = randomAlphaOfLength(20);
-        Settings indexSettings5 = Settings.builder()
+        LifecycleExecutionState.Builder lifecycleState5 = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_PHASE, phase)
+        lifecycleState5.setPhase(phase);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, action)
+        lifecycleState5.setAction(action);
-                .put(LifecycleSettings.LIFECYCLE_STEP, step)
+        lifecycleState5.setStep(step);
-                .build();
+        AssertionError error5 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(lifecycleState5.build()));
-        AssertionError error5 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(indexSettings5));
         assertEquals(null, error5.getMessage());
 
-        phase = randomBoolean() ? null : "";
+        phase = null;
-        action = randomBoolean() ? null : "";
+        action = null;
         step = randomAlphaOfLength(20);
-        Settings indexSettings6 = Settings.builder()
+        LifecycleExecutionState.Builder lifecycleState6 = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_PHASE, phase)
+        lifecycleState6.setPhase(phase);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, action)
+        lifecycleState6.setAction(action);
-                .put(LifecycleSettings.LIFECYCLE_STEP, step)
+        lifecycleState6.setStep(step);
-                .build();
+        AssertionError error6 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(lifecycleState6.build()));
-        AssertionError error6 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(indexSettings6));
         assertEquals(null, error6.getMessage());
     }
 
@@ -464,22 +463,27 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
             .put("index.number_of_replicas", 0)
             .put("index.version.created", Version.CURRENT)
             .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
-            .put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, phaseJson)
-            .put(LifecycleSettings.LIFECYCLE_PHASE, step.getKey().getPhase())
-            .put(LifecycleSettings.LIFECYCLE_ACTION, step.getKey().getAction())
-            .put(LifecycleSettings.LIFECYCLE_STEP, step.getKey().getName())
             .build();
-        IndexMetaData indexMetaData = IndexMetaData.builder(index.getName()).settings(indexSettings).build();
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhaseDefinition(phaseJson);
+        lifecycleState.setPhase(step.getKey().getPhase());
+        lifecycleState.setAction(step.getKey().getAction());
+        lifecycleState.setStep(step.getKey().getName());
+        IndexMetaData indexMetaData = IndexMetaData.builder(index.getName())
+            .settings(indexSettings)
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .build();
         SortedMap<String, LifecyclePolicyMetadata> metas = new TreeMap<>();
         metas.put(policyName, policyMetadata);
         PolicyStepsRegistry registry = new PolicyStepsRegistry(metas, firstStepMap, stepMap, REGISTRY, client);
 
         // First step is retrieved because there are no settings for the index
-        Step stepFromNoSettings = IndexLifecycleRunner.getCurrentStep(registry, policy.getName(), indexMetaData, Settings.EMPTY);
+        Step stepFromNoSettings = IndexLifecycleRunner.getCurrentStep(registry, policy.getName(), indexMetaData,
+            LifecycleExecutionState.builder().build());
         assertEquals(firstStep, stepFromNoSettings);
 
         // The step that was written into the metadata is retrieved
-        Step currentStep = IndexLifecycleRunner.getCurrentStep(registry, policy.getName(), indexMetaData, indexSettings);
+        Step currentStep = IndexLifecycleRunner.getCurrentStep(registry, policy.getName(), indexMetaData, lifecycleState.build());
         assertEquals(step.getKey(), currentStep.getKey());
     }
 
@@ -496,22 +500,25 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
 
         // test going from null lifecycle settings to next step
         ClusterState clusterState = buildClusterState(indexName,
-            Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), policyMetadatas);
+            Settings.builder()
+                .put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), LifecycleExecutionState.builder().build(), policyMetadatas);
         Index index = clusterState.metaData().index(indexName).getIndex();
         ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep,
                 () -> now);
         assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
 
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
         // test going from set currentStep settings to nextStep
         Builder indexSettingsBuilder = Settings.builder()
-            .put(LifecycleSettings.LIFECYCLE_NAME, policy.getName())
+            .put(LifecycleSettings.LIFECYCLE_NAME, policy.getName());
-            .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
-            .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
-            .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName());
         if (randomBoolean()) {
-            indexSettingsBuilder.put(LifecycleSettings.LIFECYCLE_STEP_INFO, randomAlphaOfLength(20));
+            lifecycleState.setStepInfo(randomAlphaOfLength(20));
         }
-        clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
+
+        clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
         index = clusterState.metaData().index(indexName).getIndex();
         newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now);
         assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
@@ -523,20 +530,22 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         StepKey nextStep = new StepKey("current_phase", "next_action", "next_step");
         long now = randomNonNegativeLong();
 
-        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), Collections.emptyList());
+        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), LifecycleExecutionState.builder().build(),
+            Collections.emptyList());
         Index index = clusterState.metaData().index(indexName).getIndex();
         ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep,
                 () -> now);
         assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
 
-        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        lifecycleState.setPhase(currentStep.getPhase());
-                .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
         if (randomBoolean()) {
-            indexSettingsBuilder.put(LifecycleSettings.LIFECYCLE_STEP_INFO, randomAlphaOfLength(20));
+            lifecycleState.setStepInfo(randomAlphaOfLength(20));
         }
-        clusterState = buildClusterState(indexName,
+
-                indexSettingsBuilder, Collections.emptyList());
+        clusterState = buildClusterState(indexName, Settings.builder(), lifecycleState.build(), Collections.emptyList());
         index = clusterState.metaData().index(indexName).getIndex();
         newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now);
         assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
@@ -548,20 +557,21 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         StepKey nextStep = new StepKey("current_phase", "current_action", "next_step");
         long now = randomNonNegativeLong();
 
-        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), Collections.emptyList());
+        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), LifecycleExecutionState.builder().build(),
+            Collections.emptyList());
         Index index = clusterState.metaData().index(indexName).getIndex();
         ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep,
                 () -> now);
         assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
 
-        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        lifecycleState.setPhase(currentStep.getPhase());
-                .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
         if (randomBoolean()) {
-            indexSettingsBuilder.put(LifecycleSettings.LIFECYCLE_STEP_INFO, randomAlphaOfLength(20));
+            lifecycleState.setStepInfo(randomAlphaOfLength(20));
         }
-        clusterState = buildClusterState(indexName,
+        clusterState = buildClusterState(indexName, Settings.builder(), lifecycleState.build(), Collections.emptyList());
-                indexSettingsBuilder, Collections.emptyList());
         index = clusterState.metaData().index(indexName).getIndex();
         newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now);
         assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
@@ -581,11 +591,13 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Step step = new MockStep(nextStepKey, nextStepKey);
         PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step, indexName);
 
-        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-            .put(LifecycleSettings.LIFECYCLE_PHASE, currentStepKey.getPhase())
+        lifecycleState.setPhase(currentStepKey.getPhase());
-            .put(LifecycleSettings.LIFECYCLE_ACTION, currentStepKey.getAction())
+        lifecycleState.setAction(currentStepKey.getAction());
-            .put(LifecycleSettings.LIFECYCLE_STEP, currentStepKey.getName());
+        lifecycleState.setStep(currentStepKey.getName());
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
+
+        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName);
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
         Index index = clusterState.metaData().index(indexName).getIndex();
         ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, currentStepKey,
             nextStepKey, () -> now, stepRegistry);
@@ -601,11 +613,13 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Step step = new MockStep(nextStepKey, nextStepKey);
         PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
 
-        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName);
-            .put(LifecycleSettings.LIFECYCLE_PHASE, currentStepKey.getPhase())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-            .put(LifecycleSettings.LIFECYCLE_ACTION, currentStepKey.getAction())
+        lifecycleState.setPhase(currentStepKey.getPhase());
-            .put(LifecycleSettings.LIFECYCLE_STEP, currentStepKey.getName());
+        lifecycleState.setAction(currentStepKey.getAction());
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList());
+        lifecycleState.setStep(currentStepKey.getName());
+
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
             () -> IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, currentStepKey,
                 nextStepKey, () -> now, stepRegistry));
@@ -622,11 +636,13 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Step step = new MockStep(nextStepKey, nextStepKey);
         PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
 
-        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName);
-            .put(LifecycleSettings.LIFECYCLE_PHASE, currentStepKey.getPhase())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-            .put(LifecycleSettings.LIFECYCLE_ACTION, currentStepKey.getAction())
+        lifecycleState.setPhase(currentStepKey.getPhase());
-            .put(LifecycleSettings.LIFECYCLE_STEP, currentStepKey.getName());
+        lifecycleState.setAction(currentStepKey.getAction());
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList());
+        lifecycleState.setStep(currentStepKey.getName());
+
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
             () -> IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, notCurrentStepKey,
                 nextStepKey, () -> now, stepRegistry));
@@ -643,11 +659,13 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Step step = new MockStep(currentStepKey, nextStepKey);
         PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
 
-        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName);
-            .put(LifecycleSettings.LIFECYCLE_PHASE, currentStepKey.getPhase())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-            .put(LifecycleSettings.LIFECYCLE_ACTION, currentStepKey.getAction())
+        lifecycleState.setPhase(currentStepKey.getPhase());
-            .put(LifecycleSettings.LIFECYCLE_STEP, currentStepKey.getName());
+        lifecycleState.setAction(currentStepKey.getAction());
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList());
+        lifecycleState.setStep(currentStepKey.getName());
+
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
             () -> IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, currentStepKey,
                 nextStepKey, () -> now, stepRegistry));
@@ -662,11 +680,11 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         long now = randomNonNegativeLong();
         Exception cause = new ElasticsearchException("THIS IS AN EXPECTED CAUSE");
 
-        ClusterState clusterState = buildClusterState(indexName,
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                Settings.builder().put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+        lifecycleState.setPhase(currentStep.getPhase());
-                        .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        lifecycleState.setAction(currentStep.getAction());
-                        .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()),
+        lifecycleState.setStep(currentStep.getName());
-                Collections.emptyList());
+        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), lifecycleState.build(), Collections.emptyList());
         Index index = clusterState.metaData().index(indexName).getIndex();
 
         ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToErrorStep(index, clusterState, currentStep, cause, () -> now);
@@ -689,12 +707,13 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Step step = new MockStep(failedStepKey, null);
         PolicyStepsRegistry policyRegistry = createOneStepPolicyStepRegistry(policyName, step, indexName);
         Settings.Builder indexSettingsBuilder = Settings.builder()
-                .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName);
-                .put(LifecycleSettings.LIFECYCLE_PHASE, errorStepKey.getPhase())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_ACTION, errorStepKey.getAction())
+        lifecycleState.setPhase(errorStepKey.getPhase());
-                .put(LifecycleSettings.LIFECYCLE_FAILED_STEP, failedStepKey.getName())
+        lifecycleState.setAction(errorStepKey.getAction());
-                .put(LifecycleSettings.LIFECYCLE_STEP, errorStepKey.getName());
+        lifecycleState.setStep(errorStepKey.getName());
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList());
+        lifecycleState.setFailedStep(failedStepKey.getName());
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
         Index index = clusterState.metaData().index(indexName).getIndex();
         IndexLifecycleRunner runner = new IndexLifecycleRunner(policyRegistry, null, () -> now);
         ClusterState nextClusterState = runner.moveClusterStateToFailedStep(clusterState, indices);
@@ -705,7 +724,8 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
     public void testMoveClusterStateToFailedStepIndexNotFound() {
         String existingIndexName = "my_index";
         String invalidIndexName = "does_not_exist";
-        ClusterState clusterState = buildClusterState(existingIndexName, Settings.builder(), Collections.emptyList());
+        ClusterState clusterState = buildClusterState(existingIndexName, Settings.builder(), LifecycleExecutionState.builder().build(),
+            Collections.emptyList());
         IndexLifecycleRunner runner = new IndexLifecycleRunner(null, null, () -> 0L);
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
             () -> runner.moveClusterStateToFailedStep(clusterState, new String[] { invalidIndexName }));
@@ -722,12 +742,13 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Step step = new MockStep(failedStepKey, null);
         PolicyStepsRegistry policyRegistry = createOneStepPolicyStepRegistry(policyName, step);
         Settings.Builder indexSettingsBuilder = Settings.builder()
-            .put(LifecycleSettings.LIFECYCLE_NAME, (String) null)
+            .put(LifecycleSettings.LIFECYCLE_NAME, (String) null);
-            .put(LifecycleSettings.LIFECYCLE_PHASE, errorStepKey.getPhase())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-            .put(LifecycleSettings.LIFECYCLE_ACTION, errorStepKey.getAction())
+        lifecycleState.setPhase(errorStepKey.getPhase());
-            .put(LifecycleSettings.LIFECYCLE_FAILED_STEP, failedStepKey.getName())
+        lifecycleState.setAction(errorStepKey.getAction());
-            .put(LifecycleSettings.LIFECYCLE_STEP, errorStepKey.getName());
+        lifecycleState.setStep(errorStepKey.getName());
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList());
+        lifecycleState.setFailedStep(failedStepKey.getName());
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
         IndexLifecycleRunner runner = new IndexLifecycleRunner(policyRegistry, null, () -> now);
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
             () -> runner.moveClusterStateToFailedStep(clusterState, indices));
@@ -743,11 +764,12 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Step step = new MockStep(failedStepKey, null);
         PolicyStepsRegistry policyRegistry = createOneStepPolicyStepRegistry(policyName, step);
         Settings.Builder indexSettingsBuilder = Settings.builder()
-            .put(LifecycleSettings.LIFECYCLE_NAME, (String) null)
+            .put(LifecycleSettings.LIFECYCLE_NAME, (String) null);
-            .put(LifecycleSettings.LIFECYCLE_PHASE, failedStepKey.getPhase())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-            .put(LifecycleSettings.LIFECYCLE_ACTION, failedStepKey.getAction())
+        lifecycleState.setPhase(failedStepKey.getPhase());
-            .put(LifecycleSettings.LIFECYCLE_STEP, failedStepKey.getName());
+        lifecycleState.setAction(failedStepKey.getAction());
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList());
+        lifecycleState.setStep(failedStepKey.getName());
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
         IndexLifecycleRunner runner = new IndexLifecycleRunner(policyRegistry, null, () -> now);
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
             () -> runner.moveClusterStateToFailedStep(clusterState, indices));
@@ -760,11 +782,11 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         StepKey currentStep = new StepKey("current_phase", "current_action", "current_step");
         RandomStepInfo stepInfo = new RandomStepInfo(() -> randomAlphaOfLength(10));
 
-        ClusterState clusterState = buildClusterState(indexName,
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                Settings.builder().put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+        lifecycleState.setPhase(currentStep.getPhase());
-                        .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        lifecycleState.setAction(currentStep.getAction());
-                        .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()),
+        lifecycleState.setStep(currentStep.getName());
-                Collections.emptyList());
+        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), lifecycleState.build(), Collections.emptyList());
         Index index = clusterState.metaData().index(indexName).getIndex();
         ClusterState newClusterState = IndexLifecycleRunner.addStepInfoToClusterState(index, clusterState, stepInfo);
         assertClusterStateStepInfo(clusterState, index, currentStep, newClusterState, stepInfo);
@@ -776,13 +798,14 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
     public void testSkipped() {
         String policy = randomAlphaOfLength(5);
         String index = randomAlphaOfLength(10);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(randomAlphaOfLength(5));
+        lifecycleState.setAction(randomAlphaOfLength(5));
+        lifecycleState.setStep(randomAlphaOfLength(5));
         ClusterState clusterState = buildClusterState(index,
-            Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy)
+            Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy).put(LifecycleSettings.LIFECYCLE_SKIP, true),
-                .put(LifecycleSettings.LIFECYCLE_PHASE, randomAlphaOfLength(5))
+            lifecycleState.build(),
-                .put(LifecycleSettings.LIFECYCLE_ACTION, randomAlphaOfLength(5))
+            Collections.emptyList());
-                .put(LifecycleSettings.LIFECYCLE_STEP, randomAlphaOfLength(5))
-                        .put(LifecycleSettings.LIFECYCLE_SKIP, true),
-                Collections.emptyList());
         Step step = mock(randomFrom(TerminalPolicyStep.class, ClusterStateActionStep.class,
             ClusterStateWaitStep.class, AsyncActionStep.class, AsyncWaitStep.class));
         PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policy, step);
@@ -793,11 +816,14 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
     }
 
     private ClusterState buildClusterState(String indexName, Settings.Builder indexSettingsBuilder,
-            List<LifecyclePolicyMetadata> lifecyclePolicyMetadatas) {
+                                           LifecycleExecutionState lifecycleState,
+                                           List<LifecyclePolicyMetadata> lifecyclePolicyMetadatas) {
         Settings indexSettings = indexSettingsBuilder.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexMetaData indexMetadata = IndexMetaData.builder(indexName).settings(indexSettings)
+        IndexMetaData indexMetadata = IndexMetaData.builder(indexName)
-                .build();
+            .settings(indexSettings)
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.asMap())
+            .build();
 
         Map<String, LifecyclePolicyMetadata> lifecyclePolicyMetadatasMap = lifecyclePolicyMetadatas.stream()
                 .collect(Collectors.toMap(LifecyclePolicyMetadata::getName, Function.identity()));
@@ -819,16 +845,18 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
                 new StepKey(phaseName, MockAction.NAME, randomAlphaOfLength(9)), null);
         LifecyclePolicy oldPolicy = createPolicy(oldPolicyName, currentStep, null);
         Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName)
-                .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+                .put(LifecycleSettings.LIFECYCLE_SKIP, true);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true);
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
         List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
         policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong()));
         LifecyclePolicyMetadata newPolicyMetadata = new LifecyclePolicyMetadata(newPolicy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong());
         policyMetadatas.add(newPolicyMetadata);
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
         Index index = clusterState.metaData().index(indexName).getIndex();
         Index[] indices = new Index[] { index };
         List<String> failedIndexes = new ArrayList<>();
@@ -847,9 +875,10 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         LifecyclePolicy newPolicy = newTestLifecyclePolicy(newPolicyName, Collections.emptyMap());
         LifecyclePolicyMetadata newPolicyMetadata = new LifecyclePolicyMetadata(newPolicy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong());
-        StepKey currentStep = new StepKey("", "", "");
+        StepKey currentStep = new StepKey(null, null, null);
         Settings.Builder indexSettingsBuilder = Settings.builder();
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.singletonList(newPolicyMetadata));
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, LifecycleExecutionState.builder().build(),
+            Collections.singletonList(newPolicyMetadata));
         Index index = clusterState.metaData().index(indexName).getIndex();
         Index[] indices = new Index[] { index };
         List<String> failedIndexes = new ArrayList<>();
@@ -872,13 +901,15 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
             randomNonNegativeLong(), randomNonNegativeLong());
         StepKey currentStep = AbstractStepTestCase.randomStepKey();
         Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName)
-                .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+                .put(LifecycleSettings.LIFECYCLE_SKIP, true);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true);
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
         List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
         policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong()));
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
         Index index = new Index("doesnt_exist", "im_not_here");
         Index[] indices = new Index[] { index };
         List<String> failedIndexes = new ArrayList<>();
@@ -922,13 +953,15 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         StepKey currentStep = new StepKey(randomAlphaOfLength(10), MockAction.NAME, randomAlphaOfLength(10));
         LifecyclePolicy oldPolicy = createPolicy(oldPolicyName, currentStep, null);
         Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName)
-                .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+                .put(LifecycleSettings.LIFECYCLE_SKIP, true);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true);
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
         List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
         policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong()));
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
         Index index = clusterState.metaData().index(indexName).getIndex();
         Index[] indices = new Index[] { index };
         List<String> failedIndexes = new ArrayList<>();
@@ -942,7 +975,8 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
     public void testRemovePolicyForIndexNoCurrentPolicy() {
         String indexName = randomAlphaOfLength(10);
         Settings.Builder indexSettingsBuilder = Settings.builder();
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList());
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, LifecycleExecutionState.builder().build(),
+            Collections.emptyList());
         Index index = clusterState.metaData().index(indexName).getIndex();
         Index[] indices = new Index[] { index };
         List<String> failedIndexes = new ArrayList<>();
@@ -959,13 +993,15 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         LifecyclePolicy oldPolicy = newTestLifecyclePolicy(oldPolicyName, Collections.emptyMap());
         StepKey currentStep = AbstractStepTestCase.randomStepKey();
         Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName)
-                .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+                .put(LifecycleSettings.LIFECYCLE_SKIP, true);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true);
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
         List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
         policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong()));
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
         Index index = new Index("doesnt_exist", "im_not_here");
         Index[] indices = new Index[] { index };
         List<String> failedIndexes = new ArrayList<>();
@@ -983,13 +1019,15 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         StepKey currentStep = new StepKey(randomAlphaOfLength(10), MockAction.NAME, randomAlphaOfLength(10));
         LifecyclePolicy oldPolicy = createPolicy(oldPolicyName, null, currentStep);
         Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName)
-                .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+                .put(LifecycleSettings.LIFECYCLE_SKIP, true);
-                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
-                .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true);
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
         List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
         policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong()));
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
         Index index = clusterState.metaData().index(indexName).getIndex();
         Index[] indices = new Index[] { index };
         List<String> failedIndexes = new ArrayList<>();
@@ -1025,11 +1063,14 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         // With no time, always transition
         assertTrue("index should be able to transition with no creation date",
             runner.isReadyToTransitionToThisPhase(policyName, indexMetaData, "phase"));
+
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setIndexCreationDate(10L);
         indexMetaData = IndexMetaData.builder(indexMetaData)
             .settings(Settings.builder()
                 .put(indexMetaData.getSettings())
-                .put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, 10L)
                 .build())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
             .build();
         // Index is not old enough to transition
         assertFalse("index is not able to transition if it isn't old enough",
@@ -1050,15 +1091,6 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Settings indexSettings = indexMetadata.getSettings();
         assertNotNull(indexSettings);
         assertFalse(LifecycleSettings.LIFECYCLE_NAME_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_PHASE_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_ACTION_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_STEP_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING.exists(indexSettings));
-        assertFalse(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING.exists(indexSettings));
         assertFalse(LifecycleSettings.LIFECYCLE_SKIP_SETTING.exists(indexSettings));
         assertFalse(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.exists(indexSettings));
     }
@@ -1070,31 +1102,31 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         assertNotSame(oldClusterState.metaData(), newMetadata);
         IndexMetaData newIndexMetadata = newMetadata.getIndexSafe(index);
         assertNotSame(oldClusterState.metaData().index(index), newIndexMetadata);
-        Settings newIndexSettings = newIndexMetadata.getSettings();
+        LifecycleExecutionState newLifecycleState = LifecycleExecutionState
-        assertNotSame(oldClusterState.metaData().index(index).getSettings(), newIndexSettings);
+            .fromIndexMetadata(newClusterState.metaData().index(index));
-        assertEquals(expectedStep.getPhase(), LifecycleSettings.LIFECYCLE_PHASE_SETTING.get(newIndexSettings));
+        LifecycleExecutionState oldLifecycleState = LifecycleExecutionState
-        assertEquals(expectedStep.getAction(), LifecycleSettings.LIFECYCLE_ACTION_SETTING.get(newIndexSettings));
+            .fromIndexMetadata(oldClusterState.metaData().index(index));
-        assertEquals(expectedStep.getName(), LifecycleSettings.LIFECYCLE_STEP_SETTING.get(newIndexSettings));
+        assertNotSame(oldLifecycleState, newLifecycleState);
-        if (previousStep.getPhase().equals(expectedStep.getPhase())) {
+        assertEquals(expectedStep.getPhase(), newLifecycleState.getPhase());
-            assertEquals(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+        assertEquals(expectedStep.getAction(), newLifecycleState.getAction());
-                    LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newIndexSettings));
+        assertEquals(expectedStep.getName(), newLifecycleState.getStep());
+        if (Objects.equals(previousStep.getPhase(), expectedStep.getPhase())) {
+            assertEquals(oldLifecycleState.getPhase(), newLifecycleState.getPhase());
         } else {
-            assertEquals(now, (long) LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newIndexSettings));
+            assertEquals(now, newLifecycleState.getPhaseTime().longValue());
         }
-        if (previousStep.getAction().equals(expectedStep.getAction())) {
+        if (Objects.equals(previousStep.getAction(), expectedStep.getAction())) {
-            assertEquals(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+            assertEquals(oldLifecycleState.getActionTime(), newLifecycleState.getActionTime());
-                    LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newIndexSettings));
         } else {
-            assertEquals(now, (long) LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newIndexSettings));
+            assertEquals(now, newLifecycleState.getActionTime().longValue());
         }
-        if (previousStep.getName().equals(expectedStep.getName())) {
+        if (Objects.equals(previousStep.getName(), expectedStep.getName())) {
-            assertEquals(LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+            assertEquals(oldLifecycleState.getStepTime(), newLifecycleState.getStepTime());
-                    LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newIndexSettings));
         } else {
-            assertEquals(now, (long) LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newIndexSettings));
+            assertEquals(now, newLifecycleState.getStepTime().longValue());
         }
-        assertEquals("", LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING.get(newIndexSettings));
+        assertEquals(null, newLifecycleState.getFailedStep());
-        assertEquals("", LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newIndexSettings));
+        assertEquals(null, newLifecycleState.getStepInfo());
     }
 
     public static void assertClusterStateOnNextStep(ClusterState oldClusterState, Index index, StepKey currentStep, StepKey nextStep,
@@ -1104,26 +1136,27 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         assertNotSame(oldClusterState.metaData(), newMetadata);
         IndexMetaData newIndexMetadata = newMetadata.getIndexSafe(index);
         assertNotSame(oldClusterState.metaData().index(index), newIndexMetadata);
-        Settings newIndexSettings = newIndexMetadata.getSettings();
+        LifecycleExecutionState newLifecycleState = LifecycleExecutionState
-        assertNotSame(oldClusterState.metaData().index(index).getSettings(), newIndexSettings);
+            .fromIndexMetadata(newClusterState.metaData().index(index));
-        assertEquals(nextStep.getPhase(), LifecycleSettings.LIFECYCLE_PHASE_SETTING.get(newIndexSettings));
+        LifecycleExecutionState oldLifecycleState = LifecycleExecutionState
-        assertEquals(nextStep.getAction(), LifecycleSettings.LIFECYCLE_ACTION_SETTING.get(newIndexSettings));
+            .fromIndexMetadata(oldClusterState.metaData().index(index));
-        assertEquals(nextStep.getName(), LifecycleSettings.LIFECYCLE_STEP_SETTING.get(newIndexSettings));
+        assertNotSame(oldLifecycleState, newLifecycleState);
+        assertEquals(nextStep.getPhase(), newLifecycleState.getPhase());
+        assertEquals(nextStep.getAction(), newLifecycleState.getAction());
+        assertEquals(nextStep.getName(), newLifecycleState.getStep());
         if (currentStep.getPhase().equals(nextStep.getPhase())) {
-            assertEquals(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+            assertEquals(oldLifecycleState.getPhaseTime(), newLifecycleState.getPhaseTime());
-                    LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newIndexSettings));
         } else {
-            assertEquals(now, (long) LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newIndexSettings));
+            assertEquals(now, newLifecycleState.getPhaseTime().longValue());
         }
         if (currentStep.getAction().equals(nextStep.getAction())) {
-            assertEquals(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+            assertEquals(oldLifecycleState.getActionTime(), newLifecycleState.getActionTime());
-                    LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newIndexSettings));
         } else {
-            assertEquals(now, (long) LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newIndexSettings));
+            assertEquals(now, newLifecycleState.getActionTime().longValue());
         }
-        assertEquals(now, (long) LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newIndexSettings));
+        assertEquals(now, newLifecycleState.getStepTime().longValue());
-        assertEquals("", LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING.get(newIndexSettings));
+        assertEquals(null, newLifecycleState.getFailedStep());
-        assertEquals("", LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newIndexSettings));
+        assertEquals(null, newLifecycleState.getStepInfo());
     }
 
     private void assertClusterStateOnErrorStep(ClusterState oldClusterState, Index index, StepKey currentStep,
@@ -1133,18 +1166,19 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         assertNotSame(oldClusterState.metaData(), newMetadata);
         IndexMetaData newIndexMetadata = newMetadata.getIndexSafe(index);
         assertNotSame(oldClusterState.metaData().index(index), newIndexMetadata);
-        Settings newIndexSettings = newIndexMetadata.getSettings();
+        LifecycleExecutionState newLifecycleState = LifecycleExecutionState
-        assertNotSame(oldClusterState.metaData().index(index).getSettings(), newIndexSettings);
+            .fromIndexMetadata(newClusterState.metaData().index(index));
-        assertEquals(currentStep.getPhase(), LifecycleSettings.LIFECYCLE_PHASE_SETTING.get(newIndexSettings));
+        LifecycleExecutionState oldLifecycleState = LifecycleExecutionState
-        assertEquals(currentStep.getAction(), LifecycleSettings.LIFECYCLE_ACTION_SETTING.get(newIndexSettings));
+            .fromIndexMetadata(oldClusterState.metaData().index(index));
-        assertEquals(ErrorStep.NAME, LifecycleSettings.LIFECYCLE_STEP_SETTING.get(newIndexSettings));
+        assertNotSame(oldLifecycleState, newLifecycleState);
-        assertEquals(currentStep.getName(), LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING.get(newIndexSettings));
+        assertEquals(currentStep.getPhase(), newLifecycleState.getPhase());
-        assertEquals(expectedCauseValue, LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newIndexSettings));
+        assertEquals(currentStep.getAction(), newLifecycleState.getAction());
-        assertEquals(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+        assertEquals(ErrorStep.NAME, newLifecycleState.getStep());
-                LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newIndexSettings));
+        assertEquals(currentStep.getName(), newLifecycleState.getFailedStep());
-        assertEquals(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+        assertEquals(expectedCauseValue, newLifecycleState.getStepInfo());
-                LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newIndexSettings));
+        assertEquals(oldLifecycleState.getPhaseTime(), newLifecycleState.getPhaseTime());
-        assertEquals(now, (long) LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newIndexSettings));
+        assertEquals(oldLifecycleState.getActionTime(), newLifecycleState.getActionTime());
+        assertEquals(now, newLifecycleState.getStepTime().longValue());
     }
 
     private void assertClusterStateStepInfo(ClusterState oldClusterState, Index index, StepKey currentStep, ClusterState newClusterState,
@@ -1157,18 +1191,18 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         assertNotSame(oldClusterState.metaData(), newMetadata);
         IndexMetaData newIndexMetadata = newMetadata.getIndexSafe(index);
         assertNotSame(oldClusterState.metaData().index(index), newIndexMetadata);
-        Settings newIndexSettings = newIndexMetadata.getSettings();
+        LifecycleExecutionState newLifecycleState = LifecycleExecutionState
-        assertNotSame(oldClusterState.metaData().index(index).getSettings(), newIndexSettings);
+            .fromIndexMetadata(newClusterState.metaData().index(index));
-        assertEquals(currentStep.getPhase(), LifecycleSettings.LIFECYCLE_PHASE_SETTING.get(newIndexSettings));
+        LifecycleExecutionState oldLifecycleState = LifecycleExecutionState
-        assertEquals(currentStep.getAction(), LifecycleSettings.LIFECYCLE_ACTION_SETTING.get(newIndexSettings));
+            .fromIndexMetadata(oldClusterState.metaData().index(index));
-        assertEquals(currentStep.getName(), LifecycleSettings.LIFECYCLE_STEP_SETTING.get(newIndexSettings));
+        assertNotSame(oldLifecycleState, newLifecycleState);
-        assertEquals(expectedstepInfoValue, LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newIndexSettings));
+        assertEquals(currentStep.getPhase(), newLifecycleState.getPhase());
-        assertEquals(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+        assertEquals(currentStep.getAction(), newLifecycleState.getAction());
-                LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newIndexSettings));
+        assertEquals(currentStep.getName(), newLifecycleState.getStep());
-        assertEquals(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+        assertEquals(expectedstepInfoValue, newLifecycleState.getStepInfo());
-                LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newIndexSettings));
+        assertEquals(oldLifecycleState.getPhaseTime(), newLifecycleState.getPhaseTime());
-        assertEquals(LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(oldClusterState.metaData().index(index).getSettings()),
+        assertEquals(oldLifecycleState.getActionTime(), newLifecycleState.getActionTime());
-                LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newIndexSettings));
+        assertEquals(newLifecycleState.getStepTime(), newLifecycleState.getStepTime());
     }
 
     private static class MockAsyncActionStep extends AsyncActionStep {

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java

@@ -24,13 +24,14 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.xpack.core.indexlifecycle.Phase;
 import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
@@ -49,6 +50,7 @@ import java.util.concurrent.ExecutorService;
 
 import static org.elasticsearch.node.Node.NODE_MASTER_SETTING;
 import static org.elasticsearch.xpack.core.indexlifecycle.AbstractStepTestCase.randomStepKey;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.newTestLifecyclePolicy;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
@@ -148,11 +150,13 @@ public class IndexLifecycleServiceTests extends ESTestCase {
         policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong()));
         Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(mockShrinkStep.getPhase());
+        lifecycleState.setAction(mockShrinkStep.getAction());
+        lifecycleState.setStep(mockShrinkStep.getName());
         IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
-            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName)
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
-                .put(LifecycleSettings.LIFECYCLE_PHASE, mockShrinkStep.getPhase())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
-                .put(LifecycleSettings.LIFECYCLE_ACTION, mockShrinkStep.getAction())
-                .put(LifecycleSettings.LIFECYCLE_STEP, mockShrinkStep.getName()))
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
         ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
             .fPut(index.getName(), indexMetadata);
@@ -189,11 +193,13 @@ public class IndexLifecycleServiceTests extends ESTestCase {
         policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap(),
             randomNonNegativeLong(), randomNonNegativeLong()));
         Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStepKey.getPhase());
+        lifecycleState.setAction(currentStepKey.getAction());
+        lifecycleState.setStep(currentStepKey.getName());
         IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
-            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName)
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
-                .put(LifecycleSettings.LIFECYCLE_PHASE, currentStepKey.getPhase())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
-                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStepKey.getAction())
-                .put(LifecycleSettings.LIFECYCLE_STEP, currentStepKey.getName()))
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
         ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
             .fPut(index.getName(), indexMetadata);

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java

@@ -19,6 +19,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
@@ -31,7 +32,9 @@ import org.junit.Before;
 import java.io.IOException;
 import java.util.Collections;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class MoveToErrorStepUpdateTaskTests extends ESTestCase {
@@ -70,21 +73,20 @@ public class MoveToErrorStepUpdateTaskTests extends ESTestCase {
 
         MoveToErrorStepUpdateTask task = new MoveToErrorStepUpdateTask(index, policy, currentStepKey, cause, () -> now);
         ClusterState newState = task.execute(clusterState);
-        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(newState.metaData().index(index).getSettings());
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
         assertThat(actualKey, equalTo(new StepKey(currentStepKey.getPhase(), currentStepKey.getAction(), ErrorStep.NAME)));
-        assertThat(LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING.get(newState.metaData().index(index).getSettings()),
+        assertThat(lifecycleState.getFailedStep(), equalTo(currentStepKey.getName()));
-                equalTo(currentStepKey.getName()));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getActionTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getStepTime(), equalTo(now));
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(now));
 
         XContentBuilder causeXContentBuilder = JsonXContent.contentBuilder();
         causeXContentBuilder.startObject();
         ElasticsearchException.generateThrowableXContent(causeXContentBuilder, ToXContent.EMPTY_PARAMS, cause);
         causeXContentBuilder.endObject();
         String expectedCauseValue = BytesReference.bytes(causeXContentBuilder).utf8ToString();
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newState.metaData().index(index).getSettings()),
+        assertThat(lifecycleState.getStepInfo(), equalTo(expectedCauseValue));
-                equalTo(expectedCauseValue));
     }
 
     public void testExecuteNoopDifferentStep() throws IOException {
@@ -133,11 +135,15 @@ public class MoveToErrorStepUpdateTaskTests extends ESTestCase {
 
     }
     private void setStateToKey(StepKey stepKey) {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.metaData().index(index)));
+        lifecycleState.setPhase(stepKey.getPhase());
+        lifecycleState.setAction(stepKey.getAction());
+        lifecycleState.setStep(stepKey.getName());
+
         clusterState = ClusterState.builder(clusterState)
-            .metaData(MetaData.builder(clusterState.metaData())
+            .metaData(MetaData.builder(clusterState.getMetaData())
-                .updateSettings(Settings.builder()
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
-                    .put(LifecycleSettings.LIFECYCLE_PHASE, stepKey.getPhase())
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
-                    .put(LifecycleSettings.LIFECYCLE_ACTION, stepKey.getAction())
-                    .put(LifecycleSettings.LIFECYCLE_STEP, stepKey.getName()).build(), index.getName())).build();
     }
 }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java

@@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
@@ -28,6 +29,7 @@ import org.junit.Before;
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.hamcrest.Matchers.equalTo;
 
 public class MoveToNextStepUpdateTaskTests extends ESTestCase {
@@ -68,11 +70,12 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
 
         MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, nextStepKey, () -> now);
         ClusterState newState = task.execute(clusterState);
-        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(newState.metaData().index(index).getSettings());
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
         assertThat(actualKey, equalTo(nextStepKey));
-        assertThat(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(now));
+        assertThat(lifecycleState.getPhaseTime(), equalTo(now));
-        assertThat(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(now));
+        assertThat(lifecycleState.getActionTime(), equalTo(now));
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(now));
+        assertThat(lifecycleState.getStepTime(), equalTo(now));
         task.clusterStateProcessed("source", clusterState, newState);
     }
 
@@ -107,11 +110,12 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
         SetOnce<Boolean> changed = new SetOnce<>();
         MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, invalidNextStep, () -> now);
         ClusterState newState = task.execute(clusterState);
-        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(newState.metaData().index(index).getSettings());
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
         assertThat(actualKey, equalTo(invalidNextStep));
-        assertThat(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(now));
+        assertThat(lifecycleState.getPhaseTime(), equalTo(now));
-        assertThat(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(now));
+        assertThat(lifecycleState.getActionTime(), equalTo(now));
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(now));
+        assertThat(lifecycleState.getStepTime(), equalTo(now));
         task.clusterStateProcessed("source", clusterState, newState);
     }
 
@@ -139,15 +143,18 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
 
     }
     private void setStateToKey(StepKey stepKey, long now) {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.metaData().index(index)));
+        lifecycleState.setPhase(stepKey.getPhase());
+        lifecycleState.setPhaseTime(now);
+        lifecycleState.setAction(stepKey.getAction());
+        lifecycleState.setActionTime(now);
+        lifecycleState.setStep(stepKey.getName());
+        lifecycleState.setStepTime(now);
+        lifecycleState.setPhaseDefinition("{\"actions\":{\"TEST_ACTION\":{}}}");
         clusterState = ClusterState.builder(clusterState)
-            .metaData(MetaData.builder(clusterState.metaData())
+            .metaData(MetaData.builder(clusterState.getMetaData())
-                .updateSettings(Settings.builder()
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
-                    .put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, "{\"actions\":{\"TEST_ACTION\":{}}}")
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
-                    .put(LifecycleSettings.LIFECYCLE_PHASE, stepKey.getPhase())
-                    .put(LifecycleSettings.LIFECYCLE_PHASE_TIME, now)
-                    .put(LifecycleSettings.LIFECYCLE_ACTION, stepKey.getAction())
-                    .put(LifecycleSettings.LIFECYCLE_ACTION_TIME, now)
-                    .put(LifecycleSettings.LIFECYCLE_STEP, stepKey.getName())
-                    .put(LifecycleSettings.LIFECYCLE_STEP_TIME, now).build(), index.getName())).build();
     }
 }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java

@@ -25,6 +25,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.InitializePolicyContextStep;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
@@ -48,6 +49,7 @@ import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
@@ -91,14 +93,16 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         String phaseJson = Strings.toString(pei);
         LifecycleAction action = randomFrom(phase.getActions().values());
         Step step = randomFrom(action.toSteps(client, phaseName, MOCK_STEP_KEY));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhaseDefinition(phaseJson);
         IndexMetaData indexMetaData = IndexMetaData.builder("test")
             .settings(Settings.builder()
                 .put("index.number_of_shards", 1)
                 .put("index.number_of_replicas", 0)
                 .put("index.version.created", Version.CURRENT)
                 .put(LifecycleSettings.LIFECYCLE_NAME, "policy")
-                .put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, phaseJson)
                 .build())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
             .build();
         SortedMap<String, LifecyclePolicyMetadata> metas = new TreeMap<>();
         metas.put("policy", policyMetadata);
@@ -157,14 +161,16 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         String phaseJson = Strings.toString(pei);
         LifecycleAction action = randomFrom(phase.getActions().values());
         Step step = randomFrom(action.toSteps(client, phaseName, MOCK_STEP_KEY));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhaseDefinition(phaseJson);
         IndexMetaData indexMetaData = IndexMetaData.builder("test")
             .settings(Settings.builder()
                 .put("index.number_of_shards", 1)
                 .put("index.number_of_replicas", 0)
                 .put("index.version.created", Version.CURRENT)
                 .put(LifecycleSettings.LIFECYCLE_NAME, "policy")
-                .put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, phaseJson)
                 .build())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
             .build();
         SortedMap<String, LifecyclePolicyMetadata> metas = new TreeMap<>();
         metas.put("policy", policyMetadata);
@@ -190,6 +196,8 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         Map<String, LifecyclePolicyMetadata> policyMap = Collections.singletonMap(newPolicy.getName(),
                 new LifecyclePolicyMetadata(newPolicy, headers, randomNonNegativeLong(), randomNonNegativeLong()));
         IndexLifecycleMetadata lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase("new");
         MetaData metaData = MetaData.builder()
             .persistentSettings(settings(Version.CURRENT).build())
             .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)
@@ -199,8 +207,8 @@ public class PolicyStepsRegistryTests extends ESTestCase {
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .put("index.version.created", Version.CURRENT.id)
-                    .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+                    .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
-                    .put(LifecycleSettings.LIFECYCLE_PHASE, "new")))
+                .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))
             .build();
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             builder.startObject();
@@ -232,11 +240,13 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         Map<Step.StepKey, Step> registeredStepsForPolicy = registry.getStepMap().get(newPolicy.getName());
         assertThat(registeredStepsForPolicy.size(), equalTo(policySteps.size()));
         for (Step step : policySteps) {
+            LifecycleExecutionState.Builder newIndexState = LifecycleExecutionState.builder();
+            newIndexState.setPhase(step.getKey().getPhase());
             currentState = ClusterState.builder(currentState)
                 .metaData(MetaData.builder(currentState.metaData())
                     .put(IndexMetaData.builder(currentState.metaData().index("test"))
-                        .settings(Settings.builder().put(currentState.metaData().index("test").getSettings())
+                        .settings(Settings.builder().put(currentState.metaData().index("test").getSettings()))
-                            .put(LifecycleSettings.LIFECYCLE_PHASE, step.getKey().getPhase()))))
+                        .putCustom(ILM_CUSTOM_METADATA_KEY, newIndexState.build().asMap())))
                 .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
                 .build();
             registry.update(currentState);
@@ -334,6 +344,9 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         Map<String, LifecyclePolicyMetadata> policyMap = Collections.singletonMap(newPolicy.getName(),
             new LifecyclePolicyMetadata(newPolicy, headers, randomNonNegativeLong(), randomNonNegativeLong()));
         IndexLifecycleMetadata lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase("warm");
+        lifecycleState.setPhaseDefinition(phaseJson);
         MetaData metaData = MetaData.builder()
             .persistentSettings(settings(Version.CURRENT).build())
             .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)
@@ -343,9 +356,8 @@ public class PolicyStepsRegistryTests extends ESTestCase {
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .put("index.version.created", Version.CURRENT.id)
-                    .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+                    .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
-                    .put(LifecycleSettings.LIFECYCLE_PHASE, "warm")
+                .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))
-                    .put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, phaseJson)))
             .build();
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             builder.startObject();

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTaskTests.java

@@ -20,13 +20,16 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 import org.junit.Before;
 
 import java.io.IOException;
 
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class SetStepInfoUpdateTaskTests extends ESTestCase {
@@ -57,17 +60,17 @@ public class SetStepInfoUpdateTaskTests extends ESTestCase {
 
         SetStepInfoUpdateTask task = new SetStepInfoUpdateTask(index, policy, currentStepKey, stepInfo);
         ClusterState newState = task.execute(clusterState);
-        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(newState.metaData().index(index).getSettings());
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
         assertThat(actualKey, equalTo(currentStepKey));
-        assertThat(LifecycleSettings.LIFECYCLE_PHASE_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_ACTION_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getActionTime(), nullValue());
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_TIME_SETTING.get(newState.metaData().index(index).getSettings()), equalTo(-1L));
+        assertThat(lifecycleState.getStepTime(), nullValue());
 
         XContentBuilder infoXContentBuilder = JsonXContent.contentBuilder();
         stepInfo.toXContent(infoXContentBuilder, ToXContent.EMPTY_PARAMS);
         String expectedCauseValue = BytesReference.bytes(infoXContentBuilder).utf8ToString();
-        assertThat(LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING.get(newState.metaData().index(index).getSettings()),
+        assertThat(lifecycleState.getStepInfo(), equalTo(expectedCauseValue));
-                equalTo(expectedCauseValue));
     }
 
     private ToXContentObject getRandomStepInfo() {
@@ -124,11 +127,15 @@ public class SetStepInfoUpdateTaskTests extends ESTestCase {
 
     }
     private void setStateToKey(StepKey stepKey) {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.metaData().index(index)));
+        lifecycleState.setPhase(stepKey.getPhase());
+        lifecycleState.setAction(stepKey.getAction());
+        lifecycleState.setStep(stepKey.getName());
+
         clusterState = ClusterState.builder(clusterState)
-            .metaData(MetaData.builder(clusterState.metaData())
+            .metaData(MetaData.builder(clusterState.getMetaData())
-                .updateSettings(Settings.builder()
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
-                    .put(LifecycleSettings.LIFECYCLE_PHASE, stepKey.getPhase())
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
-                    .put(LifecycleSettings.LIFECYCLE_ACTION, stepKey.getAction())
-                    .put(LifecycleSettings.LIFECYCLE_STEP, stepKey.getName()).build(), index.getName())).build();
     }
 }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java

@@ -9,11 +9,14 @@ package org.elasticsearch.xpack.indexlifecycle;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
@@ -30,6 +33,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -75,8 +79,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         createNewSingletonPolicy(randomFrom("warm", "cold"), allocateAction);
         updatePolicy(index, policy);
         assertBusy(() -> {
-            Map<String, Object> settings = getOnlyIndexSettings(index);
+            assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
-            assertThat(getStepKey(settings), equalTo(TerminalPolicyStep.KEY));
         });
         ensureGreen(index);
     }
@@ -92,7 +95,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         updatePolicy(index, policy);
         assertBusy(() -> {
             Map<String, Object> settings = getOnlyIndexSettings(index);
-            assertThat(getStepKey(settings), equalTo(TerminalPolicyStep.KEY));
+            assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
             assertThat(settings.get(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey()), equalTo(String.valueOf(finalNumReplicas)));
         });
     }
@@ -112,7 +115,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         updatePolicy(index, policy);
         assertBusy(() -> {
             Map<String, Object> settings = getOnlyIndexSettings(index);
-            assertThat(getStepKey(settings), equalTo(TerminalPolicyStep.KEY));
+            assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
             assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true"));
         });
     }
@@ -146,7 +149,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         updatePolicy(index, policy);
 
         assertBusy(() -> {
-            assertThat(getStepKey(getOnlyIndexSettings(index)), equalTo(TerminalPolicyStep.KEY));
+            assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
             assertThat(numSegments.get(), equalTo(1));
         });
     }
@@ -164,7 +167,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
             assertTrue(indexExists(shrunkenIndex));
             assertTrue(aliasExists(shrunkenIndex, index));
             Map<String, Object> settings = getOnlyIndexSettings(shrunkenIndex);
-            assertThat(getStepKey(settings), equalTo(TerminalPolicyStep.KEY));
+            assertThat(getStepKeyForIndex(shrunkenIndex), equalTo(TerminalPolicyStep.KEY));
             assertThat(settings.get(IndexMetaData.SETTING_NUMBER_OF_SHARDS), equalTo(String.valueOf(expectedFinalShards)));
         });
     }
@@ -198,10 +201,22 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         return (Map<String, Object>) response.get("settings");
     }
 
-    private StepKey getStepKey(Map<String, Object> settings) {
+    private StepKey getStepKeyForIndex(String indexName) throws IOException {
-        String phase = (String) settings.get(LifecycleSettings.LIFECYCLE_PHASE);
+        Request explainRequest = new Request("GET", indexName + "/_ilm/explain");
-        String action = (String) settings.get(LifecycleSettings.LIFECYCLE_ACTION);
+        Response response = client().performRequest(explainRequest);
-        String step = (String) settings.get(LifecycleSettings.LIFECYCLE_STEP);
+        Map<String, Object> responseMap;
+        try (InputStream is = response.getEntity().getContent()) {
+            responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+        }
+
+        @SuppressWarnings("unchecked") Map<String, String> indexResponse = ((Map<String, Map<String, String>>) responseMap.get("indices"))
+            .get(indexName);
+        if (indexResponse == null) {
+            return new StepKey(null, null, null);
+        }
+        String phase = indexResponse.get("phase");
+        String action = indexResponse.get("action");
+        String step = indexResponse.get("step");
         return new StepKey(phase, action, step);
     }
 }

x-pack/plugin/src/test/resources/rest-api-spec/test/ilm/20_move_to_step.yml

@@ -84,13 +84,13 @@ teardown:
             name: "forcemerge"
 
   - do:
-      acknowledge: true
+        acknowledge: true
-      indices.get:
+        ilm.explain_lifecycle:
-        index: my_index
+          index: "my_index"
-  - match: { my_index.settings.index.lifecycle.name: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
-  - match: { my_index.settings.index.lifecycle.step: "forcemerge" }
+  - match: { indices.my_index.step: "forcemerge" }
-  - match: { my_index.settings.index.lifecycle.action: "forcemerge" }
+  - match: { indices.my_index.action: "forcemerge" }
-  - match: { my_index.settings.index.lifecycle.phase: "warm" }
+  - match: { indices.my_index.phase: "warm" }
 
 ---
 "Test Invalid Move To Step With Incorrect Current Step":
@@ -111,14 +111,15 @@ teardown:
   - match: { error.root_cause.0.type: "illegal_argument_exception" }
   - match: { error.root_cause.0.reason: "index [my_index] is not on current step [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"forcemerge\"}]" }
 
+
   - do:
       acknowledge: true
-      indices.get:
+      ilm.explain_lifecycle:
-        index: my_index
+        index: "my_index"
-  - match: { my_index.settings.index.lifecycle.name: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
-  - match: { my_index.settings.index.lifecycle.step: "complete" }
+  - match: { indices.my_index.step: "complete" }
-  - match: { my_index.settings.index.lifecycle.action: "complete" }
+  - match: { indices.my_index.action: "complete" }
-  - match: { my_index.settings.index.lifecycle.phase: "new" }
+  - match: { indices.my_index.phase: "new" }
 
 ---
 "Test Invalid Move To Step With Invalid Next Step":
@@ -141,12 +142,12 @@ teardown:
 
   - do:
       acknowledge: true
-      indices.get:
+      ilm.explain_lifecycle:
-        index: my_index
+        index: "my_index"
-  - match: { my_index.settings.index.lifecycle.name: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
-  - match: { my_index.settings.index.lifecycle.step: "complete" }
+  - match: { indices.my_index.step: "complete" }
-  - match: { my_index.settings.index.lifecycle.action: "complete" }
+  - match: { indices.my_index.action: "complete" }
-  - match: { my_index.settings.index.lifecycle.phase: "new" }
+  - match: { indices.my_index.phase: "new" }
 
 ---
 "Test Invalid Move To Step With Invalid Policy":

x-pack/plugin/src/test/resources/rest-api-spec/test/ilm/30_retry.yml

@@ -70,12 +70,12 @@ teardown:
 
   - do:
       acknowledge: true
-      indices.get:
+      ilm.explain_lifecycle:
-        index: my_index
+        index: "my_index"
-  - match: { my_index.settings.index.lifecycle.name: "my_lifecycle" }
+  - match: { indices.my_index.policy: "my_lifecycle" }
-  - match: { my_index.settings.index.lifecycle.step: "complete" }
+  - match: { indices.my_index.step: "complete" }
-  - match: { my_index.settings.index.lifecycle.action: "complete" }
+  - match: { indices.my_index.action: "complete" }
-  - match: { my_index.settings.index.lifecycle.phase: "new" }
+  - match: { indices.my_index.phase: "new" }
 
 
 ---