add new phase definition setting used for retrieving phase to execute (#33289)

Since policies can be updated independent of execution plans for the current
phase being executed, it would be nice to know what the phase that is executing
looks like in JSON. This PR does just that, while also using that index setting
to recontruct the phase steps to execute (for consistency)

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java

@@ -12,7 +12,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 
 public final class InitializePolicyContextStep extends ClusterStateActionStep {
-    public static final StepKey KEY = new StepKey("new", "init", "init");
+    public static final String INITIALIZATION_PHASE = "new";
+    public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
 
     public InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
         super(key, nextStepKey);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicy.java

@@ -90,7 +90,7 @@ public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
      *            a {@link Map} of {@link Phase}s which make up this
      *            {@link LifecyclePolicy}.
      */
-    LifecyclePolicy(LifecycleType type, String name, Map<String, Phase> phases) {
+    public LifecyclePolicy(LifecycleType type, String name, Map<String, Phase> phases) {
         this.name = name;
         this.phases = phases;
         this.type = type;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java

@@ -24,6 +24,7 @@ public class LifecycleSettings {
     public static final String LIFECYCLE_FAILED_STEP = "index.lifecycle.failed_step";
     public static final String LIFECYCLE_STEP_INFO = "index.lifecycle.step_info";
     public static final String LIFECYCLE_SKIP = "index.lifecycle.skip";
+    public static final String LIFECYCLE_PHASE_DEFINITION = "index.lifecycle.phase_definition";
 
     public static final Setting<TimeValue> LIFECYCLE_POLL_INTERVAL_SETTING = Setting.positiveTimeSetting(LIFECYCLE_POLL_INTERVAL,
         TimeValue.timeValueMinutes(10), Setting.Property.Dynamic, Setting.Property.NodeScope);
@@ -49,4 +50,6 @@ public class LifecycleSettings {
             Setting.Property.IndexScope, Setting.Property.NotCopyableOnResize, Setting.Property.InternalIndex);
     public static final Setting<Boolean> LIFECYCLE_SKIP_SETTING = Setting.boolSetting(LIFECYCLE_SKIP, false,
         Setting.Property.Dynamic, Setting.Property.IndexScope);
+    public static final Setting<String> LIFECYCLE_PHASE_DEFINITION_SETTING = Setting.simpleString(LIFECYCLE_PHASE_DEFINITION,
+        Setting.Property.Dynamic, Setting.Property.IndexScope);
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStep.java

@@ -6,7 +6,8 @@
 package org.elasticsearch.xpack.core.indexlifecycle;
 
 public class TerminalPolicyStep extends Step {
-    public static final StepKey KEY = new StepKey("completed", "completed", "completed");
+    public static final String COMPLETED_PHASE = "completed";
+    public static final StepKey KEY = new StepKey(COMPLETED_PHASE, "completed", "completed");
     public static final TerminalPolicyStep INSTANCE = new TerminalPolicyStep(KEY, null);
 
     TerminalPolicyStep(StepKey key, StepKey nextStepKey) {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java

@@ -133,6 +133,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin {
             LifecycleSettings.LIFECYCLE_STEP_INFO_SETTING,
             LifecycleSettings.LIFECYCLE_FAILED_STEP_SETTING,
             LifecycleSettings.LIFECYCLE_SKIP_SETTING,
+            LifecycleSettings.LIFECYCLE_PHASE_DEFINITION_SETTING,
             RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING);
     }
 
@@ -145,7 +146,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin {
             return emptyList();
         }
         indexLifecycleInitialisationService
-            .set(new IndexLifecycleService(settings, client, clusterService, getClock(), System::currentTimeMillis));
+            .set(new IndexLifecycleService(settings, client, clusterService, getClock(), System::currentTimeMillis, xContentRegistry));
         return Collections.singletonList(indexLifecycleInitialisationService.get());
     }
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java

@@ -26,8 +26,10 @@ import org.elasticsearch.xpack.core.indexlifecycle.AsyncWaitStep;
 import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateActionStep;
 import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep;
 import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
 import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
@@ -206,7 +208,9 @@ public class IndexLifecycleRunner {
     static ClusterState moveClusterStateToNextStep(Index index, ClusterState clusterState, StepKey currentStep, StepKey nextStep,
             LongSupplier nowSupplier) {
         IndexMetaData idxMeta = clusterState.getMetaData().index(index);
-        Settings.Builder indexSettings = moveIndexSettingsToNextStep(idxMeta.getSettings(), currentStep, nextStep, nowSupplier);
+        IndexLifecycleMetadata ilmMeta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
+        LifecyclePolicy policy = ilmMeta.getPolicies().get(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()));
+        Settings.Builder indexSettings = moveIndexSettingsToNextStep(policy, idxMeta.getSettings(), currentStep, nextStep, nowSupplier);
         ClusterState.Builder newClusterStateBuilder = newClusterStateWithIndexSettings(index, clusterState, indexSettings);
         return newClusterStateBuilder.build();
     }
@@ -214,11 +218,13 @@ public class IndexLifecycleRunner {
     static ClusterState moveClusterStateToErrorStep(Index index, ClusterState clusterState, StepKey currentStep, Exception cause,
             LongSupplier nowSupplier) throws IOException {
         IndexMetaData idxMeta = clusterState.getMetaData().index(index);
+        IndexLifecycleMetadata ilmMeta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
+        LifecyclePolicy policy = ilmMeta.getPolicies().get(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()));
         XContentBuilder causeXContentBuilder = JsonXContent.contentBuilder();
         causeXContentBuilder.startObject();
         ElasticsearchException.generateThrowableXContent(causeXContentBuilder, ToXContent.EMPTY_PARAMS, cause);
         causeXContentBuilder.endObject();
-        Settings.Builder indexSettings = moveIndexSettingsToNextStep(idxMeta.getSettings(), currentStep,
+        Settings.Builder indexSettings = moveIndexSettingsToNextStep(policy, idxMeta.getSettings(), currentStep,
                 new StepKey(currentStep.getPhase(), currentStep.getAction(), ErrorStep.NAME), nowSupplier)
                         .put(LifecycleSettings.LIFECYCLE_FAILED_STEP, currentStep.getName())
                         .put(LifecycleSettings.LIFECYCLE_STEP_INFO, BytesReference.bytes(causeXContentBuilder).utf8ToString());
@@ -247,8 +253,8 @@ public class IndexLifecycleRunner {
         return newState;
     }
 
-    private static Settings.Builder moveIndexSettingsToNextStep(Settings existingSettings, StepKey currentStep, StepKey nextStep,
+    private static Settings.Builder moveIndexSettingsToNextStep(LifecyclePolicy policy, Settings existingSettings,
-            LongSupplier nowSupplier) {
+                                                                StepKey currentStep, StepKey nextStep, LongSupplier nowSupplier) {
         long nowAsMillis = nowSupplier.getAsLong();
         Settings.Builder newSettings = Settings.builder().put(existingSettings).put(LifecycleSettings.LIFECYCLE_PHASE, nextStep.getPhase())
                 .put(LifecycleSettings.LIFECYCLE_ACTION, nextStep.getAction()).put(LifecycleSettings.LIFECYCLE_STEP, nextStep.getName())
@@ -257,6 +263,18 @@ public class IndexLifecycleRunner {
                 .put(LifecycleSettings.LIFECYCLE_FAILED_STEP, (String) null)
                 .put(LifecycleSettings.LIFECYCLE_STEP_INFO, (String) null);
         if (currentStep.getPhase().equals(nextStep.getPhase()) == false) {
+            final String newPhaseDefinition;
+            if ("new".equals(nextStep.getPhase()) || TerminalPolicyStep.KEY.equals(nextStep)) {
+                newPhaseDefinition = nextStep.getPhase();
+            } else {
+                Phase nextPhase = policy.getPhases().get(nextStep.getPhase());
+                if (nextPhase == null) {
+                    newPhaseDefinition = null;
+                } else {
+                    newPhaseDefinition = Strings.toString(nextPhase, false, false);
+                }
+            }
+            newSettings.put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, newPhaseDefinition);
             newSettings.put(LifecycleSettings.LIFECYCLE_PHASE_TIME, nowAsMillis);
         }
         if (currentStep.getAction().equals(nextStep.getAction()) == false) {
@@ -356,7 +374,7 @@ public class IndexLifecycleRunner {
             // next available step
             StepKey nextValidStepKey = newPolicy.getNextValidStep(currentStepKey);
             if (nextValidStepKey.equals(currentStepKey) == false) {
-                newSettings = moveIndexSettingsToNextStep(idxSettings, currentStepKey, nextValidStepKey, nowSupplier);
+                newSettings = moveIndexSettingsToNextStep(newPolicy, idxSettings, currentStepKey, nextValidStepKey, nowSupplier);
             }
         }
         newSettings.put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), newPolicyName);

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java

@@ -22,6 +22,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
@@ -57,14 +58,15 @@ public class IndexLifecycleService extends AbstractComponent
     private LongSupplier nowSupplier;
     private SchedulerEngine.Job scheduledJob;
 
-    public IndexLifecycleService(Settings settings, Client client, ClusterService clusterService, Clock clock, LongSupplier nowSupplier) {
+    public IndexLifecycleService(Settings settings, Client client, ClusterService clusterService, Clock clock, LongSupplier nowSupplier,
+                                 NamedXContentRegistry xContentRegistry) {
         super(settings);
         this.client = client;
         this.clusterService = clusterService;
         this.clock = clock;
         this.nowSupplier = nowSupplier;
         this.scheduledJob = null;
-        this.policyRegistry = new PolicyStepsRegistry();
+        this.policyRegistry = new PolicyStepsRegistry(xContentRegistry);
         this.lifecycleRunner = new IndexLifecycleRunner(policyRegistry, clusterService, nowSupplier);
         this.pollInterval = LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING.get(settings);
         clusterService.addStateApplier(this);
@@ -144,7 +146,6 @@ public class IndexLifecycleService extends AbstractComponent
                 policyRegistry.removeIndices(event.indicesDeleted());
             }
             if (event.state().metaData().custom(IndexLifecycleMetadata.TYPE) != null) {
-                // update policy steps registry
                 policyRegistry.update(event.state(), client, nowSupplier);
             }
         }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java

@@ -16,14 +16,23 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.InitializePolicyContextStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -44,21 +53,24 @@ public class PolicyStepsRegistry {
     private final Map<String, Map<Step.StepKey, Step>> stepMap;
     // A map of index to a list of compiled steps for the current phase
     private final Map<Index, List<Step>> indexPhaseSteps;
+    private final NamedXContentRegistry xContentRegistry;
 
-    public PolicyStepsRegistry() {
+    public PolicyStepsRegistry(NamedXContentRegistry xContentRegistry) {
         this.lifecyclePolicyMap = new TreeMap<>();
         this.firstStepMap = new HashMap<>();
         this.stepMap = new HashMap<>();
         this.indexPhaseSteps = new HashMap<>();
+        this.xContentRegistry = xContentRegistry;
     }
 
     PolicyStepsRegistry(SortedMap<String, LifecyclePolicyMetadata> lifecyclePolicyMap,
                         Map<String, Step> firstStepMap, Map<String, Map<Step.StepKey, Step>> stepMap,
-                        Map<Index, List<Step>> indexPhaseSteps) {
+                        Map<Index, List<Step>> indexPhaseSteps, NamedXContentRegistry xContentRegistry) {
         this.lifecyclePolicyMap = lifecyclePolicyMap;
         this.firstStepMap = firstStepMap;
         this.stepMap = stepMap;
         this.indexPhaseSteps = indexPhaseSteps;
+        this.xContentRegistry = xContentRegistry;
     }
 
     SortedMap<String, LifecyclePolicyMetadata> getLifecyclePolicyMap() {
@@ -138,7 +150,7 @@ public class PolicyStepsRegistry {
         for (ObjectCursor<IndexMetaData> imd : clusterState.metaData().getIndices().values()) {
             final Index index = imd.value.getIndex();
             final String policy = imd.value.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
-            if (policy == null) {
+            if (policy == null || lifecyclePolicyMap.containsKey(policy) == false) {
                 indexPhaseSteps.remove(index);
             } else {
                 final List<Step> currentSteps = indexPhaseSteps.get(index);
@@ -146,22 +158,48 @@ public class PolicyStepsRegistry {
                 final String existingPhase = (currentSteps == null || currentSteps.size() == 0) ?
                     "_none_" : currentSteps.get(0).getKey().getPhase();
                 // Retrieve the current phase, defaulting to "new" if no phase is set
-                final String currentPhase = imd.value.getSettings().get(LifecycleSettings.LIFECYCLE_PHASE, "new");
+                final String currentPhase = imd.value.getSettings().get(LifecycleSettings.LIFECYCLE_PHASE,
+                    InitializePolicyContextStep.INITIALIZATION_PHASE);
 
                 if (existingPhase.equals(currentPhase) == false) {
                     logger.debug("index [{}] has transitioned phases [{} -> {}], rebuilding step list",
                         index, existingPhase, currentPhase);
-                    // Only rebuild the index's steps if the phase of the existing steps does not match our index's current phase
+                    // parse existing phase steps from the phase definition in the index settings
-                    final Map<Step.StepKey, Step> steps = stepMap.get(policy);
+                    String phaseDef = imd.value.getSettings().get(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION,
-
+                        InitializePolicyContextStep.INITIALIZATION_PHASE);
+                    final Phase phase;
+                    LifecyclePolicy currentPolicy = lifecyclePolicyMap.get(policy).getPolicy();
+                    final LifecyclePolicy policyToExecute;
+                    if (InitializePolicyContextStep.INITIALIZATION_PHASE.equals(phaseDef)
+                            || TerminalPolicyStep.COMPLETED_PHASE.equals(phaseDef)) {
+                        // It is ok to re-use potentially modified policy here since we are in an initialization or completed phase
+                        policyToExecute = currentPolicy;
+                    } else {
+                        // if the current phase definition describes an internal step/phase, do not parse
+                        try (XContentParser parser = JsonXContent.jsonXContent.createParser(xContentRegistry,
+                            DeprecationHandler.THROW_UNSUPPORTED_OPERATION, phaseDef)) {
+                            phase = Phase.parse(parser, currentPhase);
+                        } catch (IOException e) {
+                            logger.error("failed to configure phase [" + currentPhase + "] for index [" + index.getName() + "]", e);
+                            indexPhaseSteps.remove(index);
+                            continue;
+                        }
+                        Map<String, Phase> phaseMap = new HashMap<>(currentPolicy.getPhases());
+                        if (phase != null) {
+                            phaseMap.put(currentPhase, phase);
+                        }
+                        policyToExecute = new LifecyclePolicy(currentPolicy.getType(), currentPolicy.getName(), phaseMap);
+                    }
+                    LifecyclePolicySecurityClient policyClient = new LifecyclePolicySecurityClient(client,
+                        ClientHelper.INDEX_LIFECYCLE_ORIGIN, lifecyclePolicyMap.get(policy).getHeaders());
+                    final List<Step> steps = policyToExecute.toSteps(policyClient, nowSupplier);
                     // Build a list of steps that correspond with the phase the index is currently in
                     final List<Step> phaseSteps;
                     if (steps == null) {
                         phaseSteps = new ArrayList<>();
                     } else {
-                        phaseSteps = steps.entrySet().stream()
+                        phaseSteps = steps.stream()
                             .filter(e -> e.getKey().getPhase().equals(currentPhase))
-                            .map(Map.Entry::getValue)
                             .collect(Collectors.toList());
                     }
                     indexPhaseSteps.put(index, phaseSteps);

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java

@@ -15,11 +15,14 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
 import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
@@ -49,9 +52,9 @@ import static org.hamcrest.Matchers.sameInstance;
 
 public class ExecuteStepsUpdateTaskTests extends ESTestCase {
 
-    private static final StepKey firstStepKey = new StepKey("phase_1", "action_1", "step_1");
+    private static final StepKey firstStepKey = new StepKey("first_phase", "action_1", "step_1");
-    private static final StepKey secondStepKey = new StepKey("phase_1", "action_1", "step_2");
+    private static final StepKey secondStepKey = new StepKey("first_phase", "action_1", "step_2");
-    private static final StepKey thirdStepKey = new StepKey("phase_1", "action_1", "step_3");
+    private static final StepKey thirdStepKey = new StepKey("first_phase", "action_1", "step_3");
     private static final StepKey invalidStepKey = new StepKey("invalid", "invalid", "invalid");
     private ClusterState clusterState;
     private PolicyStepsRegistry policyStepsRegistry;
@@ -68,7 +71,7 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
     private String indexName;
 
     @Before
-    public void prepareState() {
+    public void prepareState() throws IOException {
         client = Mockito.mock(Client.class);
         Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
         firstStep = new MockClusterStateActionStep(firstStepKey, secondStepKey);
@@ -96,7 +99,7 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
         policyMap.put(mixedPolicyName, new LifecyclePolicyMetadata(mixedPolicy, Collections.emptyMap()));
         policyMap.put(allClusterPolicyName, new LifecyclePolicyMetadata(allClusterPolicy, Collections.emptyMap()));
         policyMap.put(invalidPolicyName, new LifecyclePolicyMetadata(invalidPolicy, Collections.emptyMap()));
-        policyStepsRegistry = new PolicyStepsRegistry();
+        policyStepsRegistry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY);
 
         indexName = randomAlphaOfLength(5);
         lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING);
@@ -130,10 +133,16 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
     }
 
     public void testExecuteAllUntilEndOfPhase() throws IOException {
+        NamedXContentRegistry registry = new NamedXContentRegistry(
+            Collections.singletonList(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MockAction.NAME),
+                (p) -> {
+                    MockAction.parse(p);
+                    return new MockAction(Arrays.asList(firstStep, allClusterSecondStep));
+                })));
+        policyStepsRegistry = new PolicyStepsRegistry(registry);
         setupIndexPolicy(allClusterPolicyName);
 
         Step startStep = policyStepsRegistry.getFirstStep(allClusterPolicyName);
-        Step afterStep = policyStepsRegistry.getStep(index, startStep.getNextStepKey());
         long now = randomNonNegativeLong();
         // test execute start till end of phase `new`
         ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(allClusterPolicyName, index, startStep, policyStepsRegistry, () -> now);
@@ -236,7 +245,7 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
                 equalTo(stepInfo.toString()));
     }
 
-    public void testOnFailure() {
+    public void testOnFailure() throws IOException {
         setStateToKey(secondStepKey);
         Step startStep = policyStepsRegistry.getStep(index, secondStepKey);
         long now = randomNonNegativeLong();
@@ -249,7 +258,7 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
         assertSame(expectedException, exception.getCause());
     }
 
-    private void setStateToKey(StepKey stepKey) {
+    private void setStateToKey(StepKey stepKey) throws IOException {
         clusterState = ClusterState.builder(clusterState)
             .metaData(MetaData.builder(clusterState.metaData())
                 .put(IndexMetaData.builder(clusterState.metaData().index(indexName))

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationIT.java

@@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -18,6 +19,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -61,6 +63,13 @@ import static org.hamcrest.core.IsNull.nullValue;
 public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
     private Settings settings;
     private LifecyclePolicy lifecyclePolicy;
+    private static final ObservableAction OBSERVABLE_ACTION;
+    static {
+        List<Step> steps = new ArrayList<>();
+        Step.StepKey key = new Step.StepKey("mock", ObservableAction.NAME, ObservableClusterStateWaitStep.NAME);
+        steps.add(new ObservableClusterStateWaitStep(key, TerminalPolicyStep.KEY));
+        OBSERVABLE_ACTION = new ObservableAction(steps, true);
+    }
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -111,7 +120,7 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
         List<Step> steps = new ArrayList<>();
         Step.StepKey key = new Step.StepKey("mock", ObservableAction.NAME, ObservableClusterStateWaitStep.NAME);
         steps.add(new ObservableClusterStateWaitStep(key, TerminalPolicyStep.KEY));
-        Map<String, LifecycleAction> actions = Collections.singletonMap(ObservableAction.NAME, new ObservableAction(steps, true));
+        Map<String, LifecycleAction> actions = Collections.singletonMap(ObservableAction.NAME, OBSERVABLE_ACTION);
         Map<String, Phase> phases = Collections.singletonMap("mock", new Phase("mock", TimeValue.timeValueSeconds(0), actions));
         lifecyclePolicy = newLockableLifecyclePolicy("test", phases);
     }
@@ -230,6 +239,16 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
             assertThat(step, equalTo(ObservableClusterStateWaitStep.NAME));
         });
 
+        if (randomBoolean()) {
+            // this checks that the phase execution is picked up from the phase definition settings
+            logger.info("updating lifecycle [test_lifecycle] to be empty");
+            PutLifecycleAction.Request updateLifecycleRequest = new PutLifecycleAction.Request
+                (newLockableLifecyclePolicy(lifecyclePolicy.getName(), Collections.emptyMap()));
+            PutLifecycleAction.Response updateLifecycleResponse = client()
+                .execute(PutLifecycleAction.INSTANCE, updateLifecycleRequest).get();
+            assertAcked(updateLifecycleResponse);
+        }
+
 
         logger.info("Closing server1");
         // kill the first server
@@ -265,8 +284,9 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
         }
 
         // update the poll interval
-        TimeValue newPollInterval = TimeValue.timeValueHours(randomLongBetween(6, 10));
+        TimeValue newPollInterval = TimeValue.timeValueHours(randomLongBetween(6, 1000));
-        Settings newIntervalSettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, newPollInterval).build();
+        Settings newIntervalSettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL,
+            newPollInterval.getStringRep()).build();
         assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(newIntervalSettings));
         {
             TimeValueSchedule schedule = (TimeValueSchedule) indexLifecycleService.getScheduledJob().getSchedule();
@@ -290,6 +310,18 @@ public class IndexLifecycleInitialisationIT extends ESIntegTestCase {
                 Setting.Property.Dynamic, Setting.Property.IndexScope);
             return Collections.singletonList(COMPLETE_SETTING);
         }
+
+        @Override
+        public List<NamedXContentRegistry.Entry> getNamedXContent() {
+            return Arrays.asList(
+                new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ObservableAction.NAME), (p) -> {
+                    MockAction.parse(p);
+                    return OBSERVABLE_ACTION;
+                })
+            );
+        }
+
+        @Override
         public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             return Arrays.asList(new NamedWriteableRegistry.Entry(LifecycleType.class, LockableLifecycleType.TYPE,
                     (in) -> LockableLifecycleType.INSTANCE),

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java

@@ -16,11 +16,13 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
 import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.AbstractStepTestCase;
@@ -78,7 +80,7 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         steps.add(step);
         Index index = new Index(indexName, indexName + "uuid");
         indexSteps.put(index, steps);
-        return new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps);
+        return new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps, NamedXContentRegistry.EMPTY);
     }
 
     public void testRunPolicyTerminalPolicyStep() {
@@ -340,7 +342,8 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
     public void testRunPolicyWithNoStepsInRegistry() {
         String policyName = "cluster_state_action_policy";
         ClusterService clusterService = mock(ClusterService.class);
-        IndexLifecycleRunner runner = new IndexLifecycleRunner(new PolicyStepsRegistry(), clusterService, () -> 0L);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(new PolicyStepsRegistry(NamedXContentRegistry.EMPTY),
+            clusterService, () -> 0L);
         IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
         // verify that no exception is thrown
@@ -466,7 +469,8 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         phase1Steps.add(thirdStep);
         Index index = new Index("test", "uuid");
         indexSteps.put(index, phase1Steps);
-        PolicyStepsRegistry registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps,
+            NamedXContentRegistry.EMPTY);
 
         Settings indexSettings = Settings.EMPTY;
         Step actualStep = IndexLifecycleRunner.getCurrentStep(registry, policyName, index, indexSettings);
@@ -500,7 +504,7 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         // TODO: it'd be nice if we used the actual registry.update method for this
         indexSteps.clear();
         indexSteps.put(index, Collections.singletonList(fourthStep));
-        registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps);
+        registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps, NamedXContentRegistry.EMPTY);
 
         indexSettings = Settings.builder()
                 .put(LifecycleSettings.LIFECYCLE_PHASE, "phase_2")
@@ -521,7 +525,7 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         // Back to phase_1
         indexSteps.clear();
         indexSteps.put(index, phase1Steps);
-        registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps);
+        registry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, indexSteps, NamedXContentRegistry.EMPTY);
 
         indexSettings = Settings.builder()
                 .put(LifecycleSettings.LIFECYCLE_PHASE, "phase_1")
@@ -550,24 +554,32 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
 
     public void testMoveClusterStateToNextStep() {
         String indexName = "my_index";
+        LifecyclePolicy policy = LifecyclePolicyTests.randomTestLifecyclePolicy("policy");
+        Phase nextPhase = policy.getPhases().values().stream().findFirst().get();
+        List<LifecyclePolicyMetadata> policyMetadatas = Collections.singletonList(
+            new LifecyclePolicyMetadata(policy, Collections.emptyMap()));
         StepKey currentStep = new StepKey("current_phase", "current_action", "current_step");
-        StepKey nextStep = new StepKey("next_phase", "next_action", "next_step");
+        StepKey nextStep = new StepKey(nextPhase.getName(), "next_action", "next_step");
         long now = randomNonNegativeLong();
 
-        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), Collections.emptyList());
+        // test going from null lifecycle settings to next step
+        ClusterState clusterState = buildClusterState(indexName,
+            Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), policyMetadatas);
         Index index = clusterState.metaData().index(indexName).getIndex();
         ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep,
                 () -> now);
         assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
 
-        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
+        // test going from set currentStep settings to nextStep
+        Builder indexSettingsBuilder = Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_NAME, policy.getName())
+            .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase())
             .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction())
             .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName());
         if (randomBoolean()) {
             indexSettingsBuilder.put(LifecycleSettings.LIFECYCLE_STEP_INFO, randomAlphaOfLength(20));
         }
-        clusterState = buildClusterState(indexName,
+        clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
-                indexSettingsBuilder, Collections.emptyList());
         index = clusterState.metaData().index(indexName).getIndex();
         newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now);
         assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
@@ -626,8 +638,13 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
     public void testSuccessfulValidatedMoveClusterStateToNextStep() {
         String indexName = "my_index";
         String policyName = "my_policy";
+        LifecyclePolicy policy = randomValueOtherThanMany(p -> p.getPhases().size() == 0,
+            () -> LifecyclePolicyTests.randomTestLifecyclePolicy(policyName));
+        Phase nextPhase = policy.getPhases().values().stream().findFirst().get();
+        List<LifecyclePolicyMetadata> policyMetadatas = Collections.singletonList(
+            new LifecyclePolicyMetadata(policy, Collections.emptyMap()));
         StepKey currentStepKey = new StepKey("current_phase", "current_action", "current_step");
-        StepKey nextStepKey = new StepKey("next_phase", "next_action", "next_step");
+        StepKey nextStepKey = new StepKey(nextPhase.getName(), "next_action", "next_step");
         long now = randomNonNegativeLong();
         Step step = new MockStep(nextStepKey, nextStepKey);
         PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step, indexName);
@@ -636,7 +653,7 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
             .put(LifecycleSettings.LIFECYCLE_PHASE, currentStepKey.getPhase())
             .put(LifecycleSettings.LIFECYCLE_ACTION, currentStepKey.getAction())
             .put(LifecycleSettings.LIFECYCLE_STEP, currentStepKey.getName());
-        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, Collections.emptyList());
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
         Index index = clusterState.metaData().index(indexName).getIndex();
         ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, currentStepKey,
             nextStepKey, () -> now, stepRegistry);
@@ -866,7 +883,7 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         String newPolicyName = "new_policy";
         String phaseName = randomAlphaOfLength(10);
         StepKey currentStep = new StepKey(phaseName, MockAction.NAME, randomAlphaOfLength(10));
-        LifecyclePolicy newPolicy = createPolicy(oldPolicyName,
+        LifecyclePolicy newPolicy = createPolicy(newPolicyName,
                 new StepKey(phaseName, MockAction.NAME, randomAlphaOfLength(9)), null);
         LifecyclePolicy oldPolicy = createPolicy(oldPolicyName, currentStep, null);
         Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName)
@@ -875,6 +892,7 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
                 .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true);
         List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
         policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap()));
+        policyMetadatas.add(new LifecyclePolicyMetadata(newPolicy, Collections.emptyMap()));
         ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, policyMetadatas);
         Index index = clusterState.metaData().index(indexName).getIndex();
         Index[] indices = new Index[] { index };

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java

@@ -94,7 +94,7 @@ public class IndexLifecycleServiceTests extends ESTestCase {
         when(adminClient.indices()).thenReturn(indicesClient);
         when(client.settings()).thenReturn(Settings.EMPTY);
 
-        indexLifecycleService = new IndexLifecycleService(Settings.EMPTY, client, clusterService, clock, () -> now);
+        indexLifecycleService = new IndexLifecycleService(Settings.EMPTY, client, clusterService, clock, () -> now, null);
         Mockito.verify(clusterService).addListener(indexLifecycleService);
         Mockito.verify(clusterService).addStateApplier(indexLifecycleService);
     }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java

@@ -19,11 +19,17 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
@@ -37,14 +43,19 @@ public class MoveToErrorStepUpdateTaskTests extends ESTestCase {
     @Before
     public void setupClusterState() {
         policy = randomAlphaOfLength(10);
+        LifecyclePolicy lifecyclePolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policy);
         IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
             .settings(settings(Version.CURRENT)
                 .put(LifecycleSettings.LIFECYCLE_NAME, policy))
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
         index = indexMetadata.getIndex();
+        IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(
+            Collections.singletonMap(policy, new LifecyclePolicyMetadata(lifecyclePolicy, Collections.emptyMap())),
+            OperationMode.RUNNING);
         MetaData metaData = MetaData.builder()
             .persistentSettings(settings(Version.CURRENT).build())
             .put(IndexMetaData.builder(indexMetadata))
+            .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta)
             .build();
         clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
     }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java

@@ -15,10 +15,19 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 import org.junit.Before;
 
+import java.util.Collections;
+import java.util.List;
+
 import static org.hamcrest.Matchers.equalTo;
 
 public class MoveToNextStepUpdateTaskTests extends ESTestCase {
@@ -26,6 +35,7 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
     String policy;
     ClusterState clusterState;
     Index index;
+    LifecyclePolicy lifecyclePolicy;
 
     @Before
     public void setupClusterState() {
@@ -35,19 +45,25 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
                 .put(LifecycleSettings.LIFECYCLE_NAME, policy))
             .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
         index = indexMetadata.getIndex();
+        lifecyclePolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policy);
+        IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(
+            Collections.singletonMap(policy, new LifecyclePolicyMetadata(lifecyclePolicy, Collections.emptyMap())),
+            OperationMode.RUNNING);
         MetaData metaData = MetaData.builder()
             .persistentSettings(settings(Version.CURRENT).build())
             .put(IndexMetaData.builder(indexMetadata))
+            .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta)
             .build();
         clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
     }
 
     public void testExecuteSuccessfullyMoved() {
-        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
-        StepKey nextStepKey = new StepKey("next-phase", "next-action", "next-name");
         long now = randomNonNegativeLong();
+        List<Step> steps = lifecyclePolicy.toSteps(null, () -> now);
+        StepKey currentStepKey = steps.get(0).getKey();
+        StepKey nextStepKey = steps.get(0).getNextStepKey();
 
-        setStateToKey(currentStepKey);
+        setStateToKey(currentStepKey, now);
 
         SetOnce<Boolean> changed = new SetOnce<>();
         MoveToNextStepUpdateTask.Listener listener = (c) -> changed.set(true);
@@ -66,7 +82,7 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
         StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
         StepKey notCurrentStepKey = new StepKey("not-current", "not-current", "not-current");
         long now = randomNonNegativeLong();
-        setStateToKey(notCurrentStepKey);
+        setStateToKey(notCurrentStepKey, now);
         MoveToNextStepUpdateTask.Listener listener = (c) -> {
         };
         MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, null, () -> now, listener);
@@ -77,7 +93,7 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
     public void testExecuteDifferentPolicy() {
         StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
         long now = randomNonNegativeLong();
-        setStateToKey(currentStepKey);
+        setStateToKey(currentStepKey, now);
         setStatePolicy("not-" + policy);
         MoveToNextStepUpdateTask.Listener listener = (c) -> {};
         MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, null, () -> now, listener);
@@ -86,11 +102,12 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
     }
 
     public void testExecuteSuccessfulMoveWithInvalidNextStep() {
-        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
-        StepKey invalidNextStep = new StepKey("next-invalid", "next-invalid", "next-invalid");
         long now = randomNonNegativeLong();
+        List<Step> steps = lifecyclePolicy.toSteps(null, () -> now);
+        StepKey currentStepKey = steps.get(0).getKey();
+        StepKey invalidNextStep = new StepKey("next-invalid", "next-invalid", "next-invalid");
 
-        setStateToKey(currentStepKey);
+        setStateToKey(currentStepKey, now);
 
         SetOnce<Boolean> changed = new SetOnce<>();
         MoveToNextStepUpdateTask.Listener listener = (c) -> changed.set(true);
@@ -108,7 +125,7 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
     public void testClusterProcessedWithNoChange() {
         StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
         long now = randomNonNegativeLong();
-        setStateToKey(currentStepKey);
+        setStateToKey(currentStepKey, now);
         SetOnce<Boolean> changed = new SetOnce<>();
         MoveToNextStepUpdateTask.Listener listener = (c) -> changed.set(true);
         MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, null, () -> now, listener);
@@ -121,7 +138,7 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
         StepKey nextStepKey = new StepKey("next-phase", "next-action", "next-name");
         long now = randomNonNegativeLong();
 
-        setStateToKey(currentStepKey);
+        setStateToKey(currentStepKey, now);
 
         SetOnce<Boolean> changed = new SetOnce<>();
         MoveToNextStepUpdateTask.Listener listener = (c) -> changed.set(true);
@@ -141,12 +158,16 @@ public class MoveToNextStepUpdateTaskTests extends ESTestCase {
                     .put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), index.getName())).build();
 
     }
-    private void setStateToKey(StepKey stepKey) {
+    private void setStateToKey(StepKey stepKey, long now) {
         clusterState = ClusterState.builder(clusterState)
             .metaData(MetaData.builder(clusterState.metaData())
                 .updateSettings(Settings.builder()
+                    .put(LifecycleSettings.LIFECYCLE_PHASE_DEFINITION, "{\"actions\":{\"TEST_ACTION\":{}}}")
                     .put(LifecycleSettings.LIFECYCLE_PHASE, stepKey.getPhase())
+                    .put(LifecycleSettings.LIFECYCLE_PHASE_TIME, now)
                     .put(LifecycleSettings.LIFECYCLE_ACTION, stepKey.getAction())
-                    .put(LifecycleSettings.LIFECYCLE_STEP, stepKey.getName()).build(), index.getName())).build();
+                    .put(LifecycleSettings.LIFECYCLE_ACTION_TIME, now)
+                    .put(LifecycleSettings.LIFECYCLE_STEP, stepKey.getName())
+                    .put(LifecycleSettings.LIFECYCLE_STEP_TIME, now).build(), index.getName())).build();
     }
 }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java

@@ -17,6 +17,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -53,7 +54,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         String policyName = randomAlphaOfLengthBetween(2, 10);
         Step expectedFirstStep = new MockStep(MOCK_STEP_KEY, null);
         Map<String, Step> firstStepMap = Collections.singletonMap(policyName, expectedFirstStep);
-        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, null);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, null, NamedXContentRegistry.EMPTY);
         Step actualFirstStep = registry.getFirstStep(policyName);
         assertThat(actualFirstStep, sameInstance(expectedFirstStep));
     }
@@ -62,7 +63,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         String policyName = randomAlphaOfLengthBetween(2, 10);
         Step expectedFirstStep = new MockStep(MOCK_STEP_KEY, null);
         Map<String, Step> firstStepMap = Collections.singletonMap(policyName, expectedFirstStep);
-        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, null);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, null, NamedXContentRegistry.EMPTY);
         Step actualFirstStep = registry.getFirstStep(policyName + "unknown");
         assertNull(actualFirstStep);
     }
@@ -71,7 +72,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         Step expectedStep = new MockStep(MOCK_STEP_KEY, null);
         Index index = new Index("test", "uuid");
         Map<Index, List<Step>> indexSteps = Collections.singletonMap(index, Collections.singletonList(expectedStep));
-        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps, NamedXContentRegistry.EMPTY);
         Step actualStep = registry.getStep(index, MOCK_STEP_KEY);
         assertThat(actualStep, sameInstance(expectedStep));
     }
@@ -81,13 +82,13 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         Step expectedStep = new ErrorStep(errorStepKey);
         Index index = new Index("test", "uuid");
         Map<Index, List<Step>> indexSteps = Collections.singletonMap(index, Collections.singletonList(expectedStep));
-        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps, NamedXContentRegistry.EMPTY);
         Step actualStep = registry.getStep(index, errorStepKey);
         assertThat(actualStep, equalTo(expectedStep));
     }
 
     public void testGetStepUnknownPolicy() {
-        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, Collections.emptyMap());
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, Collections.emptyMap(), NamedXContentRegistry.EMPTY);
         assertNull(registry.getStep(new Index("test", "uuid"), MOCK_STEP_KEY));
     }
 
@@ -95,7 +96,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         Step expectedStep = new MockStep(MOCK_STEP_KEY, null);
         Index index = new Index("test", "uuid");
         Map<Index, List<Step>> indexSteps = Collections.singletonMap(index, Collections.singletonList(expectedStep));
-        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, indexSteps, NamedXContentRegistry.EMPTY);
         Step.StepKey unknownStepKey = new Step.StepKey(MOCK_STEP_KEY.getPhase(),
             MOCK_STEP_KEY.getAction(),MOCK_STEP_KEY.getName() + "not");
         assertNull(registry.getStep(index, unknownStepKey));
@@ -145,7 +146,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
             .build();
 
         // start with empty registry
-        PolicyStepsRegistry registry = new PolicyStepsRegistry();
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY);
 
         // add new policy
         registry.update(currentState, client, () -> 0L);
@@ -191,7 +192,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         assertTrue(registry.getStepMap().isEmpty());
     }
 
-    public void testUpdateChangedPolicy() {
+    public void testUpdateChangedPolicy() throws Exception {
         Client client = Mockito.mock(Client.class);
         Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
         String policyName = randomAlphaOfLengthBetween(5, 10);
@@ -216,7 +217,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
             .metaData(metaData)
             .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
             .build();
-        PolicyStepsRegistry registry = new PolicyStepsRegistry();
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY);
         // add new policy
         registry.update(currentState, client, () -> 0L);
 
@@ -285,7 +286,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
             .build();
 
         // start with empty registry
-        PolicyStepsRegistry registry = new PolicyStepsRegistry();
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY);
 
         // add new policy
         registry.update(currentState, client, () -> 0L);