Introduce Maintenance Mode to ILM (#31164)

This PR introduces a concept of a maintenance mode for the
lifecycle service. During maintenance mode, no policies are
executed.

To be placed into maintenance mode, users must first issue a
request to be placed in maintenance mode. Once the service
is assured that no policies are in actions that are not to be
interrupted (like ShrinkAction), the service will place itself
in maintenance mode.

APIs to-be introduced:

- POST _xpack/index_lifecycle/maintenance/_request
   - issues a request to be placed into maintenenance mode.
     This is not immediate, since we must first verify that
     it is safe to go from REQUESTED -> IN maintenance mode.
- POST _xpack/index_lifecycle/maintenance/_stop
   - issues a request to be taken out (this is immediate)
- GET _xpack/index_lifecycle/maintenance
   - get back the current mode our lifecycle management is in

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java

@@ -33,24 +33,29 @@ import java.util.stream.Collectors;
 
 public class IndexLifecycleMetadata implements XPackMetaDataCustom {
     public static final String TYPE = "index_lifecycle";
+    public static final ParseField MAINTENANCE_MODE_FIELD = new ParseField("maintenance_mode");
     public static final ParseField POLICIES_FIELD = new ParseField("policies");
-    public static final IndexLifecycleMetadata EMPTY = new IndexLifecycleMetadata(Collections.emptySortedMap());
+    public static final IndexLifecycleMetadata EMPTY = new IndexLifecycleMetadata(Collections.emptySortedMap(), OperationMode.NORMAL);
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<IndexLifecycleMetadata, Void> PARSER = new ConstructingObjectParser<>(
             TYPE, a -> new IndexLifecycleMetadata(
-                    ObjectParserUtils.convertListToMapValues(LifecyclePolicyMetadata::getName, (List<LifecyclePolicyMetadata>) a[0])));
+                    ObjectParserUtils.convertListToMapValues(LifecyclePolicyMetadata::getName, (List<LifecyclePolicyMetadata>) a[0]),
+                        OperationMode.valueOf((String) a[1])));
     static {
         PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> LifecyclePolicyMetadata.parse(p, n),
                 v -> {
                     throw new IllegalArgumentException("ordered " + POLICIES_FIELD.getPreferredName() + " are not supported");
                 }, POLICIES_FIELD);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), MAINTENANCE_MODE_FIELD);
     }
 
     private final Map<String, LifecyclePolicyMetadata> policyMetadatas;
+    private final OperationMode maintenanceMode;
 
-    public IndexLifecycleMetadata(Map<String, LifecyclePolicyMetadata> policies) {
+    public IndexLifecycleMetadata(Map<String, LifecyclePolicyMetadata> policies, OperationMode maintenanceMode) {
         this.policyMetadatas = Collections.unmodifiableMap(policies);
+        this.maintenanceMode = maintenanceMode;
     }
 
     public IndexLifecycleMetadata(StreamInput in) throws IOException {
@@ -60,6 +65,7 @@ public class IndexLifecycleMetadata implements XPackMetaDataCustom {
             policies.put(in.readString(), new LifecyclePolicyMetadata(in));
         }
         this.policyMetadatas = policies;
+        this.maintenanceMode = in.readEnum(OperationMode.class);
     }
 
     @Override
@@ -69,12 +75,17 @@ public class IndexLifecycleMetadata implements XPackMetaDataCustom {
             out.writeString(entry.getKey());
             entry.getValue().writeTo(out);
         }
+        out.writeEnum(maintenanceMode);
     }
 
     public Map<String, LifecyclePolicyMetadata> getPolicyMetadatas() {
         return policyMetadatas;
     }
 
+    public OperationMode getMaintenanceMode() {
+        return maintenanceMode;
+    }
+
     public Map<String, LifecyclePolicy> getPolicies() {
         return policyMetadatas.values().stream().map(LifecyclePolicyMetadata::getPolicy)
                 .collect(Collectors.toMap(LifecyclePolicy::getName, Function.identity()));
@@ -88,6 +99,7 @@ public class IndexLifecycleMetadata implements XPackMetaDataCustom {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field(POLICIES_FIELD.getPreferredName(), policyMetadatas);
+        builder.field(MAINTENANCE_MODE_FIELD.getPreferredName(), maintenanceMode);
         return builder;
     }
 
@@ -108,7 +120,7 @@ public class IndexLifecycleMetadata implements XPackMetaDataCustom {
 
     @Override
     public int hashCode() {
-        return Objects.hash(policyMetadatas);
+        return Objects.hash(policyMetadatas, maintenanceMode);
     }
 
     @Override
@@ -120,7 +132,8 @@ public class IndexLifecycleMetadata implements XPackMetaDataCustom {
             return false;
         }
         IndexLifecycleMetadata other = (IndexLifecycleMetadata) obj;
-        return Objects.equals(policyMetadatas, other.policyMetadatas);
+        return Objects.equals(policyMetadatas, other.policyMetadatas)
+            && Objects.equals(maintenanceMode, other.maintenanceMode);
     }
 
     @Override
@@ -131,26 +144,30 @@ public class IndexLifecycleMetadata implements XPackMetaDataCustom {
     public static class IndexLifecycleMetadataDiff implements NamedDiff<MetaData.Custom> {
 
         final Diff<Map<String, LifecyclePolicyMetadata>> policies;
+        final OperationMode maintenanceMode;
 
         IndexLifecycleMetadataDiff(IndexLifecycleMetadata before, IndexLifecycleMetadata after) {
             this.policies = DiffableUtils.diff(before.policyMetadatas, after.policyMetadatas, DiffableUtils.getStringKeySerializer());
+            this.maintenanceMode = after.maintenanceMode;
         }
 
         public IndexLifecycleMetadataDiff(StreamInput in) throws IOException {
             this.policies = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), LifecyclePolicyMetadata::new,
                     IndexLifecycleMetadataDiff::readLifecyclePolicyDiffFrom);
+            this.maintenanceMode = in.readEnum(OperationMode.class);
         }
 
         @Override
         public MetaData.Custom apply(MetaData.Custom part) {
             TreeMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(
                     policies.apply(((IndexLifecycleMetadata) part).policyMetadatas));
-            return new IndexLifecycleMetadata(newPolicies);
+            return new IndexLifecycleMetadata(newPolicies, this.maintenanceMode);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             policies.writeTo(out);
+            out.writeEnum(maintenanceMode);
         }
 
         @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java

@@ -13,6 +13,7 @@ import org.elasticsearch.common.unit.TimeValue;
  */
 public class LifecycleSettings {
     public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval";
+    public static final String LIFECYCLE_MAINTENANCE_MODE = "indices.lifecycle.maintenance";
     public static final String LIFECYCLE_NAME = "index.lifecycle.name";
     public static final String LIFECYCLE_PHASE = "index.lifecycle.phase";
     public static final String LIFECYCLE_ACTION = "index.lifecycle.action";

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OperationMode.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+/**
+ * Enum representing the different modes that Index Lifecycle Service can operate in.
+ */
+public enum OperationMode {
+    /**
+     * This represents a state where no policies are executed
+     */
+    MAINTENANCE {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == NORMAL;
+        }
+    },
+
+    /**
+     * this representes a state where only sensitive actions (like {@link ShrinkAction}) will be executed
+     * until they finish, at which point the operation mode will move to maintenance mode.
+     */
+    MAINTENANCE_REQUESTED {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == NORMAL || nextMode == MAINTENANCE;
+        }
+    },
+
+    /**
+     * Normal operation where all policies are executed as normal.
+     */
+    NORMAL {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == MAINTENANCE_REQUESTED;
+        }
+    };
+
+    public abstract boolean isValidChange(OperationMode nextMode);
+}

x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java

@@ -57,6 +57,10 @@ public class IndexLifecycleRunner {
             return;
         }
         Step currentStep = getCurrentStep(stepRegistry, policy, indexSettings);
+        if (currentStep == null) {
+            throw new IllegalStateException(
+                "current step for index [" + indexMetaData.getIndex().getName() + "] with policy [" + policy + "] is not recognized");
+        }
         logger.debug("running policy with current-step[" + currentStep.getKey() + "]");
         if (currentStep instanceof TerminalPolicyStep) {
             logger.debug("policy [" + policy + "] for index [" + indexMetaData.getIndex().getName() + "] complete, skipping execution");

x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.indexlifecycle;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.Client;
@@ -12,6 +13,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -21,11 +23,15 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
 
 import java.io.Closeable;
 import java.time.Clock;
+import java.util.Collections;
+import java.util.Set;
 import java.util.function.LongSupplier;
 
 /**
@@ -34,6 +40,7 @@ import java.util.function.LongSupplier;
 public class IndexLifecycleService extends AbstractComponent
         implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable {
     private static final Logger logger = ESLoggerFactory.getLogger(IndexLifecycleService.class);
+    private static final Set<String> IGNORE_ACTIONS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkAction.NAME);
 
     private final SetOnce<SchedulerEngine> scheduler = new SetOnce<>();
     private final Clock clock;
@@ -93,7 +100,6 @@ public class IndexLifecycleService extends AbstractComponent
 
             boolean pollIntervalSettingChanged = !pollInterval.equals(previousPollInterval);
 
-
             if (scheduler.get() == null) { // metadata installed and scheduler should be kicked off. start your engines.
                 scheduler.set(new SchedulerEngine(clock));
                 scheduler.get().register(this);
@@ -142,16 +148,51 @@ public class IndexLifecycleService extends AbstractComponent
         }
     }
 
+    /**
+     * executes the policy execution on the appropriate indices by running cluster-state tasks per index.
+     *
+     * If maintenance-mode was requested, and it is safe to move into maintenance-mode, this will also be done here
+     * when possible after no policies are executed.
+     *
+     * @param clusterState the current cluster state
+     * @param fromClusterStateChange whether things are triggered from the cluster-state-listener or the scheduler
+     */
     void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) {
+        IndexLifecycleMetadata currentMetadata = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
+
+        if (currentMetadata == null) {
+            return;
+        }
+
+        OperationMode currentMode = currentMetadata.getMaintenanceMode();
+
+        if (OperationMode.MAINTENANCE.equals(currentMode)) {
+            return;
+        }
+
+        boolean safeToEnterMaintenanceMode = true; // true until proven false by a run policy
+
         // loop through all indices in cluster state and filter for ones that are
         // managed by the Index Lifecycle Service they have a index.lifecycle.name setting
         // associated to a policy
-        clusterState.metaData().indices().valuesIt().forEachRemaining((idxMeta) -> {
+        for (ObjectCursor<IndexMetaData> cursor : clusterState.metaData().indices().values()) {
+            IndexMetaData idxMeta = cursor.value;
             String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings());
             if (Strings.isNullOrEmpty(policyName) == false) {
-                lifecycleRunner.runPolicy(policyName, idxMeta, clusterState, fromClusterStateChange);
+                StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(idxMeta.getSettings());
+                if (OperationMode.MAINTENANCE_REQUESTED == currentMode && stepKey != null
+                        && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) {
+                    logger.info("skipping policy [" + policyName + "] for index [" + idxMeta.getIndex().getName()
+                        + "]. maintenance mode requested");
+                    continue;
+                }
+                lifecycleRunner.runPolicy(policyName, idxMeta, clusterState, fromClusterStateChange);
+                safeToEnterMaintenanceMode = false; // proven false!
+            }
+        }
+        if (safeToEnterMaintenanceMode && OperationMode.MAINTENANCE_REQUESTED == currentMode) {
+            submitMaintenanceModeUpdate(OperationMode.MAINTENANCE);
         }
-        });
     }
 
     @Override
@@ -161,4 +202,9 @@ public class IndexLifecycleService extends AbstractComponent
             engine.stop();
         }
     }
+
+    public void submitMaintenanceModeUpdate(OperationMode mode) {
+        clusterService.submitStateUpdateTask("ilm_maintenance_update",
+            new MaintenanceModeUpdateTask(mode));
+    }
 }

x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/MaintenanceModeUpdateTask.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+
+public class MaintenanceModeUpdateTask extends ClusterStateUpdateTask {
+    private static final Logger logger = ESLoggerFactory.getLogger(MaintenanceModeUpdateTask.class);
+    private final OperationMode mode;
+
+    public MaintenanceModeUpdateTask(OperationMode mode) {
+        this.mode = mode;
+    }
+
+    OperationMode getOperationMode() {
+        return mode;
+    }
+
+    @Override
+    public ClusterState execute(ClusterState currentState) {
+        IndexLifecycleMetadata currentMetadata = currentState.metaData().custom(IndexLifecycleMetadata.TYPE);
+
+
+        if (currentMetadata.getMaintenanceMode().isValidChange(mode) == false) {
+            return currentState;
+        }
+
+        ClusterState.Builder builder = new ClusterState.Builder(currentState);
+        MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
+        metadataBuilder.putCustom(IndexLifecycleMetadata.TYPE,
+            new IndexLifecycleMetadata(currentMetadata.getPolicyMetadatas(), mode));
+        builder.metaData(metadataBuilder.build());
+        return builder.build();
+    }
+
+    @Override
+    public void onFailure(String source, Exception e) {
+        logger.error("unable to update lifecycle metadata with new mode [" + mode + "]", e);
+    }
+}

x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java

@@ -81,7 +81,7 @@ public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Re
                         }
                         SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
                         newPolicies.remove(request.getPolicyName());
-                        IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies);
+                        IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getMaintenanceMode());
                         newState.metaData(MetaData.builder(currentState.getMetaData())
                                 .putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
                         return newState.build();

x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java

@@ -23,6 +23,7 @@ import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
 import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Request;
 import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Response;
@@ -82,7 +83,7 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<Reque
                                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
                         LifecyclePolicyMetadata lifecyclePolicyMetadata = new LifecyclePolicyMetadata(request.getPolicy(), filteredHeaders);
                         newPolicies.put(lifecyclePolicyMetadata.getName(), lifecyclePolicyMetadata);
-                        IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies);
+                        IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, OperationMode.NORMAL);
                         newState.metaData(MetaData.builder(currentState.getMetaData())
                                 .putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
                         return newState.build();

x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java

@@ -24,6 +24,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
 import org.elasticsearch.xpack.core.indexlifecycle.MockStep;
 import org.elasticsearch.xpack.core.indexlifecycle.Phase;
@@ -98,7 +99,7 @@ public class ExecuteStepsUpdateTaskTests extends ESTestCase {
 
         MetaData metaData = MetaData.builder()
             .persistentSettings(settings(Version.CURRENT).build())
-            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap))
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.NORMAL))
             .put(IndexMetaData.builder(indexMetadata))
             .build();
         String nodeId = randomAlphaOfLength(10);

x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java

@@ -24,6 +24,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.xpack.core.indexlifecycle.Phase;
 import org.elasticsearch.xpack.core.indexlifecycle.TestLifecycleType;
 
@@ -59,7 +60,7 @@ public class IndexLifecycleMetadataTests extends AbstractDiffableSerializationTe
             policies.put(policyName, new LifecyclePolicyMetadata(new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName, phases),
                     Collections.emptyMap()));
         }
-        return new IndexLifecycleMetadata(policies);
+        return new IndexLifecycleMetadata(policies, randomFrom(OperationMode.values()));
     }
 
     @Override
@@ -93,10 +94,15 @@ public class IndexLifecycleMetadataTests extends AbstractDiffableSerializationTe
         IndexLifecycleMetadata metadata = (IndexLifecycleMetadata) instance;
         Map<String, LifecyclePolicyMetadata> policies = metadata.getPolicyMetadatas();
         policies = new TreeMap<>(policies);
+        OperationMode mode = metadata.getMaintenanceMode();
+        if (randomBoolean()) {
             String policyName = randomAlphaOfLength(10);
             policies.put(policyName, new LifecyclePolicyMetadata(
                 new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName, Collections.emptyMap()), Collections.emptyMap()));
-        return new IndexLifecycleMetadata(policies);
+        } else {
+            mode = randomValueOtherThan(metadata.getMaintenanceMode(), () -> randomFrom(OperationMode.values()));
+        }
+        return new IndexLifecycleMetadata(policies, mode);
     }
 
     @Override

x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java

@@ -321,6 +321,22 @@ public class IndexLifecycleRunnerTests extends ESTestCase {
         Mockito.verifyZeroInteractions(clusterService);
     }
 
+    public void testRunPolicyWithNoStepsInRegistry() {
+        String policyName = "cluster_state_action_policy";
+        StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");
+        ClusterService clusterService = mock(ClusterService.class);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(new PolicyStepsRegistry(), clusterService, () -> 0L);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        IllegalStateException exception = expectThrows(IllegalStateException.class,
+            () -> runner.runPolicy(policyName, indexMetaData, null, randomBoolean()));
+        assertEquals("current step for index [my_index] with policy [cluster_state_action_policy] is not recognized",
+            exception.getMessage());
+        Mockito.verifyZeroInteractions(clusterService);
+
+    }
+
     public void testRunPolicyUnknownStepType() {
         String policyName = "cluster_state_action_policy";
         StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");

x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.indexlifecycle;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
@@ -13,28 +14,42 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.TestLifecycleType;
 import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
 import org.junit.After;
 import org.junit.Before;
 import org.mockito.Mockito;
 
-import java.io.IOException;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.ZoneId;
+import java.util.Collections;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.concurrent.ExecutorService;
 
 import static org.elasticsearch.node.Node.NODE_MASTER_SETTING;
+import static org.elasticsearch.xpack.core.indexlifecycle.AbstractStepTestCase.randomStepKey;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
@@ -77,12 +92,18 @@ public class IndexLifecycleServiceTests extends ESTestCase {
         indicesClient = mock(IndicesAdminClient.class);
         when(client.admin()).thenReturn(adminClient);
         when(adminClient.indices()).thenReturn(indicesClient);
+        when(client.settings()).thenReturn(Settings.EMPTY);
 
         indexLifecycleService = new IndexLifecycleService(Settings.EMPTY, client, clusterService, clock, () -> now);
         Mockito.verify(clusterService).addListener(indexLifecycleService);
         Mockito.verify(clusterService).addStateApplier(indexLifecycleService);
     }
 
+    @After
+    public void cleanup() {
+        indexLifecycleService.close();
+    }
+
     public void testOnlyChangesStateOnMasterAndMetadataExists() {
         boolean isMaster = randomBoolean();
         String localNodeId = isMaster ? nodeId : nodeId + "not_master";
@@ -176,11 +197,6 @@ public class IndexLifecycleServiceTests extends ESTestCase {
         assertNotNull(indexLifecycleService.getScheduledJob());
     }
 
-    @After
-    public void cleanup() throws IOException {
-        indexLifecycleService.close();
-    }
-
     public void testSchedulerInitializationAndUpdate() {
         TimeValue pollInterval = TimeValue.timeValueSeconds(randomIntBetween(1, 59));
         MetaData metaData = MetaData.builder()
@@ -221,222 +237,126 @@ public class IndexLifecycleServiceTests extends ESTestCase {
         Mockito.verifyNoMoreInteractions(clusterService);
     }
 
-//    /**
+    public void testMaintenanceModeSkip() {
-//     * Checks that a new index does the following successfully:
+        String policyName = randomAlphaOfLengthBetween(1, 20);
-//     *
+        IndexLifecycleRunnerTests.MockInitializePolicyContextStep mockStep =
-//     * 1. setting index.lifecycle.date
+            new IndexLifecycleRunnerTests.MockInitializePolicyContextStep(randomStepKey(), randomStepKey());
-//     * 2. sets phase
+        MockAction mockAction = new MockAction(Collections.singletonList(mockStep));
-//     * 3. sets action
+        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
-//     * 4. executes action
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName,
-//     */
+                Collections.singletonMap(phase.getName(), phase));
-//    @SuppressWarnings("unchecked")
+        SortedMap<String, LifecyclePolicyMetadata> policyMap = new TreeMap<>();
-//    public void testTriggeredWithMatchingPolicy() {
+        policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap()));
-//        String policyName = randomAlphaOfLengthBetween(1, 20);
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
-//        MockAction mockAction = new MockAction(Collections.emptyList());
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
-//        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
-//        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName,
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
-//                Collections.singletonMap(phase.getName(), phase));
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
-//        SortedMap<String, LifecyclePolicy> policyMap = new TreeMap<>();
+            .fPut(index.getName(), indexMetadata);
-//        policyMap.put(policyName, policy);
+        MetaData metaData = MetaData.builder()
-//        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.MAINTENANCE))
-//        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
+            .indices(indices.build())
-//            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
+            .persistentSettings(settings(Version.CURRENT).build())
-//            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+            .build();
-//        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
-//            .fPut(index.getName(), indexMetadata);
+            .metaData(metaData)
-//        MetaData metaData = MetaData.builder()
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
-//            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap))
+            .build();
-//            .indices(indices.build())
+        indexLifecycleService.triggerPolicies(currentState, randomBoolean());
-//            .persistentSettings(settings(Version.CURRENT).build())
+        assertThat(mockStep.getExecuteCount(), equalTo(0L));
-//            .build();
+    }
-//        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
-//            .metaData(metaData)
-//            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
-//            .build();
-//
-//        SchedulerEngine.Event schedulerEvent = new SchedulerEngine.Event(IndexLifecycle.NAME, randomLong(), randomLong());
-//
-//        when(clusterService.state()).thenReturn(currentState);
-//
-//        SetOnce<Boolean> dateUpdated = new SetOnce<>();
-//        SetOnce<Boolean> phaseUpdated = new SetOnce<>();
-//        SetOnce<Boolean> actionUpdated = new SetOnce<>();
-//        doAnswer(invocationOnMock -> {
-//            UpdateSettingsRequest request = (UpdateSettingsRequest)  invocationOnMock.getArguments()[0];
-//            ActionListener<UpdateSettingsResponse> listener = (ActionListener<UpdateSettingsResponse>) invocationOnMock.getArguments()[1];
-//            UpdateSettingsTestHelper.assertSettingsRequest(request, Settings.builder()
-//                .put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING.getKey(),
-//                    indexMetadata.getCreationDate()).build(), index.getName());
-//            dateUpdated.set(true);
-//            listener.onResponse(UpdateSettingsTestHelper.createMockResponse(true));
-//            return null;
-//        }).doAnswer(invocationOnMock -> {
-//            UpdateSettingsRequest request = (UpdateSettingsRequest)  invocationOnMock.getArguments()[0];
-//            ActionListener<UpdateSettingsResponse> listener = (ActionListener<UpdateSettingsResponse>) invocationOnMock.getArguments()[1];
-//            UpdateSettingsTestHelper.assertSettingsRequest(request, Settings.builder()
-//                .put(LifecycleSettings.LIFECYCLE_ACTION, "")
-//                .put(LifecycleSettings.LIFECYCLE_ACTION_TIME, -1L)
-//                .put(LifecycleSettings.LIFECYCLE_PHASE_TIME, now)
-//                .put(LifecycleSettings.LIFECYCLE_PHASE, "phase").build(), index.getName());
-//            phaseUpdated.set(true);
-//            listener.onResponse(UpdateSettingsTestHelper.createMockResponse(true));
-//            return null;
-//        }).doAnswer(invocationOnMock -> {
-//            UpdateSettingsRequest request = (UpdateSettingsRequest)  invocationOnMock.getArguments()[0];
-//            ActionListener<UpdateSettingsResponse> listener = (ActionListener<UpdateSettingsResponse>) invocationOnMock.getArguments()[1];
-//            UpdateSettingsTestHelper.assertSettingsRequest(request, Settings.builder()
-//                .put(LifecycleSettings.LIFECYCLE_ACTION, MockAction.NAME)
-//                .put(LifecycleSettings.LIFECYCLE_ACTION_TIME, now).build(), index.getName());
-//            actionUpdated.set(true);
-//            listener.onResponse(UpdateSettingsTestHelper.createMockResponse(true));
-//            return null;
-//        }).when(indicesClient).updateSettings(any(), any());
-//
-//        indexLifecycleService.triggered(schedulerEvent);
-//
-//        assertThat(dateUpdated.get(), equalTo(true));
-//        assertThat(phaseUpdated.get(), equalTo(true));
-//        assertThat(actionUpdated.get(), equalTo(true));
-//    }
 
-//    /**
+    public void testRequestedMaintenanceOnShrink() {
-//     * Check that a policy is executed without first setting the `index.lifecycle.date` setting
+        Step.StepKey mockShrinkStep = new Step.StepKey(randomAlphaOfLength(4), ShrinkAction.NAME, randomAlphaOfLength(5));
-//     */
+        String policyName = randomAlphaOfLengthBetween(1, 20);
-//    @SuppressWarnings("unchecked")
+        IndexLifecycleRunnerTests.MockInitializePolicyContextStep mockStep =
-//    public void testTriggeredWithDateSettingAlreadyPresent() {
+            new IndexLifecycleRunnerTests.MockInitializePolicyContextStep(mockShrinkStep, randomStepKey());
-//        String policyName = randomAlphaOfLengthBetween(1, 20);
+        MockAction mockAction = new MockAction(Collections.singletonList(mockStep));
-//        MockAction mockAction = new MockAction(Collections.emptyList());
+        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
-//        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName,
-//        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName,
+            Collections.singletonMap(phase.getName(), phase));
-//            Collections.singletonMap(phase.getName(), phase));
+        SortedMap<String, LifecyclePolicyMetadata> policyMap = new TreeMap<>();
-//        SortedMap<String, LifecyclePolicy> policyMap = new TreeMap<>();
+        policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap()));
-//        policyMap.put(policyName, policy);
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
-//        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
-//        long creationDate = randomLongBetween(0, now - 1);
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName)
-//        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
+                .put(LifecycleSettings.LIFECYCLE_PHASE, mockShrinkStep.getPhase())
-//            .settings(settings(Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_ACTION, mockShrinkStep.getAction())
-//                .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName)
+                .put(LifecycleSettings.LIFECYCLE_STEP, mockShrinkStep.getName()))
-//                .put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING.getKey(), creationDate))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
-//            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).creationDate(creationDate).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
-//        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
+            .fPut(index.getName(), indexMetadata);
-//            .fPut(index.getName(), indexMetadata);
+        MetaData metaData = MetaData.builder()
-//        MetaData metaData = MetaData.builder()
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.MAINTENANCE_REQUESTED))
-//            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap))
+            .indices(indices.build())
-//            .indices(indices.build())
+            .persistentSettings(settings(Version.CURRENT).build())
-//            .persistentSettings(settings(Version.CURRENT).build())
+            .build();
-//            .build();
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
-//        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
-//            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
-//            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
-//            .build();
-//
-//        SchedulerEngine.Event schedulerEvent = new SchedulerEngine.Event(IndexLifecycle.NAME, randomLong(), randomLong());
-//
-//        when(clusterService.state()).thenReturn(currentState);
-//
-//        SetOnce<Boolean> dateUpdated = new SetOnce<>();
-//        doAnswer(invocationOnMock -> {
-//            UpdateSettingsRequest request = (UpdateSettingsRequest)  invocationOnMock.getArguments()[0];
-//            ActionListener<UpdateSettingsResponse> listener = (ActionListener<UpdateSettingsResponse>) invocationOnMock.getArguments()[1];
-//            try {
-//                UpdateSettingsTestHelper.assertSettingsRequest(request, Settings.builder()
-//                    .put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE_SETTING.getKey(),
-//                        indexMetadata.getCreationDate()).build(), index.getName());
-//                dateUpdated.set(true);
-//            } catch (AssertionError e) {
-//                // noop: here because we are either updating the phase or action prior to executing MockAction
-//            }
-//            listener.onResponse(UpdateSettingsTestHelper.createMockResponse(true));
-//            return null;
-//        }).when(indicesClient).updateSettings(any(), any());
-//
-//        indexLifecycleService.triggered(schedulerEvent);
-//
-//        assertNull(dateUpdated.get());
-//    }
 
-//    /**
+        ClusterChangedEvent event = new ClusterChangedEvent("_source", currentState, ClusterState.EMPTY_STATE);
-//     * Check that if an index has an unknown lifecycle policy set it does not
+        SetOnce<Boolean> executedShrink = new SetOnce<>();
-//     * execute any policy but does process other indexes.
+        doAnswer(invocationOnMock -> {
-//     */
+            executedShrink.set(true);
-//    public void testTriggeredUnknownPolicyNameSet() {
+            return null;
-//        String policyName = randomAlphaOfLengthBetween(1, 20);
+        }).when(clusterService).submitStateUpdateTask(anyString(), any(ExecuteStepsUpdateTask.class));
-//        MockAction mockAction = new MockAction(Collections.emptyList());
+        indexLifecycleService.applyClusterState(new ClusterChangedEvent("change", currentState, ClusterState.EMPTY_STATE));
-//        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
+        indexLifecycleService.clusterChanged(event);
-//        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName,
+        assertTrue(executedShrink.get());
-//                Collections.singletonMap(phase.getName(), phase));
+    }
-//        SortedMap<String, LifecyclePolicy> policyMap = new TreeMap<>();
-//        policyMap.put(policyName, policy);
-//        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
-//        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
-//                .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), "foo"))
-//                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
-//        Index index2 = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
-//        IndexMetaData indexMetadata2 = IndexMetaData.builder(index2.getName())
-//                .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
-//                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
-//        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
-//                indexMetadata).fPut(index2.getName(), indexMetadata2);
-//        MetaData metaData = MetaData.builder().putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap))
-//                .indices(indices.build()).persistentSettings(settings(Version.CURRENT).build()).build();
-//        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData)
-//                .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()).build();
-//
-//        SchedulerEngine.Event schedulerEvent = new SchedulerEngine.Event(IndexLifecycle.NAME, randomLong(), randomLong());
-//
-//        when(clusterService.state()).thenReturn(currentState);
-//
-//        doAnswer(invocationOnMock -> {
-//            @SuppressWarnings("unchecked")
-//            ActionListener<UpdateSettingsResponse> listener = (ActionListener<UpdateSettingsResponse>) invocationOnMock.getArguments()[1];
-//            listener.onResponse(UpdateSettingsTestHelper.createMockResponse(true));
-//            return null;
-//
-//        }).when(indicesClient).updateSettings(any(), any());
-//
-//        indexLifecycleService.triggered(schedulerEvent);
-//    }
 
-//    /**
+    public void testRequestedMaintenanceOnSafeAction() {
-//     * Check that if an index has no lifecycle policy set it does not execute
+        String policyName = randomAlphaOfLengthBetween(1, 20);
-//     * any policy but does process other indexes.
+        Step.StepKey currentStepKey = randomStepKey();
-//     */
+        IndexLifecycleRunnerTests.MockInitializePolicyContextStep mockStep =
-//    public void testTriggeredNoPolicyNameSet() {
+            new IndexLifecycleRunnerTests.MockInitializePolicyContextStep(currentStepKey, randomStepKey());
-//        String policyName = randomAlphaOfLengthBetween(1, 20);
+        MockAction mockAction = new MockAction(Collections.singletonList(mockStep));
-//        MockAction mockAction = new MockAction(Collections.emptyList());
+        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
-//        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName,
-//        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName,
+            Collections.singletonMap(phase.getName(), phase));
-//                Collections.singletonMap(phase.getName(), phase));
+        SortedMap<String, LifecyclePolicyMetadata> policyMap = new TreeMap<>();
-//        SortedMap<String, LifecyclePolicy> policyMap = new TreeMap<>();
+        policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap()));
-//        policyMap.put(policyName, policy);
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
-//        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
-//        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName()).settings(settings(Version.CURRENT))
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName)
-//                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+                .put(LifecycleSettings.LIFECYCLE_PHASE, currentStepKey.getPhase())
-//        Index index2 = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+                .put(LifecycleSettings.LIFECYCLE_ACTION, currentStepKey.getAction())
-//        IndexMetaData indexMetadata2 = IndexMetaData.builder(index2.getName())
+                .put(LifecycleSettings.LIFECYCLE_STEP, currentStepKey.getName()))
-//                .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
-//                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
-//        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
+            .fPut(index.getName(), indexMetadata);
-//                indexMetadata).fPut(index2.getName(), indexMetadata2);
+        MetaData metaData = MetaData.builder()
-//        MetaData metaData = MetaData.builder().putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap))
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.MAINTENANCE_REQUESTED))
-//                .indices(indices.build()).persistentSettings(settings(Version.CURRENT).build()).build();
+            .indices(indices.build())
-//        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData)
+            .persistentSettings(settings(Version.CURRENT).build())
-//                .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()).build();
+            .build();
-//
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
-//        SchedulerEngine.Event schedulerEvent = new SchedulerEngine.Event(IndexLifecycle.NAME, randomLong(), randomLong());
+            .metaData(metaData)
-//
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
-//        when(clusterService.state()).thenReturn(currentState);
+            .build();
-//
+
-//        doAnswer(invocationOnMock -> {
+        ClusterChangedEvent event = new ClusterChangedEvent("_source", currentState, ClusterState.EMPTY_STATE);
-//            @SuppressWarnings("unchecked")
+
-//            ActionListener<UpdateSettingsResponse> listener = (ActionListener<UpdateSettingsResponse>) invocationOnMock.getArguments()[1];
+        SetOnce<Boolean> ranPolicy = new SetOnce<>();
-//            listener.onResponse(UpdateSettingsTestHelper.createMockResponse(true));
+        SetOnce<Boolean> moveToMaintenance = new SetOnce<>();
-//            return null;
+        doAnswer(invocationOnMock -> {
-//
+            ranPolicy.set(true);
-//        }).when(indicesClient).updateSettings(any(), any());
+            throw new AssertionError("invalid invocation");
-//
+        }).when(clusterService).submitStateUpdateTask(anyString(), any(ExecuteStepsUpdateTask.class));
-//        indexLifecycleService.triggered(schedulerEvent);
+
-//    }
+        doAnswer(invocationOnMock -> {
+            MaintenanceModeUpdateTask task = (MaintenanceModeUpdateTask) invocationOnMock.getArguments()[1];
+            assertThat(task.getOperationMode(), equalTo(OperationMode.MAINTENANCE));
+            moveToMaintenance.set(true);
+            return null;
+        }).when(clusterService).submitStateUpdateTask(anyString(), any(MaintenanceModeUpdateTask.class));
+
+        indexLifecycleService.clusterChanged(event);
+        assertNull(ranPolicy.get());
+        assertTrue(moveToMaintenance.get());
+    }
 
     public void testTriggeredDifferentJob() {
         SchedulerEngine.Event schedulerEvent = new SchedulerEngine.Event("foo", randomLong(), randomLong());

x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/MaintenanceModeUpdateTaskTests.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MaintenanceModeUpdateTaskTests extends ESTestCase {
+
+    public void testExecute() {
+        assertMove(OperationMode.NORMAL, randomFrom(OperationMode.MAINTENANCE_REQUESTED));
+        assertMove(OperationMode.MAINTENANCE_REQUESTED, randomFrom(OperationMode.NORMAL, OperationMode.MAINTENANCE));
+        assertMove(OperationMode.MAINTENANCE, randomFrom(OperationMode.NORMAL));
+
+        OperationMode mode = randomFrom(OperationMode.values());
+        assertNoMove(mode, mode);
+        assertNoMove(OperationMode.MAINTENANCE, randomFrom(OperationMode.MAINTENANCE_REQUESTED));
+        assertNoMove(OperationMode.NORMAL, randomFrom(OperationMode.MAINTENANCE));
+    }
+
+    private void assertMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeUpdate(currentMode, requestedMode, false);
+        assertThat(newMode, equalTo(requestedMode));
+    }
+
+    private void assertNoMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeUpdate(currentMode, requestedMode, true);
+        assertThat(newMode, equalTo(currentMode));
+    }
+
+    private OperationMode executeUpdate(OperationMode currentMode, OperationMode requestMode, boolean assertSameClusterState) {
+        IndexLifecycleMetadata indexLifecycleMetadata = new IndexLifecycleMetadata(Collections.emptyMap(), currentMode);
+        ImmutableOpenMap.Builder<String, MetaData.Custom> customsMapBuilder = ImmutableOpenMap.builder();
+        MetaData metaData = MetaData.builder()
+            .customs(customsMapBuilder.fPut(IndexLifecycleMetadata.TYPE, indexLifecycleMetadata).build())
+            .persistentSettings(settings(Version.CURRENT).build())
+            .build();
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        MaintenanceModeUpdateTask task = new MaintenanceModeUpdateTask(requestMode);
+        ClusterState newState = task.execute(state);
+        if (assertSameClusterState) {
+            assertSame(state, newState);
+        }
+        IndexLifecycleMetadata newMetaData = newState.metaData().custom(IndexLifecycleMetadata.TYPE);
+        assertThat(newMetaData.getPolicyMetadatas(), equalTo(indexLifecycleMetadata.getPolicyMetadatas()));
+        return newMetaData.getMaintenanceMode();
+    }
+}

x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java

@@ -21,6 +21,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.xpack.core.indexlifecycle.MockStep;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
 import org.mockito.Mockito;
@@ -108,7 +109,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
                 new LifecyclePolicyMetadata(newPolicy, headers));
         MetaData metaData = MetaData.builder()
             .persistentSettings(settings(Version.CURRENT).build())
-            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap))
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.NORMAL))
             .build();
         String nodeId = randomAlphaOfLength(10);
         DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
@@ -150,7 +151,8 @@ public class PolicyStepsRegistryTests extends ESTestCase {
         currentState = ClusterState.builder(currentState)
             .metaData(
                 MetaData.builder(metaData)
-                    .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(Collections.emptyMap()))).build();
+                    .putCustom(IndexLifecycleMetadata.TYPE,
+                        new IndexLifecycleMetadata(Collections.emptyMap(), OperationMode.NORMAL))).build();
         registry.update(currentState, client, () -> 0L);
         assertTrue(registry.getLifecyclePolicyMap().isEmpty());
         assertTrue(registry.getFirstStepMap().isEmpty());
@@ -171,7 +173,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
                 new LifecyclePolicyMetadata(newPolicy, headers));
         MetaData metaData = MetaData.builder()
             .persistentSettings(settings(Version.CURRENT).build())
-            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap))
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.NORMAL))
             .build();
         String nodeId = randomAlphaOfLength(10);
         DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
@@ -192,7 +194,7 @@ public class PolicyStepsRegistryTests extends ESTestCase {
                 MetaData.builder(metaData)
                     .putCustom(IndexLifecycleMetadata.TYPE,
                                         new IndexLifecycleMetadata(Collections.singletonMap(policyName,
-                                                new LifecyclePolicyMetadata(newPolicy, Collections.emptyMap())))))
+                                                new LifecyclePolicyMetadata(newPolicy, Collections.emptyMap())), OperationMode.NORMAL)))
                 .build();
         registry.update(currentState, client, () -> 0L);
         // TODO(talevy): assert changes... right now we do not support updates to policies. will require internal cleanup