Add dedicated step for checking shrink allocation status (#35161)

This adds a new step for checking whether an index is allocated correctly based
on the rules added prior to running the shrink step. It also fixes a bug where
for shrink we are not allowed to have the shards relocating for the shrink step.

This also allows us to simplify AllocationRoutedStep and provide better
feedback in the step info for why either the allocation or the shrink checks
have failed.

Resolves #34938

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocateAction.java

@@ -145,7 +145,7 @@ public class AllocateAction implements LifecycleAction {
         exclude.forEach((key, value) -> newSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + key, value));
         require.forEach((key, value) -> newSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + key, value));
         UpdateSettingsStep allocateStep = new UpdateSettingsStep(allocateKey, allocationRoutedKey, client, newSettings.build());
-        AllocationRoutedStep routedCheckStep = new AllocationRoutedStep(allocationRoutedKey, nextStepKey, true);
+        AllocationRoutedStep routedCheckStep = new AllocationRoutedStep(allocationRoutedKey, nextStepKey);
         return Arrays.asList(allocateStep, routedCheckStep);
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStep.java

@@ -40,15 +40,8 @@ public class AllocationRoutedStep extends ClusterStateWaitStep {
     private static final AllocationDeciders ALLOCATION_DECIDERS = new AllocationDeciders(Collections.singletonList(
             new FilterAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))));
 
-    private boolean waitOnAllShardCopies;
+    AllocationRoutedStep(StepKey key, StepKey nextStepKey) {
-
-    AllocationRoutedStep(StepKey key, StepKey nextStepKey, boolean waitOnAllShardCopies) {
         super(key, nextStepKey);
-        this.waitOnAllShardCopies = waitOnAllShardCopies;
-    }
-
-    public boolean getWaitOnAllShardCopies() {
-        return waitOnAllShardCopies;
     }
 
     @Override
@@ -68,42 +61,35 @@ public class AllocationRoutedStep extends ClusterStateWaitStep {
         // if the allocation has happened
         RoutingAllocation allocation = new RoutingAllocation(ALLOCATION_DECIDERS, clusterState.getRoutingNodes(), clusterState, null,
                 System.nanoTime());
+
         int allocationPendingAllShards = 0;
 
         ImmutableOpenIntMap<IndexShardRoutingTable> allShards = clusterState.getRoutingTable().index(index).getShards();
         for (ObjectCursor<IndexShardRoutingTable> shardRoutingTable : allShards.values()) {
-            int allocationPendingThisShard = 0;
-            int shardCopiesThisShard = shardRoutingTable.value.size();
             for (ShardRouting shardRouting : shardRoutingTable.value.shards()) {
                 String currentNodeId = shardRouting.currentNodeId();
                 boolean canRemainOnCurrentNode = ALLOCATION_DECIDERS
                         .canRemain(shardRouting, clusterState.getRoutingNodes().node(currentNodeId), allocation)
                         .type() == Decision.Type.YES;
                 if (canRemainOnCurrentNode == false) {
-                    allocationPendingThisShard++;
-                }
-            }
-
-            if (waitOnAllShardCopies) {
-                allocationPendingAllShards += allocationPendingThisShard;
-            } else if (shardCopiesThisShard - allocationPendingThisShard == 0) {
                     allocationPendingAllShards++;
                 }
             }
+        }
+
         if (allocationPendingAllShards > 0) {
-            logger.debug(
+            logger.debug("{} lifecycle action [{}] waiting for [{}] shards to be allocated to nodes matching the given filters",
-                    "[{}] lifecycle action for index [{}] waiting for [{}] shards " + "to be allocated to nodes matching the given filters",
+                index, getKey().getAction(), allocationPendingAllShards);
-                    getKey().getAction(), index, allocationPendingAllShards);
             return new Result(false, new Info(idxMeta.getNumberOfReplicas(), allocationPendingAllShards, true));
         } else {
-            logger.debug("[{}] lifecycle action for index [{}] complete", getKey().getAction(), index);
+            logger.debug("{} lifecycle action for [{}] complete", index, getKey().getAction());
             return new Result(true, null);
         }
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), waitOnAllShardCopies);
+        return 611;
     }
 
     @Override
@@ -114,9 +100,7 @@ public class AllocationRoutedStep extends ClusterStateWaitStep {
         if (getClass() != obj.getClass()) {
             return false;
         }
-        AllocationRoutedStep other = (AllocationRoutedStep) obj;
+        return super.equals(obj);
-        return super.equals(obj) &&
-                Objects.equals(waitOnAllShardCopies, other.waitOnAllShardCopies);
     }
 
     public static final class Info implements ToXContentObject {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CheckShrinkReadyStep.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * This step is used prior to running a shrink step in order to ensure that the index being shrunk
+ * has a copy of each shard allocated on one particular node (the node used by the require
+ * parameter) and that the shards are not relocating.
+ */
+public class CheckShrinkReadyStep extends ClusterStateWaitStep {
+    public static final String NAME = "check-shrink-allocation";
+
+    private static final Logger logger = LogManager.getLogger(CheckShrinkReadyStep.class);
+
+    CheckShrinkReadyStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+
+    @Override
+    public Result isConditionMet(Index index, ClusterState clusterState) {
+        IndexMetaData idxMeta = clusterState.metaData().index(index);
+
+        if (idxMeta == null) {
+            // Index must have been since deleted, ignore it
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists",
+                getKey().getAction(), index.getName());
+            return new Result(false, null);
+        }
+
+        // How many shards the node should have
+        int expectedShardCount = idxMeta.getNumberOfShards();
+
+        if (ActiveShardCount.ALL.enoughShardsActive(clusterState, index.getName()) == false) {
+            logger.debug("[{}] shrink action for [{}] cannot make progress because not all shards are active",
+                getKey().getAction(), index.getName());
+            return new Result(false, new CheckShrinkReadyStep.Info("", expectedShardCount, -1));
+        }
+
+        // The id of the node the shards should be on
+        final String idShardsShouldBeOn = idxMeta.getSettings().get(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id");
+        if (idShardsShouldBeOn == null) {
+            throw new IllegalStateException("Cannot check shrink allocation as there are no allocation rules by _id");
+        }
+
+        final IndexRoutingTable routingTable = clusterState.getRoutingTable().index(index);
+        int foundShards = 0;
+        for (ShardRouting shard : routingTable.shardsWithState(ShardRoutingState.STARTED)) {
+            final String currentNodeId = shard.currentNodeId();
+            if (idShardsShouldBeOn.equals(currentNodeId) && shard.relocating() == false) {
+                foundShards++;
+            }
+        }
+
+        logger.trace("{} checking for shrink readiness on [{}], found {} shards and need {}",
+            index, idShardsShouldBeOn, foundShards, expectedShardCount);
+
+        if (foundShards == expectedShardCount) {
+            logger.trace("{} successfully found {} allocated shards for shrink readiness on node [{}] ({})",
+                index, expectedShardCount, idShardsShouldBeOn, getKey().getAction());
+            return new Result(true, null);
+        } else {
+            logger.trace("{} failed to find {} allocated shards (found {}) on node [{}] for shrink readiness ({})",
+                index, expectedShardCount, foundShards, idShardsShouldBeOn, getKey().getAction());
+            return new Result(false, new CheckShrinkReadyStep.Info(idShardsShouldBeOn, expectedShardCount,
+                expectedShardCount - foundShards));
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return 612;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        return super.equals(obj);
+    }
+
+    public static final class Info implements ToXContentObject {
+
+        private final String nodeId;
+        private final long actualReplicas;
+        private final long numberShardsLeftToAllocate;
+        private final String message;
+
+        static final ParseField NODE_ID = new ParseField("node_id");
+        static final ParseField EXPECTED_SHARDS = new ParseField("expected_shards");
+        static final ParseField SHARDS_TO_ALLOCATE = new ParseField("shards_left_to_allocate");
+        static final ParseField MESSAGE = new ParseField("message");
+        static final ConstructingObjectParser<CheckShrinkReadyStep.Info, Void> PARSER = new ConstructingObjectParser<>(
+            "check_shrink_ready_step_info", a -> new CheckShrinkReadyStep.Info((String) a[0], (long) a[1], (long) a[2]));
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), NODE_ID);
+            PARSER.declareLong(ConstructingObjectParser.constructorArg(), EXPECTED_SHARDS);
+            PARSER.declareLong(ConstructingObjectParser.constructorArg(), SHARDS_TO_ALLOCATE);
+            PARSER.declareString((i, s) -> {}, MESSAGE);
+        }
+
+        public Info(String nodeId, long expectedShards, long numberShardsLeftToAllocate) {
+            this.nodeId = nodeId;
+            this.actualReplicas = expectedShards;
+            this.numberShardsLeftToAllocate = numberShardsLeftToAllocate;
+            if (numberShardsLeftToAllocate < 0) {
+                this.message = "Waiting for all shards to become active";
+            } else {
+                this.message = String.format(Locale.ROOT, "Waiting for node [%s] to contain [%d] shards, found [%d], remaining [%d]",
+                    nodeId, expectedShards, expectedShards - numberShardsLeftToAllocate, numberShardsLeftToAllocate);
+            }
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(MESSAGE.getPreferredName(), message);
+            builder.field(NODE_ID.getPreferredName(), nodeId);
+            builder.field(SHARDS_TO_ALLOCATE.getPreferredName(), numberShardsLeftToAllocate);
+            builder.field(EXPECTED_SHARDS.getPreferredName(), actualReplicas);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(nodeId, actualReplicas, numberShardsLeftToAllocate);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            CheckShrinkReadyStep.Info other = (CheckShrinkReadyStep.Info) obj;
+            return Objects.equals(actualReplicas, other.actualReplicas) &&
+                Objects.equals(numberShardsLeftToAllocate, other.numberShardsLeftToAllocate) &&
+                Objects.equals(nodeId, other.nodeId);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java

@@ -87,7 +87,7 @@ public class ShrinkAction implements LifecycleAction {
 
         StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
         StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
-        StepKey allocationRoutedKey = new StepKey(phase, NAME, AllocationRoutedStep.NAME);
+        StepKey allocationRoutedKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
         StepKey shrinkKey = new StepKey(phase, NAME, ShrinkStep.NAME);
         StepKey enoughShardsKey = new StepKey(phase, NAME, ShrunkShardsAllocatedStep.NAME);
         StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
@@ -96,13 +96,13 @@ public class ShrinkAction implements LifecycleAction {
 
         UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, setSingleNodeKey, client, readOnlySettings);
         SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client);
-        AllocationRoutedStep allocationStep = new AllocationRoutedStep(allocationRoutedKey, shrinkKey, false);
+        CheckShrinkReadyStep checkShrinkReadyStep = new CheckShrinkReadyStep(allocationRoutedKey, shrinkKey);
         ShrinkStep shrink = new ShrinkStep(shrinkKey, enoughShardsKey, client, numberOfShards, SHRUNKEN_INDEX_PREFIX);
         ShrunkShardsAllocatedStep allocated = new ShrunkShardsAllocatedStep(enoughShardsKey, copyMetadataKey, SHRUNKEN_INDEX_PREFIX);
         CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX);
         ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX);
         ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX);
-        return Arrays.asList(readOnlyStep, setSingleNodeStep, allocationStep, shrink, allocated, copyMetadata,
+        return Arrays.asList(readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated, copyMetadata,
             aliasSwapAndDelete, waitOnShrinkTakeover);
     }
 
@@ -110,13 +110,13 @@ public class ShrinkAction implements LifecycleAction {
     public List<StepKey> toStepKeys(String phase) {
         StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
         StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
-        StepKey allocationRoutedKey = new StepKey(phase, NAME, AllocationRoutedStep.NAME);
+        StepKey checkShrinkReadyKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
         StepKey shrinkKey = new StepKey(phase, NAME, ShrinkStep.NAME);
         StepKey enoughShardsKey = new StepKey(phase, NAME, ShrunkShardsAllocatedStep.NAME);
         StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
         StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
         StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
-        return Arrays.asList(readOnlyKey, setSingleNodeKey, allocationRoutedKey, shrinkKey, enoughShardsKey,
+        return Arrays.asList(readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey,
             copyMetadataKey, aliasKey, isShrunkIndexKey);
     }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStepTests.java

@@ -36,37 +36,32 @@ public class AllocationRoutedStepTests extends AbstractStepTestCase<AllocationRo
     public AllocationRoutedStep createRandomInstance() {
         StepKey stepKey = randomStepKey();
         StepKey nextStepKey = randomStepKey();
-        boolean waitOnAllShardCopies = randomBoolean();
 
-        return new AllocationRoutedStep(stepKey, nextStepKey, waitOnAllShardCopies);
+        return new AllocationRoutedStep(stepKey, nextStepKey);
     }
 
     @Override
     public AllocationRoutedStep mutateInstance(AllocationRoutedStep instance) {
         StepKey key = instance.getKey();
         StepKey nextKey = instance.getNextStepKey();
-        boolean waitOnAllShardCopies = instance.getWaitOnAllShardCopies();
 
-        switch (between(0, 2)) {
+        switch (between(0, 1)) {
         case 0:
             key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
             break;
         case 1:
             nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
             break;
-        case 2:
-            waitOnAllShardCopies = waitOnAllShardCopies == false;
-            break;
         default:
             throw new AssertionError("Illegal randomisation branch");
         }
 
-        return new AllocationRoutedStep(key, nextKey, waitOnAllShardCopies);
+        return new AllocationRoutedStep(key, nextKey);
     }
 
     @Override
     public AllocationRoutedStep copyInstance(AllocationRoutedStep instance) {
-        return new AllocationRoutedStep(instance.getKey(), instance.getNextStepKey(), instance.getWaitOnAllShardCopies());
+        return new AllocationRoutedStep(instance.getKey(), instance.getNextStepKey());
     }
 
     public void testConditionMet() {
@@ -132,9 +127,9 @@ public class AllocationRoutedStepTests extends AbstractStepTestCase<AllocationRo
                 .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
                         ShardRoutingState.STARTED));
 
-        AllocationRoutedStep step = new AllocationRoutedStep(randomStepKey(), randomStepKey(), false);
+        AllocationRoutedStep step = new AllocationRoutedStep(randomStepKey(), randomStepKey());
         assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
-                new ClusterStateWaitStep.Result(true, null));
+                new ClusterStateWaitStep.Result(false, new AllocationRoutedStep.Info(0, 1, true)));
     }
 
     public void testExecuteAllocateNotComplete() throws Exception {
@@ -202,7 +197,7 @@ public class AllocationRoutedStepTests extends AbstractStepTestCase<AllocationRo
                 .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
                         ShardRoutingState.STARTED));
 
-        AllocationRoutedStep step = new AllocationRoutedStep(randomStepKey(), randomStepKey(), true);
+        AllocationRoutedStep step = new AllocationRoutedStep(randomStepKey(), randomStepKey());
         assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
                 new ClusterStateWaitStep.Result(false, new AllocationRoutedStep.Info(0, 1, true)));
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CheckShrinkReadyStepTests.java

@@ -0,0 +1,323 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.node.Node;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class CheckShrinkReadyStepTests extends AbstractStepTestCase<CheckShrinkReadyStep> {
+
+    @Override
+    public CheckShrinkReadyStep createRandomInstance() {
+        Step.StepKey stepKey = randomStepKey();
+        Step.StepKey nextStepKey = randomStepKey();
+
+        return new CheckShrinkReadyStep(stepKey, nextStepKey);
+    }
+
+    @Override
+    public CheckShrinkReadyStep mutateInstance(CheckShrinkReadyStep instance) {
+        Step.StepKey key = instance.getKey();
+        Step.StepKey nextKey = instance.getNextStepKey();
+
+        switch (between(0, 1)) {
+            case 0:
+                key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 1:
+                nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new CheckShrinkReadyStep(key, nextKey);
+    }
+
+    @Override
+    public CheckShrinkReadyStep copyInstance(CheckShrinkReadyStep instance) {
+        return new CheckShrinkReadyStep(instance.getKey(), instance.getNextStepKey());
+    }
+
+    public void testNoSetting() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
+            assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+                new ClusterStateWaitStep.Result(true, null));
+        });
+        assertThat(e.getMessage(), containsString("Cannot check shrink allocation as there are no allocation rules by _id"));
+    }
+
+    public void testConditionMet() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(true, null));
+    }
+
+    public void testConditionMetOnlyOneCopyAllocated() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        boolean primaryOnNode1 = randomBoolean();
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", primaryOnNode1, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
+                ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = new CheckShrinkReadyStep(randomStepKey(), randomStepKey());
+        assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(true, null));
+    }
+
+    public void testConditionNotMetDueToRelocation() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        boolean primaryOnNode1 = randomBoolean();
+        ShardRouting shardOnNode1 = TestShardRouting.newShardRouting(new ShardId(index, 0),
+            "node1", primaryOnNode1, ShardRoutingState.STARTED);
+        shardOnNode1 = shardOnNode1.relocate("node3", 230);
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(shardOnNode1)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
+                ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = new CheckShrinkReadyStep(randomStepKey(), randomStepKey());
+        assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("node1", 1, 1)));
+    }
+
+    public void testExecuteAllocateNotComplete() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 1), "node2", true, ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("node1", 2, 1)));
+    }
+
+    public void testExecuteAllocateNotCompleteOnlyOneCopyAllocated() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        boolean primaryOnNode1 = randomBoolean();
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", primaryOnNode1, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
+                ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = new CheckShrinkReadyStep(randomStepKey(), randomStepKey());
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("node1", 2, 1)));
+    }
+
+    public void testExecuteAllocateUnassigned() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 1), null, null, true, ShardRoutingState.UNASSIGNED,
+                new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "the shard is intentionally unassigned")));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("", 2, -1)));
+    }
+
+    /**
+     * this  tests the scenario where
+     *
+     * PUT index
+     * {
+     *  "settings": {
+     *   "number_of_replicas": 0,
+     *   "number_of_shards": 1
+     *  }
+     * }
+     *
+     * PUT index/_settings
+     * {
+     *  "number_of_replicas": 1,
+     *  "index.routing.allocation.include._id": "{node-name}"
+     * }
+     */
+    public void testExecuteReplicasNotAllocatedOnSingleNode() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = Collections.singletonMap("_id", "node1");
+        Settings.Builder existingSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), null, null, false, ShardRoutingState.UNASSIGNED,
+                new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, "no attempt")));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        assertAllocateStatus(index, 1, 1, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("", 1, -1)));
+    }
+
+    public void testExecuteIndexMissing() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).build();
+
+        CheckShrinkReadyStep step = createRandomInstance();
+
+        ClusterStateWaitStep.Result actualResult = step.isConditionMet(index, clusterState);
+        assertFalse(actualResult.isComplete());
+        assertNull(actualResult.getInfomationContext());
+    }
+
+    private void assertAllocateStatus(Index index, int shards, int replicas, CheckShrinkReadyStep step, Settings.Builder existingSettings,
+                                      Settings.Builder node1Settings, Settings.Builder node2Settings,
+                                      IndexRoutingTable.Builder indexRoutingTable, ClusterStateWaitStep.Result expectedResult) {
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName()).settings(existingSettings).numberOfShards(shards)
+            .numberOfReplicas(replicas).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
+            indexMetadata);
+
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
+            .nodes(DiscoveryNodes.builder()
+                .add(DiscoveryNode.createLocal(Settings.builder().put(node1Settings.build())
+                        .put(Node.NODE_NAME_SETTING.getKey(), "node1").build(),
+                    new TransportAddress(TransportAddress.META_ADDRESS, 9200),
+                    "node1"))
+                .add(DiscoveryNode.createLocal(Settings.builder().put(node2Settings.build())
+                        .put(Node.NODE_NAME_SETTING.getKey(), "node2").build(),
+                    new TransportAddress(TransportAddress.META_ADDRESS, 9201),
+                    "node2")))
+            .routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+        ClusterStateWaitStep.Result actualResult = step.isConditionMet(index, clusterState);
+        assertEquals(expectedResult.isComplete(), actualResult.isComplete());
+        assertEquals(expectedResult.getInfomationContext(), actualResult.getInfomationContext());
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java

@@ -55,7 +55,7 @@ public class ShrinkActionTests extends AbstractActionTestCase<ShrinkAction> {
         assertThat(steps.size(), equalTo(8));
         StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME);
         StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
-        StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, AllocationRoutedStep.NAME);
+        StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME);
         StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
         StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
         StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
@@ -71,10 +71,9 @@ public class ShrinkActionTests extends AbstractActionTestCase<ShrinkAction> {
         assertThat(steps.get(1).getKey(), equalTo(expectedSecondKey));
         assertThat(steps.get(1).getNextStepKey(), equalTo(expectedThirdKey));
 
-        assertTrue(steps.get(2) instanceof AllocationRoutedStep);
+        assertTrue(steps.get(2) instanceof CheckShrinkReadyStep);
         assertThat(steps.get(2).getKey(), equalTo(expectedThirdKey));
         assertThat(steps.get(2).getNextStepKey(), equalTo(expectedFourthKey));
-        assertThat(((AllocationRoutedStep) steps.get(2)).getWaitOnAllShardCopies(), equalTo(false));
 
         assertTrue(steps.get(3) instanceof ShrinkStep);
         assertThat(steps.get(3).getKey(), equalTo(expectedFourthKey));