Adds checks to ensure index metadata exists when we try to use it (#33455)

* Adds checks to ensure index metadata exists when we try to use it

* Fixes failing test

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStep.java

@@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexNotFoundException;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -56,8 +55,9 @@ public class AllocationRoutedStep extends ClusterStateWaitStep {
     public Result isConditionMet(Index index, ClusterState clusterState) {
         IndexMetaData idxMeta = clusterState.metaData().index(index);
         if (idxMeta == null) {
-            throw new IndexNotFoundException("Index not found when executing " + getKey().getAction() + " lifecycle action.",
+            // Index must have been since deleted, ignore it
-                index.getName());
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
+            return new Result(false, null);
         }
         if (ActiveShardCount.ALL.enoughShardsActive(clusterState, index.getName()) == false) {
             logger.debug("[{}] lifecycle action for index [{}] cannot make progress because not all shards are active",

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java

@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -14,6 +16,7 @@ import org.elasticsearch.index.Index;
 public final class InitializePolicyContextStep extends ClusterStateActionStep {
     public static final String INITIALIZATION_PHASE = "new";
     public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
+    private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);
 
     public InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
         super(key, nextStepKey);
@@ -21,7 +24,13 @@ public final class InitializePolicyContextStep extends ClusterStateActionStep {
 
     @Override
     public ClusterState performAction(Index index, ClusterState clusterState) {
-        Settings settings = clusterState.metaData().index(index).getSettings();
+        IndexMetaData indexMetaData = clusterState.getMetaData().index(index);
+        if (indexMetaData == null) {
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
+            // Index must have been since deleted, ignore it
+            return clusterState;
+        }
+        Settings settings = indexMetaData.getSettings();
         if (settings.hasValue(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE)) {
             return clusterState;
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStep.java

@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.ParseField;
@@ -19,6 +21,7 @@ import java.util.Objects;
 
 public class ShrunkenIndexCheckStep extends ClusterStateWaitStep {
     public static final String NAME = "is-shrunken-index";
+    private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);
     private String shrunkIndexPrefix;
 
     public ShrunkenIndexCheckStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {
@@ -32,6 +35,12 @@ public class ShrunkenIndexCheckStep extends ClusterStateWaitStep {
 
     @Override
     public Result isConditionMet(Index index, ClusterState clusterState) {
+        IndexMetaData idxMeta = clusterState.getMetaData().index(index);
+        if (idxMeta == null) {
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
+            // Index must have been since deleted, ignore it
+            return new Result(false, null);
+        }
         String shrunkenIndexSource = IndexMetaData.INDEX_RESIZE_SOURCE_NAME.get(
             clusterState.metaData().index(index).getSettings());
         if (Strings.isNullOrEmpty(shrunkenIndexSource)) {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStepTests.java

@@ -22,7 +22,6 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep.Result;
@@ -288,9 +287,9 @@ public class AllocationRoutedStepTests extends AbstractStepTestCase<AllocationRo
 
         AllocationRoutedStep step = createRandomInstance();
 
-        IndexNotFoundException thrownException = expectThrows(IndexNotFoundException.class, () -> step.isConditionMet(index, clusterState));
+        Result actualResult = step.isConditionMet(index, clusterState);
-        assertEquals("Index not found when executing " + step.getKey().getAction() + " lifecycle action.", thrownException.getMessage());
+        assertFalse(actualResult.isComplete());
-        assertEquals(index.getName(), thrownException.getIndex().getName());
+        assertNull(actualResult.getInfomationContext());
     }
 
     private void assertAllocateStatus(Index index, int shards, int replicas, AllocationRoutedStep step, Settings.Builder existingSettings,

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java

@@ -66,6 +66,7 @@ public class ExecuteStepsUpdateTask extends ClusterStateUpdateTask {
         Step currentStep = startStep;
         IndexMetaData indexMetaData = currentState.metaData().index(index);
         if (indexMetaData == null) {
+            logger.debug("lifecycle for index [{}] executed but index no longer exists", index.getName());
             // This index doesn't exist any more, there's nothing to execute currently
             return currentState;
         }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java

@@ -158,6 +158,10 @@ public class IndexLifecycleRunner {
     }
 
     private void runPolicy(IndexMetaData indexMetaData, ClusterState currentState) {
+        if (indexMetaData == null) {
+            // This index doesn't exist any more, there's nothing to execute
+            return;
+        }
         Settings indexSettings = indexMetaData.getSettings();
         String policy = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings);
         runPolicy(policy, indexMetaData, currentState, false);

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTask.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.indexlifecycle;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
@@ -49,7 +50,12 @@ public class MoveToErrorStepUpdateTask extends ClusterStateUpdateTask {
 
     @Override
     public ClusterState execute(ClusterState currentState) throws IOException {
-        Settings indexSettings = currentState.getMetaData().index(index).getSettings();
+        IndexMetaData idxMeta = currentState.getMetaData().index(index);
+        if (idxMeta == null) {
+            // Index must have been since deleted, ignore it
+            return currentState;
+        }
+        Settings indexSettings = idxMeta.getSettings();
         if (policy.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings))
             && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexSettings))) {
             return IndexLifecycleRunner.moveClusterStateToErrorStep(index, currentState, currentStepKey, cause, nowSupplier);

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTask.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.indexlifecycle;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.index.Index;
@@ -48,7 +49,12 @@ public class SetStepInfoUpdateTask extends ClusterStateUpdateTask {
 
     @Override
     public ClusterState execute(ClusterState currentState) throws IOException {
-        Settings indexSettings = currentState.getMetaData().index(index).getSettings();
+        IndexMetaData idxMeta = currentState.getMetaData().index(index);
+        if (idxMeta == null) {
+            // Index must have been since deleted, ignore it
+            return currentState;
+        }
+        Settings indexSettings = idxMeta.getSettings();
         if (policy.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings))
                 && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexSettings))) {
             return IndexLifecycleRunner.addStepInfoToClusterState(index, currentState, stepInfo);