From a8c2886b3fe525727e7af906c8a43bf9fa83b9b8 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Thu, 18 Jun 2015 17:28:31 -0400
Subject: [PATCH] Internal: Mark store as corrupted instead of deleting state
 file on engine failure

Currently, we delete the shard _state file on engine failure.
This behaviour does not persist the engine failure reason for later inspection.

This commit marks the shard store as corrupted instead of deleting
the _state file to ensure the store index can not be opened after and
the engine failure is persisted.
---
 .../elasticsearch/index/shard/IndexShard.java |  6 +++---
 .../index/shard/IndexShardTests.java          | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 9d4e498868a..74a3dae27bb 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1272,10 +1272,10 @@ public class IndexShard extends AbstractIndexShardComponent {
         @Override
         public void onFailedEngine(ShardId shardId, String reason, @Nullable Throwable failure) {
             try {
-                // delete the shard state so this folder will not be reused
-                MetaDataStateFormat.deleteMetaState(nodeEnv.availableShardPaths(shardId));
+                // mark as corrupted, so opening the store will fail
+                store.markStoreCorrupted(new IOException("failed engine (reason: [" + reason + "])", failure));
             } catch (IOException e) {
-                logger.warn("failed to delete shard state", e);
+                logger.warn("failed to mark shard store as corrupted", e);
             } finally {
                 for (Engine.FailedEngineListener listener : delegates) {
                     try {
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 2a109849c23..f101e60152f 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -33,6 +33,7 @@ import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.QueryParsingException;
+import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.indices.IndicesService;
@@ -206,6 +207,24 @@ public class IndexShardTests extends ElasticsearchSingleNodeTest {
 
     }
 
+    public void testFailShard() throws Exception {
+        createIndex("test");
+        ensureGreen();
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
+        IndexService test = indicesService.indexService("test");
+        IndexShard shard = test.shard(0);
+        // fail shard
+        shard.failShard("test shard fail", new IOException("corrupted"));
+        // check state file still exists
+        ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
+        assertEquals(shardStateMetaData, getShardStateMetadata(shard));
+        ShardPath shardPath = ShardPath.loadShardPath(logger, env, shard.shardId(), test.getIndexSettings());
+        assertNotNull(shardPath);
+        // but index can't be opened for a failed shard
+        assertThat("store index should be corrupted", Store.canOpenIndex(logger, shardPath.resolveIndex()), equalTo(false));
+    }
+
     ShardStateMetaData getShardStateMetadata(IndexShard shard) {
         ShardRouting shardRouting = shard.routingEntry();
         if (shardRouting == null) {