Tests: Enable check index in tests again.

* Run flush in beforeIndexShardClosed to prevent an empty shard.
* Only run check index if the shard state before closing was: started, relocated or post_recovery

src/test/java/org/elasticsearch/cluster/ack/AckTests.java

@@ -49,7 +49,6 @@ import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.warmer.IndexWarmersMetaData;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
-import org.elasticsearch.test.store.MockFSDirectoryService;
 import org.junit.Test;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
@@ -85,12 +84,7 @@ public class AckTests extends ElasticsearchIntegrationTest {
 
     @Test
     public void testUpdateSettingsNoAcknowledgement() {
-        // TODO: this test fails CheckIndex test for some reason ... seems like the index is being deleted while we run CheckIndex??
+        createIndex("test");
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
-                ImmutableSettings.settingsBuilder()
-                        // Never run CheckIndex in the end:
-                        .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false).build()));
-
         UpdateSettingsResponse updateSettingsResponse = client().admin().indices().prepareUpdateSettings("test").setTimeout("0s")
                 .setSettings(ImmutableSettings.builder().put("refresh_interval", 9999)).get();
         assertThat(updateSettingsResponse.isAcknowledged(), equalTo(false));
@@ -145,12 +139,7 @@ public class AckTests extends ElasticsearchIntegrationTest {
 
     @Test
     public void testDeleteWarmerAcknowledgement() {
-        // TODO: this test fails CheckIndex test for some reason ... seems like the index is being deleted while we run CheckIndex??
+        createIndex("test");
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
-                ImmutableSettings.settingsBuilder()
-                        // Never run CheckIndex in the end:
-                        .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false).build()));
-        // make sure one shard is started so the search during put warmer will not fail
         index("test", "type", "1", "f", 1);
 
         assertAcked(client().admin().indices().preparePutWarmer("custom_warmer")
@@ -166,12 +155,7 @@ public class AckTests extends ElasticsearchIntegrationTest {
 
     @Test
     public void testDeleteWarmerNoAcknowledgement() throws InterruptedException {
-        // TODO: this test fails CheckIndex test for some reason ... seems like the index is being deleted while we run CheckIndex??
+        createIndex("test");
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
-                ImmutableSettings.settingsBuilder()
-                        // Never run CheckIndex in the end:
-                        .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false).build()));
-        // make sure one shard is started so the search during put warmer will not fail
         index("test", "type", "1", "f", 1);
 
         assertAcked(client().admin().indices().preparePutWarmer("custom_warmer")
@@ -368,11 +352,7 @@ public class AckTests extends ElasticsearchIntegrationTest {
 
     @Test
     public void testIndicesAliasesNoAcknowledgement() {
-        // TODO: this test fails CheckIndex test for some reason ... seems like the index is being deleted while we run CheckIndex??
+        createIndex("test");
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
-                ImmutableSettings.settingsBuilder()
-                        // Never run CheckIndex in the end:
-                        .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false).build()));
 
         IndicesAliasesResponse indicesAliasesResponse = client().admin().indices().prepareAliases().addAlias("test", "alias").setTimeout("0s").get();
         assertThat(indicesAliasesResponse.isAcknowledged(), equalTo(false));
@@ -416,11 +396,7 @@ public class AckTests extends ElasticsearchIntegrationTest {
 
     @Test
     public void testOpenIndexNoAcknowledgement() {
-        // TODO: this test fails CheckIndex test for some reason ... seems like the index is being deleted while we run CheckIndex??
+        createIndex("test");
-        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
-                ImmutableSettings.settingsBuilder()
-                        // Never run CheckIndex in the end:
-                        .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false).build()));
         ensureGreen();
 
         CloseIndexResponse closeIndexResponse = client().admin().indices().prepareClose("test").execute().actionGet();
@@ -428,6 +404,7 @@ public class AckTests extends ElasticsearchIntegrationTest {
 
         OpenIndexResponse openIndexResponse = client().admin().indices().prepareOpen("test").setTimeout("0s").get();
         assertThat(openIndexResponse.isAcknowledged(), equalTo(false));
+        ensureGreen("test"); // make sure that recovery from disk has completed, so that check index doesn't fail.
     }
 
     @Test

src/test/java/org/elasticsearch/gateway/local/SimpleRecoveryLocalGatewayTests.java

@@ -37,6 +37,7 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import org.elasticsearch.test.InternalTestCluster.RestartCallback;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.store.MockDirectoryHelper;
+import org.elasticsearch.test.store.MockFSDirectoryService;
 import org.junit.Test;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@@ -348,6 +349,7 @@ public class SimpleRecoveryLocalGatewayTests extends ElasticsearchIntegrationTes
     public void testReusePeerRecovery() throws Exception {
         ImmutableSettings.Builder settings = settingsBuilder()
                 .put("action.admin.cluster.node.shutdown.delay", "10ms")
+                .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false)
                 .put("gateway.recover_after_nodes", 4)
                 .put(MockDirectoryHelper.CRASH_INDEX, false);
 

src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java

@@ -29,11 +29,12 @@ import org.apache.lucene.util.AbstractRandomizedTest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.settings.IndexSettings;
 import org.elasticsearch.index.shard.IndexShardException;
+import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.service.IndexShard;
 import org.elasticsearch.index.shard.service.InternalIndexShard;
@@ -47,10 +48,15 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.util.EnumSet;
 import java.util.Random;
 
 public class MockFSDirectoryService extends FsDirectoryService {
 
+    private static final EnumSet<IndexShardState> validCheckIndexStates = EnumSet.of(
+            IndexShardState.STARTED, IndexShardState.RELOCATED , IndexShardState.POST_RECOVERY
+    );
+
     private final MockDirectoryHelper helper;
     private FsDirectoryService delegateService;
     public static final String CHECK_INDEX_ON_CLOSE = "index.store.mock.check_index_on_close";
@@ -62,14 +68,36 @@ public class MockFSDirectoryService extends FsDirectoryService {
         final long seed = indexSettings.getAsLong(ElasticsearchIntegrationTest.SETTING_INDEX_SEED, 0l);
         Random random = new Random(seed);
         helper = new MockDirectoryHelper(shardId, indexSettings, logger, random, seed);
-        checkIndexOnClose = indexSettings.getAsBoolean(CHECK_INDEX_ON_CLOSE, false);
+        checkIndexOnClose = indexSettings.getAsBoolean(CHECK_INDEX_ON_CLOSE, true);
 
         delegateService = helper.randomDirectorService(indexStore);
         if (checkIndexOnClose) {
             final IndicesLifecycle.Listener listener = new IndicesLifecycle.Listener() {
+
+                boolean canRun = false;
+
+                @Override
+                public void beforeIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard) {
+                    if (indexShard != null && shardId.equals(sid)) {
+                        logger.info("Shard state before potentially flushing is {}", indexShard.state());
+                        if (validCheckIndexStates.contains(indexShard.state())) {
+                            canRun = true;
+                            // When the the internal engine closes we do a rollback, which removes uncommitted segments
+                            // By doing a commit flush we perform a Lucene commit, but don't clear the translog,
+                            // so that even in tests where don't flush we can check the integrity of the Lucene index
+                            indexShard.flush(
+                                    new Engine.Flush()
+                                            .type(Engine.Flush.Type.COMMIT) // Keep translog for tests that rely on replaying it
+                                            .waitIfOngoing(true)
+                            );
+                        }
+                    }
+                }
+
                 @Override
                 public void afterIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard) {
-                    if (shardId.equals(sid) && indexShard != null) {
+                    if (shardId.equals(sid) && indexShard != null && canRun) {
+                        assert indexShard.state() == IndexShardState.CLOSED : "Current state must be closed";
                         checkIndex(((InternalIndexShard) indexShard).store(), sid);
                     }
                     service.indicesLifecycle().removeListener(this);