fix access denied for shard deletion with WindowsFS

We randomly use the WindowsFS mock file system to simulate that
windows does not delete files if they are opened by some other
process and instead gives you java.io.IOException: access denied.
In the tests we also check if the shard was deleted while it is
being deleted. This check loads the mata data of the index
and therefore might hold on to a file while the node
is trying to delete it and deletion will fail then.

Instead we should just check if the directory was removed.

closes #13758

core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.gateway;
 
-import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -27,13 +26,16 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.junit.Test;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.LinkedHashMap;
-import java.util.concurrent.Future;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -70,14 +72,14 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
         index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
         ensureGreen();
         assertIndexInMetaState(node1, index);
-        assertIndexNotInMetaState(node2, index);
+        assertIndexDirectoryDeleted(node2, index);
         assertIndexInMetaState(masterNode, index);
 
         logger.debug("relocating index...");
         client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node2)).get();
         client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get();
         ensureGreen();
-        assertIndexNotInMetaState(node1, index);
+        assertIndexDirectoryDeleted(node1, index);
         assertIndexInMetaState(node2, index);
         assertIndexInMetaState(masterNode, index);
     }
@@ -149,48 +151,52 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
         assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN));
     }
 
-    protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception {
+    protected void assertIndexDirectoryDeleted(final String nodeName, final String indexName) throws Exception {
-        assertMetaState(nodeName, indexName, false);
+        assertBusy(new Runnable() {
+                       @Override
+                       public void run() {
+                           logger.info("checking if meta state exists...");
+                           try {
+                               assertFalse("Expecting index directory of " + indexName + " to be deleted from node " + nodeName, indexDirectoryExists(nodeName, indexName));
+                           } catch (Exception e) {
+                               logger.info("failed to check for data director of index {} on node {}", indexName, nodeName);
+                               fail("could not check if data directory still exists");
+                           }
+                       }
+                   }
+        );
     }
 
-    protected void assertIndexInMetaState(String nodeName, String indexName) throws Exception {
+    protected void assertIndexInMetaState(final String nodeName, final String indexName) throws Exception {
-        assertMetaState(nodeName, indexName, true);
+        assertBusy(new Runnable() {
+                       @Override
+                       public void run() {
+                           logger.info("checking if meta state exists...");
+                           try {
+                               assertTrue("Expecting meta state of index " + indexName + " to be on node " + nodeName, getIndicesMetaDataOnNode(nodeName).containsKey(indexName));
+                           } catch (Throwable t) {
+                               logger.info("failed to load meta state", t);
+                               fail("could not load meta state");
+                           }
+                       }
+                   }
+        );
     }
 
 
-    private void assertMetaState(final String nodeName, final String indexName, final boolean shouldBe) throws Exception {
+    private boolean indexDirectoryExists(String nodeName, String indexName) throws Exception {
-        awaitBusy(() -> {
+        NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName);
-            logger.info("checking if meta state exists...");
+        for (Path path : nodeEnv.indexPaths(new Index(indexName))) {
-            try {
+            if (Files.exists(path)) {
-                return shouldBe == metaStateExists(nodeName, indexName);
+                return true;
-            } catch (Throwable t) {
-                logger.info("failed to load meta state", t);
-                // TODO: loading of meta state fails rarely if the state is deleted while we try to load it
-                // this here is a hack, would be much better to use for example a WatchService
-                return false;
             }
-        });
-        boolean inMetaSate = metaStateExists(nodeName, indexName);
-        if (shouldBe) {
-            assertTrue("expected " + indexName + " in meta state of node " + nodeName, inMetaSate);
-        } else {
-            assertFalse("expected " + indexName + " to not be in meta state of node " + nodeName, inMetaSate);
         }
-    }
+        return false;
-
-    private boolean metaStateExists(String nodeName, String indexName) throws Exception {
-        ImmutableOpenMap<String, IndexMetaData> indices = getIndicesMetaDataOnNode(nodeName);
-        boolean inMetaSate = false;
-        for (ObjectObjectCursor<String, IndexMetaData> index : indices) {
-            inMetaSate = inMetaSate || index.key.equals(indexName);
-        }
-        return inMetaSate;
     }
 
     private ImmutableOpenMap<String, IndexMetaData> getIndicesMetaDataOnNode(String nodeName) throws Exception {
         GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName);
-        MetaData nodeMetaData = null;
+        MetaData nodeMetaData = nodeMetaState.loadMetaState();
-        nodeMetaData = nodeMetaState.loadMetaState();
         return nodeMetaData.getIndices();
     }
 }