fix access denied for shard deletion with WindowsFS

We randomly use the WindowsFS mock file system to simulate that
windows does not delete files if they are opened by some other
process and instead gives you java.io.IOException: access denied.
In the tests we also check if the shard was deleted while it is
being deleted. This check loads the mata data of the index
and therefore might hold on to a file while the node
is trying to delete it and deletion will fail then.

Instead we should just check if the directory was removed.

closes #13758

core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.gateway;
 
-import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -27,13 +26,16 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.junit.Test;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.LinkedHashMap;
-import java.util.concurrent.Future;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -70,14 +72,14 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
         index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
         ensureGreen();
         assertIndexInMetaState(node1, index);
-        assertIndexNotInMetaState(node2, index);
+        assertIndexDirectoryDeleted(node2, index);
         assertIndexInMetaState(masterNode, index);
 
         logger.debug("relocating index...");
         client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node2)).get();
         client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get();
         ensureGreen();
-        assertIndexNotInMetaState(node1, index);
+        assertIndexDirectoryDeleted(node1, index);
         assertIndexInMetaState(node2, index);
         assertIndexInMetaState(masterNode, index);
     }
@@ -149,48 +151,52 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
         assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN));
     }
 
-    protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception {
+    protected void assertIndexDirectoryDeleted(final String nodeName, final String indexName) throws Exception {
-        assertMetaState(nodeName, indexName, false);
+        assertBusy(new Runnable() {
-    }
+                       @Override
-
+                       public void run() {
-    protected void assertIndexInMetaState(String nodeName, String indexName) throws Exception {
-        assertMetaState(nodeName, indexName, true);
-    }
-
-
-    private void assertMetaState(final String nodeName, final String indexName, final boolean shouldBe) throws Exception {
-        awaitBusy(() -> {
                            logger.info("checking if meta state exists...");
                            try {
-                return shouldBe == metaStateExists(nodeName, indexName);
+                               assertFalse("Expecting index directory of " + indexName + " to be deleted from node " + nodeName, indexDirectoryExists(nodeName, indexName));
-            } catch (Throwable t) {
+                           } catch (Exception e) {
-                logger.info("failed to load meta state", t);
+                               logger.info("failed to check for data director of index {} on node {}", indexName, nodeName);
-                // TODO: loading of meta state fails rarely if the state is deleted while we try to load it
+                               fail("could not check if data directory still exists");
-                // this here is a hack, would be much better to use for example a WatchService
-                return false;
                            }
-        });
-        boolean inMetaSate = metaStateExists(nodeName, indexName);
-        if (shouldBe) {
-            assertTrue("expected " + indexName + " in meta state of node " + nodeName, inMetaSate);
-        } else {
-            assertFalse("expected " + indexName + " to not be in meta state of node " + nodeName, inMetaSate);
                        }
                    }
+        );
+    }
 
-    private boolean metaStateExists(String nodeName, String indexName) throws Exception {
+    protected void assertIndexInMetaState(final String nodeName, final String indexName) throws Exception {
-        ImmutableOpenMap<String, IndexMetaData> indices = getIndicesMetaDataOnNode(nodeName);
+        assertBusy(new Runnable() {
-        boolean inMetaSate = false;
+                       @Override
-        for (ObjectObjectCursor<String, IndexMetaData> index : indices) {
+                       public void run() {
-            inMetaSate = inMetaSate || index.key.equals(indexName);
+                           logger.info("checking if meta state exists...");
+                           try {
+                               assertTrue("Expecting meta state of index " + indexName + " to be on node " + nodeName, getIndicesMetaDataOnNode(nodeName).containsKey(indexName));
+                           } catch (Throwable t) {
+                               logger.info("failed to load meta state", t);
+                               fail("could not load meta state");
                            }
-        return inMetaSate;
+                       }
+                   }
+        );
+    }
+
+
+    private boolean indexDirectoryExists(String nodeName, String indexName) throws Exception {
+        NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName);
+        for (Path path : nodeEnv.indexPaths(new Index(indexName))) {
+            if (Files.exists(path)) {
+                return true;
+            }
+        }
+        return false;
     }
 
     private ImmutableOpenMap<String, IndexMetaData> getIndicesMetaDataOnNode(String nodeName) throws Exception {
         GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName);
-        MetaData nodeMetaData = null;
+        MetaData nodeMetaData = nodeMetaState.loadMetaState();
-        nodeMetaData = nodeMetaState.loadMetaState();
         return nodeMetaData.getIndices();
     }
 }