honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix access denied for shard deletion with WindowsFS 

We randomly use the WindowsFS mock file system to simulate that
windows does not delete files if they are opened by some other
process and instead gives you java.io.IOException: access denied.
In the tests we also check if the shard was deleted while it is
being deleted. This check loads the mata data of the index
and therefore might hold on to a file while the node
is trying to delete it and deletion will fail then.

Instead we should just check if the directory was removed.

closes #13758
This commit is contained in:
Britta Weber 2015-10-13 18:02:44 +02:00
parent 077a401c28
commit f7693b694b
1 changed files with 41 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.gateway;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -27,13 +26,16 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.InternalTestCluster;
import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashMap;
import java.util.concurrent.Future;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ESIntegTestCase.Scope;
@ -70,14 +72,14 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen();
assertIndexInMetaState(node1, index);
assertIndexNotInMetaState(node2, index);
assertIndexDirectoryDeleted(node2, index);
assertIndexInMetaState(masterNode, index);
logger.debug("relocating index...");
client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node2)).get();
client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get();
ensureGreen();
assertIndexNotInMetaState(node1, index);
assertIndexDirectoryDeleted(node1, index);
assertIndexInMetaState(node2, index);
assertIndexInMetaState(masterNode, index);
}
@ -149,48 +151,52 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN));
}
protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception {
assertMetaState(nodeName, indexName, false);
protected void assertIndexDirectoryDeleted(final String nodeName, final String indexName) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
logger.info("checking if meta state exists...");
try {
assertFalse("Expecting index directory of " + indexName + " to be deleted from node " + nodeName, indexDirectoryExists(nodeName, indexName));
} catch (Exception e) {
logger.info("failed to check for data director of index {} on node {}", indexName, nodeName);
fail("could not check if data directory still exists");
}
}
}
);
}
protected void assertIndexInMetaState(String nodeName, String indexName) throws Exception {
assertMetaState(nodeName, indexName, true);
protected void assertIndexInMetaState(final String nodeName, final String indexName) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
logger.info("checking if meta state exists...");
try {
assertTrue("Expecting meta state of index " + indexName + " to be on node " + nodeName, getIndicesMetaDataOnNode(nodeName).containsKey(indexName));
} catch (Throwable t) {
logger.info("failed to load meta state", t);
fail("could not load meta state");
}
}
}
);
}
private void assertMetaState(final String nodeName, final String indexName, final boolean shouldBe) throws Exception {
awaitBusy(() -> {
logger.info("checking if meta state exists...");
try {
return shouldBe == metaStateExists(nodeName, indexName);
} catch (Throwable t) {
logger.info("failed to load meta state", t);
// TODO: loading of meta state fails rarely if the state is deleted while we try to load it
// this here is a hack, would be much better to use for example a WatchService
return false;
private boolean indexDirectoryExists(String nodeName, String indexName) throws Exception {
NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName);
for (Path path : nodeEnv.indexPaths(new Index(indexName))) {
if (Files.exists(path)) {
return true;
}
});
boolean inMetaSate = metaStateExists(nodeName, indexName);
if (shouldBe) {
assertTrue("expected " + indexName + " in meta state of node " + nodeName, inMetaSate);
} else {
assertFalse("expected " + indexName + " to not be in meta state of node " + nodeName, inMetaSate);
}
}
private boolean metaStateExists(String nodeName, String indexName) throws Exception {
ImmutableOpenMap<String, IndexMetaData> indices = getIndicesMetaDataOnNode(nodeName);
boolean inMetaSate = false;
for (ObjectObjectCursor<String, IndexMetaData> index : indices) {
inMetaSate = inMetaSate || index.key.equals(indexName);
}
return inMetaSate;
return false;
}
private ImmutableOpenMap<String, IndexMetaData> getIndicesMetaDataOnNode(String nodeName) throws Exception {
GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName);
MetaData nodeMetaData = null;
nodeMetaData = nodeMetaState.loadMetaState();
MetaData nodeMetaData = nodeMetaState.loadMetaState();
return nodeMetaData.getIndices();
}
}