honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve RecoverAfterNodes tests

This commit is contained in:
Igor Motov 2013-01-31 09:10:01 -05:00
parent 5e811e5382
commit e32efba3d8
3 changed files with 56 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/test/java/org/elasticsearch/test/integration/AbstractNodesTests.java
View File

@ -19,17 +19,24 @@
package org.elasticsearch.test.integration;
import com.google.common.collect.ImmutableSet;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.block.ClusterBlock;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.network.NetworkUtils;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
@ -130,4 +137,33 @@ public abstract class AbstractNodesTests {
}
nodes.clear();
}
public boolean waitForNodesToShutdown(TimeValue timeout, String... nodes) throws InterruptedException {
long start = System.currentTimeMillis();
Set<String> activeNodes = newHashSet(nodes);
do {
Thread.sleep(100);
Iterator<String> nodeToCheck = activeNodes.iterator();
while (nodeToCheck.hasNext()) {
String id = nodeToCheck.next();
if (node(id).isClosed()) {
nodeToCheck.remove();
} else {
break;
}
}
} while (!activeNodes.isEmpty() && (System.currentTimeMillis() - start) < timeout.millis());
return activeNodes.isEmpty();
}
public ImmutableSet<ClusterBlock> waitForNoBlocks(TimeValue timeout, String node) throws InterruptedException {
long start = System.currentTimeMillis();
ImmutableSet<ClusterBlock> blocks;
do {
blocks = client(node).admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA);
}
while (!blocks.isEmpty() && (System.currentTimeMillis() - start) < timeout.millis());
return blocks;
}
}

16
src/test/java/org/elasticsearch/test/integration/gateway/local/SimpleRecoveryLocalGatewayTests.java
View File

@ -360,7 +360,7 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests {
logger.info("--> shutting down the nodes");
client("node1").admin().cluster().prepareNodesShutdown().setDelay("10ms").setExit(false).execute().actionGet();
assertThat(waitForNodesToShutdown(TimeValue.timeValueSeconds(30), 4), equalTo(true));
assertThat(waitForNodesToShutdown(TimeValue.timeValueSeconds(30), "node1", "node2", "node3", "node4"), equalTo(true));
logger.info("--> start the nodes back up");
startNode("node1", settings);
startNode("node2", settings);
@ -375,7 +375,7 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests {
logger.info("--> shutting down the nodes");
client("node1").admin().cluster().prepareNodesShutdown().setDelay("10ms").setExit(false).execute().actionGet();
assertThat(waitForNodesToShutdown(TimeValue.timeValueSeconds(30), 4), equalTo(true));
assertThat(waitForNodesToShutdown(TimeValue.timeValueSeconds(30), "node1", "node2", "node3", "node4"), equalTo(true));
logger.info("--> start the nodes back up");
startNode("node1", settings);
@ -430,16 +430,4 @@ public class SimpleRecoveryLocalGatewayTests extends AbstractNodesTests {
assertThat(client("node2").prepareCount("test").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().count(), equalTo(1l));
}
private boolean waitForNodesToShutdown(TimeValue timeout, int numberOfNodes) throws InterruptedException {
long start = System.currentTimeMillis();
WAIT_FOR_CLOSING:
while ((System.currentTimeMillis() - start) < timeout.millis()) {
Thread.sleep(100);
for (int i = 1; i < numberOfNodes + 1; i++) {
if (!node("node" + i).isClosed()) continue WAIT_FOR_CLOSING;
}
return true;
}
return false;
}
}

54
src/test/java/org/elasticsearch/test/integration/gateway/none/RecoverAfterNodesTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.test.integration.gateway.none;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.testng.annotations.AfterMethod;
@ -35,13 +36,15 @@ import static org.hamcrest.Matchers.hasItem;
*/
public class RecoverAfterNodesTests extends AbstractNodesTests {
private final static TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(1);
@AfterMethod
public void closeNodes() {
closeAllNodes();
}
@Test
public void testRecoverAfterNodes() {
public void testRecoverAfterNodes() throws Exception {
logger.info("--> start node (1)");
startNode("node1", settingsBuilder().put("gateway.recover_after_nodes", 3));
assertThat(client("node1").admin().cluster().prepareState().setLocal(true).execute().actionGet()
@ -50,6 +53,8 @@ public class RecoverAfterNodesTests extends AbstractNodesTests {
logger.info("--> start node (2)");
startNode("node2", settingsBuilder().put("gateway.recover_after_nodes", 3));
// Sleeping here for the same time that we wait to check for empty blocks
Thread.sleep(BLOCK_WAIT_TIMEOUT.millis());
assertThat(client("node1").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA),
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
@ -60,15 +65,9 @@ public class RecoverAfterNodesTests extends AbstractNodesTests {
logger.info("--> start node (3)");
startNode("node3", settingsBuilder().put("gateway.recover_after_nodes", 3));
assertThat(client("node1").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(client("node2").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(client("node3").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "node1").isEmpty(), equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "node2").isEmpty(), equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "node3").isEmpty(), equalTo(true));
}
@Test
@ -102,23 +101,14 @@ public class RecoverAfterNodesTests extends AbstractNodesTests {
logger.info("--> start master_node (2)");
startNode("master2", settingsBuilder().put("gateway.recover_after_master_nodes", 2).put("node.data", false).put("node.master", true));
Thread.sleep(300);
assertThat(client("master1").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(client("master2").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(client("data1").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(client("data2").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "master1").isEmpty(), equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "master2").isEmpty(), equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "data1").isEmpty(), equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "data2").isEmpty(), equalTo(true));
}
@Test
public void testRecoverAfterDataNodes() {
public void testRecoverAfterDataNodes() throws Exception {
logger.info("--> start master_node (1)");
startNode("master1", settingsBuilder().put("gateway.recover_after_data_nodes", 2).put("node.data", false).put("node.master", true));
assertThat(client("master1").admin().cluster().prepareState().setLocal(true).execute().actionGet()
@ -148,17 +138,9 @@ public class RecoverAfterNodesTests extends AbstractNodesTests {
logger.info("--> start data_node (2)");
startNode("data2", settingsBuilder().put("gateway.recover_after_data_nodes", 2).put("node.data", true).put("node.master", false));
assertThat(client("master1").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(client("master2").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(client("data1").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(client("data2").admin().cluster().prepareState().setLocal(true).execute().actionGet()
.state().blocks().global(ClusterBlockLevel.METADATA).isEmpty(),
equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "master1").isEmpty(), equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "master2").isEmpty(), equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "data1").isEmpty(), equalTo(true));
assertThat(waitForNoBlocks(BLOCK_WAIT_TIMEOUT, "data2").isEmpty(), equalTo(true));
}
}