From cd0c0de088686be7b440e4308c35e1b8fadb2b49 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 9 Apr 2014 11:37:02 +0200 Subject: [PATCH] [TEST] RecoveryWhileunderLoadTests sometimes need higher timeouts In some cases when we have a lot of docs with lots of shards recovery takes longer than 1m causing the tests to fail before all shards are recovered. This commit raises the timeout in this test to 5m max while it's rarely needed. This commit also adds an assertion to ElasticsearchAssertions that ensures that the cluster health requests are not hitting a timeout. --- .../recovery/RecoveryWhileUnderLoadTests.java | 19 ++++++++++--------- .../hamcrest/ElasticsearchAssertions.java | 10 ++++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadTests.java b/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadTests.java index c39b1f4a361..ca2f8a5b050 100644 --- a/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadTests.java +++ b/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadTests.java @@ -43,6 +43,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.hamcrest.Matchers.equalTo; public class RecoveryWhileUnderLoadTests extends ElasticsearchIntegrationTest { @@ -87,7 +88,7 @@ public class RecoveryWhileUnderLoadTests extends ElasticsearchIntegrationTest { logger.info("--> waiting for GREEN health status ..."); // make sure the cluster state is green, and all has been recovered - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForNodes(">=2").execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus().setWaitForNodes(">=2")); logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs); waitForDocs(totalNumDocs, indexer); @@ -140,7 +141,7 @@ public class RecoveryWhileUnderLoadTests extends ElasticsearchIntegrationTest { allowNodes("test", 4); logger.info("--> waiting for GREEN health status ..."); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForNodes(">=4").execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus().setWaitForNodes(">=4")); logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs); @@ -195,7 +196,7 @@ public class RecoveryWhileUnderLoadTests extends ElasticsearchIntegrationTest { allowNodes("test", 4); logger.info("--> waiting for GREEN health status ..."); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForNodes(">=4").execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus().setWaitForNodes(">=4")); logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs); @@ -207,23 +208,23 @@ public class RecoveryWhileUnderLoadTests extends ElasticsearchIntegrationTest { logger.info("--> allow 3 nodes for index [test] ..."); allowNodes("test", 3); logger.info("--> waiting for GREEN health status ..."); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForNodes(">=3").execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus().setWaitForNodes(">=3")); logger.info("--> allow 2 nodes for index [test] ..."); allowNodes("test", 2); logger.info("--> waiting for GREEN health status ..."); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().setWaitForNodes(">=2").execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus().setWaitForNodes(">=2")); logger.info("--> allow 1 nodes for index [test] ..."); allowNodes("test", 1); logger.info("--> waiting for YELLOW health status ..."); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForYellowStatus().setWaitForNodes(">=1").execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForYellowStatus().setWaitForNodes(">=1")); logger.info("--> marking and waiting for indexing threads to stop ..."); indexer.stop(); logger.info("--> indexing threads stopped"); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForYellowStatus().setWaitForNodes(">=1").execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForYellowStatus().setWaitForNodes(">=1")); logger.info("--> refreshing the index"); refreshAndAssert(); @@ -254,7 +255,7 @@ public class RecoveryWhileUnderLoadTests extends ElasticsearchIntegrationTest { allowNodes = 2 / allowNodes; allowNodes("test", allowNodes); logger.info("--> waiting for GREEN health status ..."); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForGreenStatus().execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus()); } logger.info("--> marking and waiting for indexing threads to stop ..."); @@ -264,7 +265,7 @@ public class RecoveryWhileUnderLoadTests extends ElasticsearchIntegrationTest { logger.info("--> bump up number of replicas to 1 and allow all nodes to hold the index"); allowNodes("test", 3); assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put("number_of_replicas", 1)).get()); - assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus().execute().actionGet().isTimedOut(), equalTo(false)); + assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus()); logger.info("--> refreshing the index"); refreshAndAssert(); diff --git a/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index 76882dadd53..054f6c3da7c 100644 --- a/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -28,6 +28,8 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.count.CountResponse; @@ -74,6 +76,14 @@ public class ElasticsearchAssertions { assertAcked(builder.get()); } + public static void assertNoTimeout(ClusterHealthRequestBuilder requestBuilder) { + assertNoTimeout(requestBuilder.get()); + } + + public static void assertNoTimeout(ClusterHealthResponse response) { + assertThat("ClusterHealthResponse has timed out", response.isTimedOut(), is(false)); + } + public static void assertAcked(AcknowledgedResponse response) { assertThat(response.getClass().getSimpleName() + " failed - not acked", response.isAcknowledged(), equalTo(true)); assertVersionSerializable(response);