From dfcc939ef88636954e7ccabb2949a6b70c3554ce Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 7 Jun 2018 13:05:36 -0400 Subject: [PATCH] QA: Better seed nodes for rolling restart Use all running nodes as unicast seeds in the rolling restart tests to avoid a race between pinging and the tests. Without this if the tests are too fast then when a new node comes up and pings its single configured seed node that node *might* not have a ping from the other running node. --- .../gradle/test/ClusterConfiguration.groovy | 5 +++-- qa/rolling-upgrade/build.gradle | 15 +++++++------ x-pack/qa/rolling-upgrade-basic/build.gradle | 15 +++++++------ x-pack/qa/rolling-upgrade/build.gradle | 21 +++++++++++-------- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index cd6c7c36ee6..5c363ac043a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -87,8 +87,9 @@ class ClusterConfiguration { * A closure to call which returns the unicast host to connect to for cluster formation. * * This allows multi node clusters, or a new cluster to connect to an existing cluster. - * The closure takes two arguments, the NodeInfo for the first node in the cluster, and - * an AntBuilder which may be used to wait on conditions before returning. + * The closure takes three arguments, the NodeInfo for the first node in the cluster, + * the NodeInfo for the node current being configured, an AntBuilder which may be used + * to wait on conditions before returning. */ @Input Closure unicastTransportUri = { NodeInfo seedNode, NodeInfo node, AntBuilder ant -> diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 5fc5764ceb5..bfd37863cc2 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -89,8 +89,9 @@ for (Version version : bwcVersions.wireCompatible) { Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask) - configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, - 0, { oldClusterTest.nodes.get(1).transportUri() }) + configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, 0, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oldClusterTest.nodes.get(1).transportUri() + ',' + oldClusterTest.nodes.get(2).transportUri() }) Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner") oneThirdUpgradedTestRunner.configure { @@ -101,8 +102,9 @@ for (Version version : bwcVersions.wireCompatible) { Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask) - configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, - 1, { oneThirdUpgradedTest.nodes.get(0).transportUri() }) + configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, 1, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oldClusterTest.nodes.get(2).transportUri() + ',' + oneThirdUpgradedTest.nodes.get(0).transportUri() }) Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner") twoThirdsUpgradedTestRunner.configure { @@ -113,8 +115,9 @@ for (Version version : bwcVersions.wireCompatible) { Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) - configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, - 2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() }) + configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, 2, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oneThirdUpgradedTest.nodes.get(0).transportUri() + ',' + twoThirdsUpgradedTest.nodes.get(0).transportUri() }) Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner") upgradedClusterTestRunner.configure { diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index 916f71dd05c..6d5b250b460 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -69,8 +69,9 @@ for (Version version : bwcVersions.wireCompatible) { Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask) - configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, - 0, { oldClusterTest.nodes.get(1).transportUri() }) + configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, 0, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oldClusterTest.nodes.get(1).transportUri() + ',' + oldClusterTest.nodes.get(2).transportUri() }) Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner") oneThirdUpgradedTestRunner.configure { @@ -81,8 +82,9 @@ for (Version version : bwcVersions.wireCompatible) { Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask) - configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, - 1, { oneThirdUpgradedTest.nodes.get(0).transportUri() }) + configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, 1, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oldClusterTest.nodes.get(2).transportUri() + ',' + oneThirdUpgradedTest.nodes.get(0).transportUri() }) Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner") twoThirdsUpgradedTestRunner.configure { @@ -93,8 +95,9 @@ for (Version version : bwcVersions.wireCompatible) { Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) - configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, - 2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() }) + configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, 2, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oneThirdUpgradedTest.nodes.get(0).transportUri() + ',' + twoThirdsUpgradedTest.nodes.get(0).transportUri() }) Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner") upgradedClusterTestRunner.configure { diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index f11addb42b8..351f33b9412 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -203,8 +203,9 @@ subprojects { Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask) - configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, - 0, { oldClusterTest.nodes.get(1).transportUri() }) + configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, 0, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oldClusterTest.nodes.get(1).transportUri() + ',' + oldClusterTest.nodes.get(2).transportUri() }) Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner") oneThirdUpgradedTestRunner.configure { @@ -221,8 +222,9 @@ subprojects { Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask) - configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, - 1, { oneThirdUpgradedTest.nodes.get(0).transportUri() }) + configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, 1, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oldClusterTest.nodes.get(2).transportUri() + ',' + oneThirdUpgradedTest.nodes.get(0).transportUri() }) Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner") twoThirdsUpgradedTestRunner.configure { @@ -233,16 +235,17 @@ subprojects { Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) - configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, - 2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() }) + configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, 2, + // Use all running nodes as seed nodes so there is no race between pinging and the tests + { oneThirdUpgradedTest.nodes.get(0).transportUri() + ',' + twoThirdsUpgradedTest.nodes.get(0).transportUri() }) Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner") upgradedClusterTestRunner.configure { systemProperty 'tests.rest.suite', 'upgraded_cluster' /* - * Force stopping all the upgraded nodes after the test runner - * so they are alive during the test. - */ + * Force stopping all the upgraded nodes after the test runner + * so they are alive during the test. + */ finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop" finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop"