Tests: Move cluster dependencies from runner to cluster (#24142)

After splitting integ tests into cluster configuration and the test
runner task, we still have dependencies of the test runner added as deps
of the cluster. This commit adds dependencies directly to the cluster,
so that the runner can have other dependencies independent of what is
needed for the cluster.

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -136,6 +136,8 @@ class ClusterConfiguration {
 
     LinkedHashMap<String, Object[]> setupCommands = new LinkedHashMap<>()
 
+    List<Object> dependencies = new ArrayList<>()
+
     @Input
     void systemProperty(String property, String value) {
         systemProperties.put(property, value)
@@ -179,4 +181,10 @@ class ClusterConfiguration {
         }
         extraConfigFiles.put(path, sourceFile)
     }
+
+    /** Add dependencies that must be run before the first task setting up the cluster. */
+    @Input
+    void dependsOn(Object... deps) {
+        dependencies.addAll(deps)
+    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -57,7 +57,7 @@ class ClusterFormationTasks {
         // first we remove everything in the shared cluster directory to ensure there are no leftovers in repos or anything
         // in theory this should not be necessary but repositories are only deleted in the cluster-state and not on-disk
         // such that snapshots survive failures / test runs and there is no simple way today to fix that.
-        Task cleanup = project.tasks.create(name: "${prefix}#prepareCluster.cleanShared", type: Delete, dependsOn: runner.dependsOn.collect()) {
+        Task cleanup = project.tasks.create(name: "${prefix}#prepareCluster.cleanShared", type: Delete, dependsOn: config.dependencies) {
             delete sharedDir
             doLast {
                 sharedDir.mkdirs()
@@ -104,7 +104,7 @@ class ClusterFormationTasks {
             NodeInfo node = new NodeInfo(config, i, project, prefix, elasticsearchVersion, sharedDir)
             nodes.add(node)
             Task dependsOn = startTasks.empty ? cleanup : startTasks.get(0)
-            startTasks.add(configureNode(project, prefix, runner, dependsOn, node, distro, nodes.get(0)))
+            startTasks.add(configureNode(project, prefix, runner, dependsOn, node, config, distro, nodes.get(0)))
         }
 
         Task wait = configureWaitTask("${prefix}#wait", project, nodes, startTasks)
@@ -148,7 +148,8 @@ class ClusterFormationTasks {
      *
      * @return a task which starts the node.
      */
-    static Task configureNode(Project project, String prefix, Task runner, Object dependsOn, NodeInfo node, Configuration configuration, NodeInfo seedNode) {
+    static Task configureNode(Project project, String prefix, Task runner, Object dependsOn, NodeInfo node, ClusterConfiguration config,
+                              Configuration distribution, NodeInfo seedNode) {
 
         // tasks are chained so their execution order is maintained
         Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
@@ -161,7 +162,7 @@ class ClusterFormationTasks {
 
         setup = configureCheckPreviousTask(taskName(prefix, node, 'checkPrevious'), project, setup, node)
         setup = configureStopTask(taskName(prefix, node, 'stopPrevious'), project, setup, node)
-        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, configuration)
+        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
         setup = configureWriteConfigTask(taskName(prefix, node, 'configure'), project, setup, node, seedNode)
         setup = configureCreateKeystoreTask(taskName(prefix, node, 'createKeystore'), project, setup, node)
         setup = configureAddKeystoreSettingTasks(prefix, project, setup, node)
@@ -205,6 +206,13 @@ class ClusterFormationTasks {
             // if we are running in the background, make sure to stop the server when the task completes
             runner.finalizedBy(stop)
             start.finalizedBy(stop)
+            for (Object dependency : config.dependencies) {
+                if (dependency instanceof Fixture) {
+                    Task depStop = ((Fixture)dependency).stopTask
+                    runner.finalizedBy(depStop)
+                    start.finalizedBy(depStop)
+                }
+            }
         }
         return start
     }

distribution/build.gradle

@@ -127,13 +127,13 @@ configure(distributions) {
   apply plugin: 'elasticsearch.standalone-rest-test'
   apply plugin: 'elasticsearch.rest-test'
   project.integTest {
-    dependsOn project.assemble
     includePackaged project.name == 'integ-test-zip'
     if (project.name != 'integ-test-zip') {
       mustRunAfter ':distribution:integ-test-zip:integTest'
     }
   }
   project.integTestCluster {
+    dependsOn project.assemble
     distribution = project.name
   }
 

plugins/discovery-file/build.gradle

@@ -50,12 +50,10 @@ setupSeedNodeAndUnicastHostsFile.doLast {
 
 // second cluster, which will connect to the first via the unicast_hosts.txt file
 integTestCluster {
+    dependsOn setupSeedNodeAndUnicastHostsFile
     clusterName = 'discovery-file-test-cluster'
     extraConfigFile 'discovery-file/unicast_hosts.txt', srcUnicastHostsFile
 }
 
 integTestRunner.finalizedBy ':plugins:discovery-file:initialCluster#stop'
 
-integTest {
-    dependsOn(setupSeedNodeAndUnicastHostsFile)
-}

plugins/jvm-example/build.gradle

@@ -41,7 +41,7 @@ task exampleFixture(type: org.elasticsearch.gradle.test.Fixture) {
        baseDir
 }
 
-integTest {
+integTestCluster {
   dependsOn exampleFixture
 }
 integTestRunner {

plugins/repository-hdfs/build.gradle

@@ -88,7 +88,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
 }
 
 if (fixtureSupported) {
-  integTest.dependsOn hdfsFixture
+  integTestCluster.dependsOn hdfsFixture
 } else {
   logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH")
   // just tests that the plugin loads

qa/multi-cluster-search/build.gradle

@@ -36,11 +36,10 @@ remoteClusterTestRunner {
   systemProperty 'tests.rest.suite', 'remote_cluster'
 }
 
-task mixedClusterTest(type: RestIntegTestTask) {
-  dependsOn(remoteClusterTestRunner)
-}
+task mixedClusterTest(type: RestIntegTestTask) {}
 
 mixedClusterTestCluster {
+  dependsOn remoteClusterTestRunner
   distribution = 'zip'
   setting 'search.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
   setting 'search.remote.connections_per_cluster', 1

qa/rolling-upgrade/build.gradle

@@ -39,11 +39,10 @@ oldClusterTestRunner {
   systemProperty 'tests.rest.suite', 'old_cluster'
 }
 
-task mixedClusterTest(type: RestIntegTestTask) {
-  dependsOn(oldClusterTestRunner, 'oldClusterTestCluster#node1.stop')
-}
+task mixedClusterTest(type: RestIntegTestTask) {}
 
 mixedClusterTestCluster {
+  dependsOn oldClusterTestRunner, 'oldClusterTestCluster#node1.stop'
   distribution = 'zip'
   clusterName = 'rolling-upgrade'
   unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }

qa/smoke-test-client/build.gradle

@@ -37,6 +37,6 @@ singleNodeIntegTestCluster {
     setting 'discovery.type', 'single-node'
 }
 
-integTest.dependsOn(singleNodeIntegTestRunner, 'singleNodeIntegTestCluster#stop')
+integTestCluster.dependsOn(singleNodeIntegTestRunner, 'singleNodeIntegTestCluster#stop')
 
 check.dependsOn(integTest)