Build: Simplify adding plugins that are another project in the build

The current mechanism for adding plugins to the integTest cluster is to
have a FileCollection. This works well for the integTests for a single
plugin, which automatically adds itself to be installed. However, for qa
tests where many plugins may be installed, and from other projects, it
is cumbersome to add configurations, dependencies and dependsOn
statements over and over. This simplifies installing a plugin from
another project by moving this common setup into the cluster
configuration code.

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.test
 
+import org.gradle.api.Project
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 
@@ -64,7 +65,7 @@ class ClusterConfiguration {
 
     Map<String, String> settings = new HashMap<>()
 
-    LinkedHashMap<String, FileCollection> plugins = new LinkedHashMap<>()
+    LinkedHashMap<String, Object> plugins = new LinkedHashMap<>()
 
     LinkedHashMap<String, Object[]> setupCommands = new LinkedHashMap<>()
 
@@ -83,6 +84,11 @@ class ClusterConfiguration {
         plugins.put(name, file)
     }
 
+    @Input
+    void plugin(String name, Project pluginProject) {
+        plugins.put(name, pluginProject)
+    }
+
     @Input
     void setupCommand(String name, Object... args) {
         setupCommands.put(name, args)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -21,7 +21,9 @@ package org.elasticsearch.gradle.test
 import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.gradle.api.*
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.file.FileCollection
 import org.gradle.api.logging.Logger
 import org.gradle.api.tasks.Copy
@@ -102,14 +104,11 @@ class ClusterFormationTasks {
         setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node)
 
         // install plugins
-        for (Map.Entry<String, FileCollection> plugin : node.config.plugins.entrySet()) {
+        for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
             // replace every dash followed by a character with just the uppercase character
             String camelName = plugin.getKey().replaceAll(/-(\w)/) { _, c -> c.toUpperCase(Locale.ROOT) }
             String actionName = "install${camelName[0].toUpperCase(Locale.ROOT) + camelName.substring(1)}Plugin"
-            // delay reading the file location until execution time by wrapping in a closure within a GString
-            String file = "${-> new File(node.pluginsTmpDir, plugin.getValue().singleFile.getName()).toURI().toURL().toString()}"
-            Object[] args = [new File(node.homeDir, 'bin/plugin'), 'install', file]
-            setup = configureExecTask(taskName(task, node, actionName), project, setup, node, args)
+            setup = configureInstallPluginTask(taskName(task, node, actionName), project, setup, node, plugin.getValue())
         }
 
         // extra setup commands
@@ -183,12 +182,44 @@ class ClusterFormationTasks {
             return setup
         }
 
+        List<FileCollection> pluginFiles = []
+        for (Object plugin : node.config.plugins.values()) {
+            if (plugin instanceof Project) {
+                Project pluginProject = plugin
+                if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) {
+                    throw new GradleException("Task ${name} cannot project ${pluginProject.path} which is not an esplugin")
+                }
+                String configurationName = "_plugin_${pluginProject.path}"
+                Configuration configuration = project.configurations.findByName(configurationName)
+                if (configuration == null) {
+                    configuration = project.configurations.create(configurationName)
+                }
+                project.dependencies.add(configurationName, pluginProject)
+                setup.dependsOn(pluginProject.tasks.bundlePlugin)
+                plugin = configuration
+            }
+            pluginFiles.add(plugin)
+        }
+
         return project.tasks.create(name: name, type: Copy, dependsOn: setup) {
             into node.pluginsTmpDir
-            from(node.config.plugins.values())
+            from(pluginFiles)
         }
     }
 
+    static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Object plugin) {
+        FileCollection pluginZip
+        if (plugin instanceof Project) {
+            pluginZip = project.configurations.getByName("_plugin_${plugin.path}")
+        } else {
+            pluginZip = plugin
+        }
+        // delay reading the file location until execution time by wrapping in a closure within a GString
+        String file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
+        Object[] args = [new File(node.homeDir, 'bin/plugin'), 'install', file]
+        return configureExecTask(name, project, setup, node, args)
+    }
+
     /** Adds a task to execute a command to help setup the cluster */
     static Task configureExecTask(String name, Project project, Task setup, NodeInfo node, Object[] execArgs) {
         return project.tasks.create(name: name, type: Exec, dependsOn: setup) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -67,7 +67,9 @@ class RestIntegTestTask extends RandomizedTestingTask {
     }
 
     RestIntegTestTask() {
-        project.afterEvaluate {
+        // this must run after all projects have been configured, so we know any project
+        // references can be accessed as a fully configured
+        project.gradle.projectsEvaluated {
             Task test = project.tasks.findByName('test')
             if (test != null) {
                 mustRunAfter(test)

qa/smoke-test-plugins/build.gradle

@@ -25,10 +25,9 @@ ext.pluginCount = 0
 for (Project subproj : project.rootProject.subprojects) {
   if (subproj.path.startsWith(':plugins:')) {
     integTest {
-      def bundlePlugin = subproj.tasks.findByName('bundlePlugin')
-      dependsOn bundlePlugin
       cluster {
-        plugin subproj.name, bundlePlugin.outputs.files
+        // need to get a non-decorated project object, so must re-lookup the project by path
+        plugin subproj.name, project(subproj.path)
       }
     }
     pluginCount += 1