Build: Simplify plugin configuration for rest tests

This change removes the multiple ways that plugins can be added to the
integ test cluster. It also removes the use of the default
configuration, and instead adds a zip configuration to all plugins. This
will enable using project substitutions with plugins, which must be done
with the default configuration.

build.gradle

@@ -173,6 +173,11 @@ subprojects {
     "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
     "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb',
     "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
+    // for transport client
+    "org.elasticsearch.plugin:transport-netty3-client:${version}": ':modules:transport-netty3',
+    "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex',
+    "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
+    "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
   ]
   configurations.all {
     resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -71,8 +71,8 @@ public class PluginBuildPlugin extends BuildPlugin {
                 project.integTest.clusterConfig.module(project)
                 project.tasks.run.clusterConfig.module(project)
             } else {
-                project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+                project.integTest.clusterConfig.plugin(project.path)
-                project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+                project.tasks.run.clusterConfig.plugin(project.path)
                 addZipPomGeneration(project)
             }
 
@@ -83,6 +83,7 @@ public class PluginBuildPlugin extends BuildPlugin {
         }
         createIntegTestTask(project)
         createBundleTask(project)
+        project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))
         project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build
     }
 
@@ -141,13 +142,9 @@ public class PluginBuildPlugin extends BuildPlugin {
         }
         project.assemble.dependsOn(bundle)
 
-        // remove jar from the archives (things that will be published), and set it to the zip
+        // also make the zip available as a configuration (used when depending on this project)
-        project.configurations.archives.artifacts.removeAll { it.archiveTask.is project.jar }
+        project.configurations.create('zip')
-        project.artifacts.add('archives', bundle)
+        project.artifacts.add('zip', bundle)
-
-        // also make the zip the default artifact (used when depending on this project)
-        project.configurations.getByName('default').extendsFrom = []
-        project.artifacts.add('default', bundle)
     }
 
     /** Adds a task to move jar and associated files to a "-client" name. */

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -20,12 +20,15 @@ package org.elasticsearch.gradle.test
 
 import org.gradle.api.GradleException
 import org.gradle.api.Project
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 
 /** Configuration for an elasticsearch cluster, used for integration tests. */
 class ClusterConfiguration {
 
+    private final Project project
+
     @Input
     String distribution = 'integ-test-zip'
 
@@ -77,6 +80,10 @@ class ClusterConfiguration {
         return tmpFile.exists()
     }
 
+    public ClusterConfiguration(Project project) {
+        this.project = project
+    }
+
     Map<String, String> systemProperties = new HashMap<>()
 
     Map<String, String> settings = new HashMap<>()
@@ -84,7 +91,7 @@ class ClusterConfiguration {
     // map from destination path, to source file
     Map<String, Object> extraConfigFiles = new HashMap<>()
 
-    LinkedHashMap<String, Object> plugins = new LinkedHashMap<>()
+    LinkedHashMap<String, Project> plugins = new LinkedHashMap<>()
 
     List<Project> modules = new ArrayList<>()
 
@@ -101,13 +108,9 @@ class ClusterConfiguration {
     }
 
     @Input
-    void plugin(String name, FileCollection file) {
+    void plugin(String path) {
-        plugins.put(name, file)
+        Project pluginProject = project.project(path)
-    }
+        plugins.put(pluginProject.name, pluginProject)
-
-    @Input
-    void plugin(String name, Project pluginProject) {
-        plugins.put(name, pluginProject)
     }
 
     /** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -167,7 +167,7 @@ class ClusterFormationTasks {
         }
 
         // install plugins
-        for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
+        for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
             String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
             setup = configureInstallPluginTask(taskName(task, node, actionName), project, setup, node, plugin.getValue())
         }
@@ -326,38 +326,34 @@ class ClusterFormationTasks {
         Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup)
 
         List<FileCollection> pluginFiles = []
-        for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
+        for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
-            FileCollection pluginZip
-            if (plugin.getValue() instanceof Project) {
-                Project pluginProject = plugin.getValue()
-                if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) {
-                    throw new GradleException("Task ${name} cannot project ${pluginProject.path} which is not an esplugin")
-                }
-                String configurationName = "_plugin_${pluginProject.path}"
-                Configuration configuration = project.configurations.findByName(configurationName)
-                if (configuration == null) {
-                    configuration = project.configurations.create(configurationName)
-                }
-                project.dependencies.add(configurationName, pluginProject)
-                setup.dependsOn(pluginProject.tasks.bundlePlugin)
-                pluginZip = configuration
 
-                // also allow rest tests to use the rest spec from the plugin
+            Project pluginProject = plugin.getValue()
-                Copy copyRestSpec = null
+            if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) {
-                for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
+                throw new GradleException("Task ${name} cannot project ${pluginProject.path} which is not an esplugin")
-                    File restApiDir = new File(resourceDir, 'rest-api-spec/api')
-                    if (restApiDir.exists() == false) continue
-                    if (copyRestSpec == null) {
-                        copyRestSpec = project.tasks.create(name: pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec'), type: Copy)
-                        copyPlugins.dependsOn(copyRestSpec)
-                        copyRestSpec.into(project.sourceSets.test.output.resourcesDir)
-                    }
-                    copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
-                }
-            } else {
-                pluginZip = plugin.getValue()
             }
-            pluginFiles.add(pluginZip)
+            String configurationName = "_plugin_${pluginProject.path}"
+            Configuration configuration = project.configurations.findByName(configurationName)
+            if (configuration == null) {
+                configuration = project.configurations.create(configurationName)
+            }
+            project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
+            setup.dependsOn(pluginProject.tasks.bundlePlugin)
+
+            // also allow rest tests to use the rest spec from the plugin
+            String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
+            Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
+            for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
+                File restApiDir = new File(resourceDir, 'rest-api-spec/api')
+                if (restApiDir.exists() == false) continue
+                if (copyRestSpec == null) {
+                    copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
+                    copyPlugins.dependsOn(copyRestSpec)
+                    copyRestSpec.into(project.sourceSets.test.output.resourcesDir)
+                }
+                copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
+            }
+            pluginFiles.add(configuration)
         }
 
         copyPlugins.into(node.pluginsTmpDir)
@@ -379,15 +375,10 @@ class ClusterFormationTasks {
         return installModule
     }
 
-    static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Object plugin) {
+    static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin) {
-        FileCollection pluginZip
+        FileCollection pluginZip = project.configurations.getByName("_plugin_${plugin.path}")
-        if (plugin instanceof Project) {
-            pluginZip = project.configurations.getByName("_plugin_${plugin.path}")
-        } else {
-            pluginZip = plugin
-        }
         // delay reading the file location until execution time by wrapping in a closure within a GString
-        String file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
+        Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
         Object[] args = [new File(node.homeDir, 'bin/elasticsearch-plugin'), 'install', file]
         return configureExecTask(name, project, setup, node, args)
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -32,7 +32,7 @@ import org.gradle.util.ConfigureUtil
  */
 public class RestIntegTestTask extends RandomizedTestingTask {
 
-    ClusterConfiguration clusterConfig = new ClusterConfiguration()
+    ClusterConfiguration clusterConfig
 
     /** Flag indicating whether the rest tests in the rest spec should be run. */
     @Input
@@ -44,6 +44,7 @@ public class RestIntegTestTask extends RandomizedTestingTask {
         dependsOn(project.testClasses)
         classpath = project.sourceSets.test.runtimeClasspath
         testClassesDir = project.sourceSets.test.output.classesDir
+        clusterConfig = new ClusterConfiguration(project)
 
         // start with the common test configuration
         configure(BuildPlugin.commonTestConfig(project))

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy

@@ -7,11 +7,15 @@ import org.gradle.util.ConfigureUtil
 
 public class RunTask extends DefaultTask {
 
-    ClusterConfiguration clusterConfig = new ClusterConfiguration(httpPort: 9200, transportPort: 9300, daemonize: false)
+    ClusterConfiguration clusterConfig
 
     public RunTask() {
         description = "Runs elasticsearch with '${project.path}'"
         group = 'Verification'
+        clusterConfig = new ClusterConfiguration(project)
+        clusterConfig.httpPort = 9200
+        clusterConfig.transportPort = 9300
+        clusterConfig.daemonize = false
         project.afterEvaluate {
             ClusterFormationTasks.setup(project, this, clusterConfig)
         }

docs/build.gradle

@@ -36,16 +36,18 @@ integTest {
 }
 
 // Build the cluser with all plugins
+
 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
   /* Skip repositories. We just aren't going to be able to test them so it
    * doesn't make sense to waste time installing them. */
   if (subproj.path.startsWith(':plugins:repository-')) {
     return
   }
-  integTest {
+  subproj.afterEvaluate { // need to wait until the project has been configured
-    cluster {
+    integTest {
-      // We need a non-decorated project object, so we lookup the project by path
+      cluster {
-      plugin subproj.name, project(subproj.path)
+        plugin subproj.path
+      }
     }
   }
 }

qa/smoke-test-ingest-with-all-dependencies/build.gradle

@@ -29,7 +29,7 @@ dependencies {
 
 integTest {
     cluster {
-        plugin 'ingest-geoip', project(':plugins:ingest-geoip')
+        plugin ':plugins:ingest-geoip'
         setting 'script.inline', 'true'
         setting 'path.scripts', "${project.buildDir}/resources/test/scripts"
     }

qa/smoke-test-plugins/build.gradle

@@ -25,8 +25,7 @@ ext.pluginsCount = 0
 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
   integTest {
     cluster {
-      // need to get a non-decorated project object, so must re-lookup the project by path
+      plugin subproj.path
-      plugin subproj.name, project(subproj.path)
     }
   }
   pluginsCount += 1