From 2fb3cdceff39b1853f032aab75249be0dbe0e4d0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 15 Jul 2016 14:34:21 -0700 Subject: [PATCH] Build: Simplify plugin configuration for rest tests This change removes the multiple ways that plugins can be added to the integ test cluster. It also removes the use of the default configuration, and instead adds a zip configuration to all plugins. This will enable using project substitutions with plugins, which must be done with the default configuration. --- build.gradle | 5 ++ .../gradle/plugin/PluginBuildPlugin.groovy | 15 ++-- .../gradle/test/ClusterConfiguration.groovy | 19 ++--- .../gradle/test/ClusterFormationTasks.groovy | 69 ++++++++----------- .../gradle/test/RestIntegTestTask.groovy | 3 +- .../elasticsearch/gradle/test/RunTask.groovy | 6 +- docs/build.gradle | 10 +-- .../build.gradle | 2 +- qa/smoke-test-plugins/build.gradle | 3 +- 9 files changed, 67 insertions(+), 65 deletions(-) diff --git a/build.gradle b/build.gradle index 540f27503f0..807e3cfd222 100644 --- a/build.gradle +++ b/build.gradle @@ -173,6 +173,11 @@ subprojects { "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm', "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb', "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', + // for transport client + "org.elasticsearch.plugin:transport-netty3-client:${version}": ':modules:transport-netty3', + "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex', + "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache', + "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator', ] configurations.all { resolutionStrategy.dependencySubstitution { DependencySubstitutions subs -> diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index d993009111c..7d9af217bab 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -71,8 +71,8 @@ public class PluginBuildPlugin extends BuildPlugin { project.integTest.clusterConfig.module(project) project.tasks.run.clusterConfig.module(project) } else { - project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files) - project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files) + project.integTest.clusterConfig.plugin(project.path) + project.tasks.run.clusterConfig.plugin(project.path) addZipPomGeneration(project) } @@ -83,6 +83,7 @@ public class PluginBuildPlugin extends BuildPlugin { } createIntegTestTask(project) createBundleTask(project) + project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime')) project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build } @@ -141,13 +142,9 @@ public class PluginBuildPlugin extends BuildPlugin { } project.assemble.dependsOn(bundle) - // remove jar from the archives (things that will be published), and set it to the zip - project.configurations.archives.artifacts.removeAll { it.archiveTask.is project.jar } - project.artifacts.add('archives', bundle) - - // also make the zip the default artifact (used when depending on this project) - project.configurations.getByName('default').extendsFrom = [] - project.artifacts.add('default', bundle) + // also make the zip available as a configuration (used when depending on this project) + project.configurations.create('zip') + project.artifacts.add('zip', bundle) } /** Adds a task to move jar and associated files to a "-client" name. */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 19b41cc8cde..d45741a50b8 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -20,12 +20,15 @@ package org.elasticsearch.gradle.test import org.gradle.api.GradleException import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration import org.gradle.api.file.FileCollection import org.gradle.api.tasks.Input /** Configuration for an elasticsearch cluster, used for integration tests. */ class ClusterConfiguration { + private final Project project + @Input String distribution = 'integ-test-zip' @@ -77,6 +80,10 @@ class ClusterConfiguration { return tmpFile.exists() } + public ClusterConfiguration(Project project) { + this.project = project + } + Map systemProperties = new HashMap<>() Map settings = new HashMap<>() @@ -84,7 +91,7 @@ class ClusterConfiguration { // map from destination path, to source file Map extraConfigFiles = new HashMap<>() - LinkedHashMap plugins = new LinkedHashMap<>() + LinkedHashMap plugins = new LinkedHashMap<>() List modules = new ArrayList<>() @@ -101,13 +108,9 @@ class ClusterConfiguration { } @Input - void plugin(String name, FileCollection file) { - plugins.put(name, file) - } - - @Input - void plugin(String name, Project pluginProject) { - plugins.put(name, pluginProject) + void plugin(String path) { + Project pluginProject = project.project(path) + plugins.put(pluginProject.name, pluginProject) } /** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index c3004a64b86..442882dfe99 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -167,7 +167,7 @@ class ClusterFormationTasks { } // install plugins - for (Map.Entry plugin : node.config.plugins.entrySet()) { + for (Map.Entry plugin : node.config.plugins.entrySet()) { String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin') setup = configureInstallPluginTask(taskName(task, node, actionName), project, setup, node, plugin.getValue()) } @@ -326,38 +326,34 @@ class ClusterFormationTasks { Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) List pluginFiles = [] - for (Map.Entry plugin : node.config.plugins.entrySet()) { - FileCollection pluginZip - if (plugin.getValue() instanceof Project) { - Project pluginProject = plugin.getValue() - if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) { - throw new GradleException("Task ${name} cannot project ${pluginProject.path} which is not an esplugin") - } - String configurationName = "_plugin_${pluginProject.path}" - Configuration configuration = project.configurations.findByName(configurationName) - if (configuration == null) { - configuration = project.configurations.create(configurationName) - } - project.dependencies.add(configurationName, pluginProject) - setup.dependsOn(pluginProject.tasks.bundlePlugin) - pluginZip = configuration + for (Map.Entry plugin : node.config.plugins.entrySet()) { - // also allow rest tests to use the rest spec from the plugin - Copy copyRestSpec = null - for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) { - File restApiDir = new File(resourceDir, 'rest-api-spec/api') - if (restApiDir.exists() == false) continue - if (copyRestSpec == null) { - copyRestSpec = project.tasks.create(name: pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec'), type: Copy) - copyPlugins.dependsOn(copyRestSpec) - copyRestSpec.into(project.sourceSets.test.output.resourcesDir) - } - copyRestSpec.from(resourceDir).include('rest-api-spec/api/**') - } - } else { - pluginZip = plugin.getValue() + Project pluginProject = plugin.getValue() + if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) { + throw new GradleException("Task ${name} cannot project ${pluginProject.path} which is not an esplugin") } - pluginFiles.add(pluginZip) + String configurationName = "_plugin_${pluginProject.path}" + Configuration configuration = project.configurations.findByName(configurationName) + if (configuration == null) { + configuration = project.configurations.create(configurationName) + } + project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip')) + setup.dependsOn(pluginProject.tasks.bundlePlugin) + + // also allow rest tests to use the rest spec from the plugin + String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec') + Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName) + for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) { + File restApiDir = new File(resourceDir, 'rest-api-spec/api') + if (restApiDir.exists() == false) continue + if (copyRestSpec == null) { + copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy) + copyPlugins.dependsOn(copyRestSpec) + copyRestSpec.into(project.sourceSets.test.output.resourcesDir) + } + copyRestSpec.from(resourceDir).include('rest-api-spec/api/**') + } + pluginFiles.add(configuration) } copyPlugins.into(node.pluginsTmpDir) @@ -379,15 +375,10 @@ class ClusterFormationTasks { return installModule } - static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Object plugin) { - FileCollection pluginZip - if (plugin instanceof Project) { - pluginZip = project.configurations.getByName("_plugin_${plugin.path}") - } else { - pluginZip = plugin - } + static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin) { + FileCollection pluginZip = project.configurations.getByName("_plugin_${plugin.path}") // delay reading the file location until execution time by wrapping in a closure within a GString - String file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}" + Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}" Object[] args = [new File(node.homeDir, 'bin/elasticsearch-plugin'), 'install', file] return configureExecTask(name, project, setup, node, args) } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index fedcf6e87d3..c6463d28811 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -32,7 +32,7 @@ import org.gradle.util.ConfigureUtil */ public class RestIntegTestTask extends RandomizedTestingTask { - ClusterConfiguration clusterConfig = new ClusterConfiguration() + ClusterConfiguration clusterConfig /** Flag indicating whether the rest tests in the rest spec should be run. */ @Input @@ -44,6 +44,7 @@ public class RestIntegTestTask extends RandomizedTestingTask { dependsOn(project.testClasses) classpath = project.sourceSets.test.runtimeClasspath testClassesDir = project.sourceSets.test.output.classesDir + clusterConfig = new ClusterConfiguration(project) // start with the common test configuration configure(BuildPlugin.commonTestConfig(project)) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy index bebed415ad8..f045c95740b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy @@ -7,11 +7,15 @@ import org.gradle.util.ConfigureUtil public class RunTask extends DefaultTask { - ClusterConfiguration clusterConfig = new ClusterConfiguration(httpPort: 9200, transportPort: 9300, daemonize: false) + ClusterConfiguration clusterConfig public RunTask() { description = "Runs elasticsearch with '${project.path}'" group = 'Verification' + clusterConfig = new ClusterConfiguration(project) + clusterConfig.httpPort = 9200 + clusterConfig.transportPort = 9300 + clusterConfig.daemonize = false project.afterEvaluate { ClusterFormationTasks.setup(project, this, clusterConfig) } diff --git a/docs/build.gradle b/docs/build.gradle index 41e3e352b8d..26560ce064a 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -36,16 +36,18 @@ integTest { } // Build the cluser with all plugins + project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj -> /* Skip repositories. We just aren't going to be able to test them so it * doesn't make sense to waste time installing them. */ if (subproj.path.startsWith(':plugins:repository-')) { return } - integTest { - cluster { - // We need a non-decorated project object, so we lookup the project by path - plugin subproj.name, project(subproj.path) + subproj.afterEvaluate { // need to wait until the project has been configured + integTest { + cluster { + plugin subproj.path + } } } } diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle index 9c9943a1712..df90bf5b982 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle +++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle @@ -29,7 +29,7 @@ dependencies { integTest { cluster { - plugin 'ingest-geoip', project(':plugins:ingest-geoip') + plugin ':plugins:ingest-geoip' setting 'script.inline', 'true' setting 'path.scripts', "${project.buildDir}/resources/test/scripts" } diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle index bc8eace704e..ab69b02fc8c 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/qa/smoke-test-plugins/build.gradle @@ -25,8 +25,7 @@ ext.pluginsCount = 0 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj -> integTest { cluster { - // need to get a non-decorated project object, so must re-lookup the project by path - plugin subproj.name, project(subproj.path) + plugin subproj.path } } pluginsCount += 1