From be8c094e8c1022d07595c0af643bb6c09688ac7c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 11 May 2018 21:58:18 -0700 Subject: [PATCH] Build: Add mavenPlugin cluster configuration method (#30541) This commit adds the ability to specify a plugin from maven for a test cluster to use. Currently, only local projects may be used as plugins, except when testing bwc, where the coordinates of the project are used. However, that assumes all projects always keep the same coordinates, or are even still plugins, which is no longer the case for x-pack. The full cluster and rolling restart tests are changed to use this new method when pulling x-pack versions before 6.3.0. --- .../gradle/test/ClusterConfiguration.groovy | 7 +- .../gradle/test/ClusterFormationTasks.groovy | 101 +++++++++++------- x-pack/qa/full-cluster-restart/build.gradle | 2 +- x-pack/qa/rolling-upgrade-basic/build.gradle | 2 +- x-pack/qa/rolling-upgrade/build.gradle | 2 +- 5 files changed, 69 insertions(+), 45 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index afbfc747541..cd6c7c36ee6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -147,7 +147,7 @@ class ClusterConfiguration { // map from destination path, to source file Map extraConfigFiles = new HashMap<>() - LinkedHashMap plugins = new LinkedHashMap<>() + LinkedHashMap plugins = new LinkedHashMap<>() List modules = new ArrayList<>() @@ -185,6 +185,11 @@ class ClusterConfiguration { plugins.put(pluginProject.name, pluginProject) } + @Input + void mavenPlugin(String name, String mavenCoords) { + plugins.put(name, mavenCoords) + } + /** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */ @Input void module(Project moduleProject) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index ed066ddc96b..b9a38396318 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -99,8 +99,8 @@ class ClusterFormationTasks { // from mirrors using gradles built-in mechanism etc. configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion) - for (Map.Entry entry : config.plugins.entrySet()) { - configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(), bwcPlugins, config.bwcVersion) + for (Map.Entry entry : config.plugins.entrySet()) { + configureBwcPluginDependency(project, entry.getValue(), bwcPlugins, config.bwcVersion) } bwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS) bwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS) @@ -150,10 +150,15 @@ class ClusterFormationTasks { } /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */ - static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, Version elasticsearchVersion) { - verifyProjectHasBuildPlugin(name, elasticsearchVersion, project, pluginProject) - final String pluginName = findPluginName(pluginProject) - project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip") + static void configureBwcPluginDependency(Project project, Object plugin, Configuration configuration, Version elasticsearchVersion) { + if (plugin instanceof Project) { + Project pluginProject = (Project)plugin + verifyProjectHasBuildPlugin(configuration.name, elasticsearchVersion, project, pluginProject) + final String pluginName = findPluginName(pluginProject) + project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip") + } else { + project.dependencies.add(configuration.name, "${plugin}@zip") + } } /** @@ -210,9 +215,9 @@ class ClusterFormationTasks { } // install plugins - for (Map.Entry plugin : node.config.plugins.entrySet()) { - String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin') - setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue(), prefix) + for (String pluginName : node.config.plugins.keySet()) { + String actionName = pluginTaskName('install', pluginName, 'Plugin') + setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, pluginName, prefix) } // sets up any extra config files that need to be copied over to the ES instance; @@ -444,31 +449,40 @@ class ClusterFormationTasks { Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) List pluginFiles = [] - for (Map.Entry plugin : node.config.plugins.entrySet()) { + for (Map.Entry plugin : node.config.plugins.entrySet()) { - Project pluginProject = plugin.getValue() - verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject) - String configurationName = pluginConfigurationName(prefix, pluginProject) + String configurationName = pluginConfigurationName(prefix, plugin.key) Configuration configuration = project.configurations.findByName(configurationName) if (configuration == null) { configuration = project.configurations.create(configurationName) } - project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip')) - setup.dependsOn(pluginProject.tasks.bundlePlugin) - // also allow rest tests to use the rest spec from the plugin - String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec') - Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName) - for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) { - File restApiDir = new File(resourceDir, 'rest-api-spec/api') - if (restApiDir.exists() == false) continue - if (copyRestSpec == null) { - copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy) - copyPlugins.dependsOn(copyRestSpec) - copyRestSpec.into(project.sourceSets.test.output.resourcesDir) + if (plugin.getValue() instanceof Project) { + Project pluginProject = plugin.getValue() + verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject) + + project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip')) + setup.dependsOn(pluginProject.tasks.bundlePlugin) + + // also allow rest tests to use the rest spec from the plugin + String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec') + Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName) + for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) { + File restApiDir = new File(resourceDir, 'rest-api-spec/api') + if (restApiDir.exists() == false) continue + if (copyRestSpec == null) { + copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy) + copyPlugins.dependsOn(copyRestSpec) + copyRestSpec.into(project.sourceSets.test.output.resourcesDir) + } + copyRestSpec.from(resourceDir).include('rest-api-spec/api/**') } - copyRestSpec.from(resourceDir).include('rest-api-spec/api/**') + } else { + project.dependencies.add(configurationName, "${plugin.getValue()}@zip") } + + + pluginFiles.add(configuration) } @@ -477,32 +491,37 @@ class ClusterFormationTasks { return copyPlugins } - private static String pluginConfigurationName(final String prefix, final Project project) { - return "_plugin_${prefix}_${project.path}".replace(':', '_') + private static String pluginConfigurationName(final String prefix, final String name) { + return "_plugin_${prefix}_${name}".replace(':', '_') } - private static String pluginBwcConfigurationName(final String prefix, final Project project) { - return "_plugin_bwc_${prefix}_${project.path}".replace(':', '_') + private static String pluginBwcConfigurationName(final String prefix, final String name) { + return "_plugin_bwc_${prefix}_${name}".replace(':', '_') } /** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */ static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) { Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins") - for (Map.Entry plugin : node.config.plugins.entrySet()) { - Project pluginProject = plugin.getValue() - verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject) - String configurationName = pluginBwcConfigurationName(prefix, pluginProject) + for (Map.Entry plugin : node.config.plugins.entrySet()) { + String configurationName = pluginBwcConfigurationName(prefix, plugin.key) Configuration configuration = project.configurations.findByName(configurationName) if (configuration == null) { configuration = project.configurations.create(configurationName) } - final String depName = findPluginName(pluginProject) + if (plugin.getValue() instanceof Project) { + Project pluginProject = plugin.getValue() + verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject) - Dependency dep = bwcPlugins.dependencies.find { - it.name == depName + final String depName = findPluginName(pluginProject) + + Dependency dep = bwcPlugins.dependencies.find { + it.name == depName + } + configuration.dependencies.add(dep) + } else { + project.dependencies.add(configurationName, "${plugin.getValue()}@zip") } - configuration.dependencies.add(dep) } Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) { @@ -527,12 +546,12 @@ class ClusterFormationTasks { return installModule } - static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) { + static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, String pluginName, String prefix) { final FileCollection pluginZip; if (node.nodeVersion != VersionProperties.elasticsearch) { - pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, plugin)) + pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, pluginName)) } else { - pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, plugin)) + pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, pluginName)) } // delay reading the file location until execution time by wrapping in a closure within a GString final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}" diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle index 8f5952d61ed..b2bb7a63f6f 100644 --- a/x-pack/qa/full-cluster-restart/build.gradle +++ b/x-pack/qa/full-cluster-restart/build.gradle @@ -141,7 +141,7 @@ subprojects { configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { dependsOn copyTestNodeKeystore if (version.before('6.3.0')) { - plugin xpackProject('plugin').path + mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}" } bwcVersion = version numBwcNodes = 2 diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index bb1b5c58c4a..91a6d106c98 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -82,7 +82,7 @@ for (Version version : bwcVersions.wireCompatible) { configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { if (version.before('6.3.0')) { - plugin xpackProject('plugin').path + mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}" } bwcVersion = version numBwcNodes = 2 diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 433dc08e1f3..6e93041e9a0 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -123,7 +123,7 @@ subprojects { configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { dependsOn copyTestNodeKeystore if (version.before('6.3.0')) { - plugin xpackProject('plugin').path + mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}" } String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users' setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'