Build: Add gradle plugin for configuring meta plugin (#28276)

This commit adds a gradle plugin to ease development of meta plugins.
Applying the plugin will generated the meta plugin properties based on
the es_meta_plugin configuration object, which includes name and
description. The plugins to include within the meta plugin are
configured through the `plugins` list. An integ test task is also
automatically added.

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/MetaPluginBuildPlugin.groovy

@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.plugin
+
+import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.test.RestTestPlugin
+import org.elasticsearch.gradle.test.StandaloneRestTestPlugin
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.file.FileCopyDetails
+import org.gradle.api.file.RelativePath
+import org.gradle.api.tasks.bundling.Zip
+
+class MetaPluginBuildPlugin implements Plugin<Project> {
+
+    @Override
+    void apply(Project project) {
+        project.plugins.apply(StandaloneRestTestPlugin)
+        project.plugins.apply(RestTestPlugin)
+
+        createBundleTask(project)
+
+        project.integTestCluster {
+            dependsOn(project.bundlePlugin)
+            distribution = 'zip'
+            setupCommand 'installMetaPlugin',
+                    'bin/elasticsearch-plugin', 'install', 'file:' + project.bundlePlugin.archivePath
+        }
+    }
+
+    private static void createBundleTask(Project project) {
+
+        MetaPluginPropertiesTask buildProperties = project.tasks.create('pluginProperties', MetaPluginPropertiesTask.class)
+
+        // create the actual bundle task, which zips up all the files for the plugin
+        Zip bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [buildProperties]) {
+            into('elasticsearch') {
+                from(buildProperties.descriptorOutput.parentFile) {
+                    // plugin properties file
+                    include(buildProperties.descriptorOutput.name)
+                }
+            }
+
+        }
+        project.assemble.dependsOn(bundle)
+
+        // a super hacky way to inject code to run at the end of each of the bundled plugin's configuration
+        // to add itself back to this meta plugin zip
+        project.afterEvaluate {
+            buildProperties.extension.plugins.each { String bundledPluginProjectName ->
+                Project bundledPluginProject = project.project(bundledPluginProjectName)
+                bundledPluginProject.afterEvaluate {
+                    bundle.configure {
+                        dependsOn bundledPluginProject.bundlePlugin
+                        from(project.zipTree(bundledPluginProject.bundlePlugin.outputs.files.singleFile)) {
+                            eachFile { FileCopyDetails details ->
+                                details.relativePath = new RelativePath(true, 'elasticsearch', bundledPluginProjectName, details.name)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/MetaPluginPropertiesExtension.groovy

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.plugin
+
+import org.gradle.api.Project
+import org.gradle.api.tasks.Input
+
+/**
+ * A container for meta plugin properties that will be written to the meta plugin descriptor, for easy
+ * manipulation in the gradle DSL.
+ */
+class MetaPluginPropertiesExtension {
+    @Input
+    String name
+
+    @Input
+    String description
+
+    /**
+     * The plugins this meta plugin wraps.
+     * Note this is not written to the plugin descriptor, but used to setup the final zip file task.
+     */
+    @Input
+    List<String> plugins
+
+    MetaPluginPropertiesExtension(Project project) {
+        name = project.name
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/MetaPluginPropertiesTask.groovy

@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.plugin
+
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Task
+import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.OutputFile
+
+class MetaPluginPropertiesTask extends Copy {
+
+    MetaPluginPropertiesExtension extension
+
+    @OutputFile
+    File descriptorOutput = new File(project.buildDir, 'generated-resources/meta-plugin-descriptor.properties')
+
+    MetaPluginPropertiesTask() {
+        File templateFile = new File(project.buildDir, "templates/${descriptorOutput.name}")
+        Task copyPluginPropertiesTemplate = project.tasks.create('copyPluginPropertiesTemplate') {
+            doLast {
+                InputStream resourceTemplate = PluginPropertiesTask.getResourceAsStream("/${descriptorOutput.name}")
+                templateFile.parentFile.mkdirs()
+                templateFile.setText(resourceTemplate.getText('UTF-8'), 'UTF-8')
+            }
+        }
+
+        dependsOn(copyPluginPropertiesTemplate)
+        extension = project.extensions.create('es_meta_plugin', MetaPluginPropertiesExtension, project)
+        project.afterEvaluate {
+            // check require properties are set
+            if (extension.name == null) {
+                throw new InvalidUserDataException('name is a required setting for es_meta_plugin')
+            }
+            if (extension.description == null) {
+                throw new InvalidUserDataException('description is a required setting for es_meta_plugin')
+            }
+            // configure property substitution
+            from(templateFile.parentFile).include(descriptorOutput.name)
+            into(descriptorOutput.parentFile)
+            Map<String, String> properties = generateSubstitutions()
+            expand(properties)
+            inputs.properties(properties)
+        }
+    }
+
+    Map<String, String> generateSubstitutions() {
+        return ['name': extension.name,
+                'description': extension.description
+        ]
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.plugin.MetaPluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
@@ -753,9 +754,9 @@ class ClusterFormationTasks {
     }
 
     static void verifyProjectHasBuildPlugin(String name, String version, Project project, Project pluginProject) {
-        if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) {
+        if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false && pluginProject.plugins.hasPlugin(MetaPluginBuildPlugin) == false) {
             throw new GradleException("Task [${name}] cannot add plugin [${pluginProject.path}] with version [${version}] to project's " +
-                    "[${project.path}] dependencies: the plugin is not an esplugin")
+                    "[${project.path}] dependencies: the plugin is not an esplugin or es_meta_plugin")
         }
     }
 }

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.es-meta-plugin.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.plugin.MetaPluginBuildPlugin

plugins/examples/meta-plugin/build.gradle

@@ -18,39 +18,11 @@
  */
 
 // A meta plugin packaging example that bundles multiple plugins in a single zip.
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
 
-File plugins = new File(buildDir, 'plugins-unzip')
+apply plugin: 'elasticsearch.es-meta-plugin'
-subprojects {
+
-    // unzip the subproject plugins
+es_meta_plugin {
-    task unzip(type:Copy, dependsOn: "${project.path}:bundlePlugin") {
+  name 'meta-plugin'
-        File dest = new File(plugins, project.name)
+  description 'example meta plugin'
-        from { zipTree(project(project.path).bundlePlugin.outputs.files.singleFile) }
+  plugins = ['dummy-plugin1', 'dummy-plugin2']
-        eachFile { f -> f.path = f.path.replaceFirst('elasticsearch', '') }
-        into dest
-    }
 }
-
-// Build the meta plugin zip from the subproject plugins (unzipped)
-task buildZip(type:Zip) {
-    subprojects.each { dependsOn("${it.name}:unzip") }
-    from plugins
-    from 'src/main/resources/meta-plugin-descriptor.properties'
-    into 'elasticsearch'
-    includeEmptyDirs false
-}
-
-integTestCluster {
-    dependsOn buildZip
-
-    // This is important, so that all the modules are available too.
-    // There are index templates that use token filters that are in analysis-module and
-    // processors are being used that are in ingest-common module.
-    distribution = 'zip'
-
-    // Install the meta plugin before start.
-    setupCommand 'installMetaPlugin',
-            'bin/elasticsearch-plugin', 'install', 'file:' + buildZip.archivePath
-}
-check.dependsOn integTest