Add the ability to bundle multiple plugins into a meta plugin (#28022)

This commit adds the ability to package multiple plugins in a single zip.
The zip file for a meta plugin must contains the following structure:

|____elasticsearch/
| |____   <plugin1> <-- The plugin files for plugin1 (the content of the elastisearch directory)
| |____   <plugin2>  <-- The plugin files for plugin2
| |____   meta-plugin-descriptor.properties <-- example contents below
The meta plugin properties descriptor is mandatory and must contain the following properties:

description: simple summary of the meta plugin.
name: the meta plugin name
The installation process installs each plugin in a sub-folder inside the meta plugin directory.
The example above would create the following structure in the plugins directory:

|_____ plugins
| |____   <name_of_the_meta_plugin>
| | |____   meta-plugin-descriptor.properties
| | |____   <plugin1>
| | |____   <plugin2>
If the sub plugins contain a config or a bin directory, they are copied in a sub folder inside the meta plugin config/bin directory.

|_____ config
| |____   <name_of_the_meta_plugin>
| | |____   <plugin1>
| | |____   <plugin2>

|_____ bin
| |____   <name_of_the_meta_plugin>
| | |____   <plugin1>
| | |____   <plugin2>
The sub-plugins are loaded at startup like normal plugins with the same restrictions; they have a separate class loader and a sub-plugin
cannot have the same name than another plugin (or a sub-plugin inside another meta plugin).

It is also not possible to remove a sub-plugin inside a meta plugin, only full removal of the meta plugin is allowed.

Closes #27316

buildSrc/src/main/resources/meta-plugin-descriptor.properties

@@ -0,0 +1,21 @@
+# Elasticsearch meta plugin descriptor file
+# This file must exist as 'meta-plugin-descriptor.properties' in a folder named `elasticsearch`.
+#
+### example meta plugin for "meta-foo"
+#
+# meta-foo.zip <-- zip file for the meta plugin, with this structure:
+#|____elasticsearch/
+#| |____   <bundled_plugin_1> <-- The plugin files for bundled_plugin_1 (the content of the elastisearch directory)
+#| |____   <bundled_plugin_2>  <-- The plugin files for bundled_plugin_2
+#| |____   meta-plugin-descriptor.properties <-- example contents below:
+#
+# description=My meta plugin
+# name=meta-foo
+#
+### mandatory elements for all meta plugins:
+#
+# 'description': simple summary of the meta plugin
+description=${description}
+#
+# 'name': the meta plugin name
+name=${name}

core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java

@@ -30,6 +30,7 @@ import org.elasticsearch.plugins.PluginInfo;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 
 /**
@@ -60,23 +61,23 @@ public class PluginsAndModules implements Writeable, ToXContentFragment {
      */
     public List<PluginInfo> getPluginInfos() {
         List<PluginInfo> plugins = new ArrayList<>(this.plugins);
-        Collections.sort(plugins, (p1, p2) -> p1.getName().compareTo(p2.getName()));
+        Collections.sort(plugins, Comparator.comparing(PluginInfo::getName));
         return plugins;
     }
-    
+
     /**
      * Returns an ordered list based on modules name
      */
     public List<PluginInfo> getModuleInfos() {
         List<PluginInfo> modules = new ArrayList<>(this.modules);
-        Collections.sort(modules, (p1, p2) -> p1.getName().compareTo(p2.getName()));
+        Collections.sort(modules, Comparator.comparing(PluginInfo::getName));
         return modules;
     }
 
     public void addPlugin(PluginInfo info) {
         plugins.add(info);
     }
-    
+
     public void addModule(PluginInfo info) {
         modules.add(info);
     }

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -163,16 +163,8 @@ final class Security {
     static Map<String,Policy> getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
         Map<String,Policy> map = new HashMap<>();
         // collect up set of plugins and modules by listing directories.
-        Set<Path> pluginsAndModules = new LinkedHashSet<>(); // order is already lost, but some filesystems have it
+        Set<Path> pluginsAndModules = new LinkedHashSet<>(PluginInfo.extractAllPlugins(environment.pluginsFile()));
-        if (Files.exists(environment.pluginsFile())) {
+
-            try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
-                for (Path plugin : stream) {
-                    if (pluginsAndModules.add(plugin) == false) {
-                        throw new IllegalStateException("duplicate plugin: " + plugin);
-                    }
-                }
-            }
-        }
         if (Files.exists(environment.modulesFile())) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.modulesFile())) {
                 for (Path module : stream) {

core/src/main/java/org/elasticsearch/bootstrap/Spawner.java

@@ -21,14 +21,12 @@ package org.elasticsearch.bootstrap;
 
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginInfo;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -72,27 +70,23 @@ final class Spawner implements Closeable {
          * For each plugin, attempt to spawn the controller daemon. Silently ignore any plugin that
          * don't include a controller for the correct platform.
          */
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsFile)) {
+        List<Path> paths = PluginInfo.extractAllPlugins(pluginsFile);
-            for (final Path plugin : stream) {
+        for (Path plugin : paths) {
-                if (FileSystemUtils.isDesktopServicesStore(plugin)) {
+            final PluginInfo info = PluginInfo.readFromProperties(plugin);
-                    continue;
+            final Path spawnPath = Platforms.nativeControllerPath(plugin);
-                }
+            if (!Files.isRegularFile(spawnPath)) {
-                final PluginInfo info = PluginInfo.readFromProperties(plugin);
+                continue;
-                final Path spawnPath = Platforms.nativeControllerPath(plugin);
-                if (!Files.isRegularFile(spawnPath)) {
-                    continue;
-                }
-                if (!info.hasNativeController()) {
-                    final String message = String.format(
-                            Locale.ROOT,
-                            "plugin [%s] does not have permission to fork native controller",
-                            plugin.getFileName());
-                    throw new IllegalArgumentException(message);
-                }
-                final Process process =
-                        spawnNativePluginController(spawnPath, environment.tmpFile());
-                processes.add(process);
             }
+            if (!info.hasNativeController()) {
+                final String message = String.format(
+                    Locale.ROOT,
+                    "plugin [%s] does not have permission to fork native controller",
+                    plugin.getFileName());
+                throw new IllegalArgumentException(message);
+            }
+            final Process process =
+                spawnNativePluginController(spawnPath, environment.tmpFile());
+            processes.add(process);
         }
     }
 

core/src/main/java/org/elasticsearch/plugins/MetaPluginInfo.java

@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Properties;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * An in-memory representation of the meta plugin descriptor.
+ */
+public class MetaPluginInfo {
+    static final String ES_META_PLUGIN_PROPERTIES = "meta-plugin-descriptor.properties";
+
+    private final String name;
+    private final String description;
+
+    /**
+     * Construct plugin info.
+     *
+     * @param name                the name of the plugin
+     * @param description         a description of the plugin
+     */
+    private MetaPluginInfo(String name, String description) {
+        this.name = name;
+        this.description = description;
+    }
+
+    /**
+     * @return Whether the provided {@code path} is a meta plugin.
+     */
+    public static boolean isMetaPlugin(final Path path) {
+        return Files.exists(path.resolve(ES_META_PLUGIN_PROPERTIES));
+    }
+
+    /**
+     * @return Whether the provided {@code path} is a meta properties file.
+     */
+    public static boolean isPropertiesFile(final Path path) {
+        return ES_META_PLUGIN_PROPERTIES.equals(path.getFileName().toString());
+    }
+
+    /** reads (and validates) meta plugin metadata descriptor file */
+
+    /**
+     * Reads and validates the meta plugin descriptor file.
+     *
+     * @param path the path to the root directory for the meta plugin
+     * @return the meta plugin info
+     * @throws IOException if an I/O exception occurred reading the meta plugin descriptor
+     */
+    public static MetaPluginInfo readFromProperties(final Path path) throws IOException {
+        final Path descriptor = path.resolve(ES_META_PLUGIN_PROPERTIES);
+
+        final Map<String, String> propsMap;
+        {
+            final Properties props = new Properties();
+            try (InputStream stream = Files.newInputStream(descriptor)) {
+                props.load(stream);
+            }
+            propsMap = props.stringPropertyNames().stream().collect(Collectors.toMap(Function.identity(), props::getProperty));
+        }
+
+        final String name = propsMap.remove("name");
+        if (name == null || name.isEmpty()) {
+            throw new IllegalArgumentException(
+                    "property [name] is missing for meta plugin in [" + descriptor + "]");
+        }
+        final String description = propsMap.remove("description");
+        if (description == null) {
+            throw new IllegalArgumentException(
+                    "property [description] is missing for meta plugin [" + name + "]");
+        }
+
+        if (propsMap.isEmpty() == false) {
+            throw new IllegalArgumentException("Unknown properties in meta plugin descriptor: " + propsMap.keySet());
+        }
+
+        return new MetaPluginInfo(name, description);
+    }
+
+    /**
+     * The name of the meta plugin.
+     *
+     * @return the meta plugin name
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * The description of the meta plugin.
+     *
+     * @return the meta plugin description
+     */
+    public String getDescription() {
+        return description;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        MetaPluginInfo that = (MetaPluginInfo) o;
+
+        if (!name.equals(that.name)) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return name.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder information = new StringBuilder()
+                .append("- Plugin information:\n")
+                .append("Name: ").append(name).append("\n")
+                .append("Description: ").append(description);
+        return information.toString();
+    }
+
+}

core/src/main/java/org/elasticsearch/plugins/PluginInfo.java

@@ -22,7 +22,9 @@ package org.elasticsearch.plugins;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -31,14 +33,19 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -125,7 +132,46 @@ public class PluginInfo implements Writeable, ToXContentObject {
         }
     }
 
-    /** reads (and validates) plugin metadata descriptor file */
+    /**
+     * Extracts all {@link PluginInfo} from the provided {@code rootPath} expanding meta plugins if needed.
+     * @param rootPath the path where the plugins are installed
+     * @return A list of all plugin paths installed in the {@code rootPath}
+     * @throws IOException if an I/O exception occurred reading the plugin descriptors
+     */
+    public static List<Path> extractAllPlugins(final Path rootPath) throws IOException {
+        final List<Path> plugins = new LinkedList<>();  // order is already lost, but some filesystems have it
+        final Set<String> seen = new HashSet<>();
+        if (Files.exists(rootPath)) {
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(rootPath)) {
+                for (Path plugin : stream) {
+                    if (FileSystemUtils.isDesktopServicesStore(plugin) ||
+                            plugin.getFileName().toString().startsWith(".removing-")) {
+                        continue;
+                    }
+                    if (seen.add(plugin.getFileName().toString()) == false) {
+                        throw new IllegalStateException("duplicate plugin: " + plugin);
+                    }
+                    if (MetaPluginInfo.isMetaPlugin(plugin)) {
+                        try (DirectoryStream<Path> subStream = Files.newDirectoryStream(plugin)) {
+                            for (Path subPlugin : subStream) {
+                                if (MetaPluginInfo.isPropertiesFile(subPlugin) ||
+                                        FileSystemUtils.isDesktopServicesStore(subPlugin)) {
+                                    continue;
+                                }
+                                if (seen.add(subPlugin.getFileName().toString()) == false) {
+                                    throw new IllegalStateException("duplicate plugin: " + subPlugin);
+                                }
+                                plugins.add(subPlugin);
+                            }
+                        }
+                    } else {
+                        plugins.add(plugin);
+                    }
+                }
+            }
+        }
+        return plugins;
+    }
 
     /**
      * Reads and validates the plugin descriptor file.
@@ -341,16 +387,19 @@ public class PluginInfo implements Writeable, ToXContentObject {
 
     @Override
     public String toString() {
-        final StringBuilder information = new StringBuilder()
+        return toString("");
-                .append("- Plugin information:\n")
-                .append("Name: ").append(name).append("\n")
-                .append("Description: ").append(description).append("\n")
-                .append("Version: ").append(version).append("\n")
-                .append("Native Controller: ").append(hasNativeController).append("\n")
-                .append("Requires Keystore: ").append(requiresKeystore).append("\n")
-                .append("Extended Plugins: ").append(extendedPlugins).append("\n")
-                .append(" * Classname: ").append(classname);
-        return information.toString();
     }
 
+    public String toString(String prefix) {
+        final StringBuilder information = new StringBuilder()
+            .append(prefix).append("- Plugin information:\n")
+            .append(prefix).append("Name: ").append(name).append("\n")
+            .append(prefix).append("Description: ").append(description).append("\n")
+            .append(prefix).append("Version: ").append(version).append("\n")
+            .append(prefix).append("Native Controller: ").append(hasNativeController).append("\n")
+            .append(prefix).append("Requires Keystore: ").append(requiresKeystore).append("\n")
+            .append(prefix).append("Extended Plugins: ").append(extendedPlugins).append("\n")
+            .append(prefix).append(" * Classname: ").append(classname);
+        return information.toString();
+    }
 }

core/src/main/java/org/elasticsearch/plugins/PluginsService.java

@@ -34,7 +34,6 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -322,29 +321,20 @@ public class PluginsService extends AbstractComponent {
         Logger logger = Loggers.getLogger(PluginsService.class);
         Set<Bundle> bundles = new LinkedHashSet<>();
 
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
+        List<Path> infos = PluginInfo.extractAllPlugins(pluginsDirectory);
-            for (Path plugin : stream) {
+        for (Path plugin : infos) {
-                if (FileSystemUtils.isDesktopServicesStore(plugin)) {
+            logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
-                    continue;
+            final PluginInfo info;
-                }
+            try {
-                if (plugin.getFileName().toString().startsWith(".removing-")) {
+                info = PluginInfo.readFromProperties(plugin);
-                    continue;
+            } catch (IOException e) {
-                }
+                throw new IllegalStateException("Could not load plugin descriptor for existing plugin ["
-                logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
+                    + plugin.getFileName() + "]. Was the plugin built before 2.0?", e);
-                final PluginInfo info;
+            }
-                try {
+            if (bundles.add(new Bundle(info, plugin)) == false) {
-                    info = PluginInfo.readFromProperties(plugin);
+                throw new IllegalStateException("duplicate plugin: " + info);
-                } catch (IOException e) {
-                    throw new IllegalStateException("Could not load plugin descriptor for existing plugin ["
-                        + plugin.getFileName() + "]. Was the plugin built before 2.0?", e);
-                }
-
-                if (bundles.add(new Bundle(info, plugin)) == false) {
-                    throw new IllegalStateException("duplicate plugin: " + info);
-                }
             }
         }
-
         return bundles;
     }
 

core/src/test/java/org/elasticsearch/plugins/MetaPluginInfoTests.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.test.ESTestCase;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
+
+@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS")
+public class MetaPluginInfoTests extends ESTestCase {
+
+    public void testReadFromProperties() throws Exception {
+        Path pluginDir = createTempDir().resolve("fake-meta-plugin");
+        PluginTestUtil.writeMetaPluginProperties(pluginDir,
+            "description", "fake desc",
+            "name", "my_meta_plugin");
+        MetaPluginInfo info = MetaPluginInfo.readFromProperties(pluginDir);
+        assertEquals("my_meta_plugin", info.getName());
+        assertEquals("fake desc", info.getDescription());
+    }
+
+    public void testReadFromPropertiesNameMissing() throws Exception {
+        Path pluginDir = createTempDir().resolve("fake-meta-plugin");
+        PluginTestUtil.writeMetaPluginProperties(pluginDir);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetaPluginInfo.readFromProperties(pluginDir));
+        assertThat(e.getMessage(), containsString("property [name] is missing for"));
+
+        PluginTestUtil.writeMetaPluginProperties(pluginDir, "name", "");
+        e = expectThrows(IllegalArgumentException.class, () -> MetaPluginInfo.readFromProperties(pluginDir));
+        assertThat(e.getMessage(), containsString("property [name] is missing for"));
+    }
+
+    public void testReadFromPropertiesDescriptionMissing() throws Exception {
+        Path pluginDir = createTempDir().resolve("fake-meta-plugin");
+        PluginTestUtil.writeMetaPluginProperties(pluginDir, "name", "fake-meta-plugin");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetaPluginInfo.readFromProperties(pluginDir));
+        assertThat(e.getMessage(), containsString("[description] is missing"));
+    }
+
+    public void testUnknownProperties() throws Exception {
+        Path pluginDir = createTempDir().resolve("fake-meta-plugin");
+        PluginTestUtil.writeMetaPluginProperties(pluginDir,
+            "extra", "property",
+            "unknown", "property",
+            "description", "fake desc",
+            "name", "my_meta_plugin");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetaPluginInfo.readFromProperties(pluginDir));
+        assertThat(e.getMessage(), containsString("Unknown properties in meta plugin descriptor"));
+    }
+
+    public void testExtractAllPluginsWithDuplicates() throws Exception {
+        Path pluginDir = createTempDir().resolve("plugins");
+        // Simple plugin
+        Path plugin1 = pluginDir.resolve("plugin1");
+        Files.createDirectories(plugin1);
+        PluginTestUtil.writePluginProperties(plugin1,
+            "description", "fake desc",
+            "name", "plugin1",
+            "version", "1.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "FakePlugin");
+
+        // Meta plugin
+        Path metaPlugin = pluginDir.resolve("meta_plugin");
+        Files.createDirectory(metaPlugin);
+        PluginTestUtil.writeMetaPluginProperties(metaPlugin,
+            "description", "fake desc",
+            "name", "meta_plugin");
+        Path plugin2 = metaPlugin.resolve("plugin1");
+        Files.createDirectory(plugin2);
+        PluginTestUtil.writePluginProperties(plugin2,
+            "description", "fake desc",
+            "name", "plugin1",
+            "version", "1.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "FakePlugin");
+        Path plugin3 = metaPlugin.resolve("plugin2");
+        Files.createDirectory(plugin3);
+        PluginTestUtil.writePluginProperties(plugin3,
+            "description", "fake desc",
+            "name", "plugin2",
+            "version", "1.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "FakePlugin");
+
+        IllegalStateException exc =
+            expectThrows(IllegalStateException.class, () -> PluginInfo.extractAllPlugins(pluginDir));
+        assertThat(exc.getMessage(), containsString("duplicate plugin"));
+        assertThat(exc.getMessage(), endsWith("plugin1"));
+    }
+}

core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java

@@ -41,7 +41,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testReadFromProperties() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", "my_plugin",
             "version", "1.0",
@@ -58,25 +58,25 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testReadFromPropertiesNameMissing() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir);
+        PluginTestUtil.writePluginProperties(pluginDir);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir));
         assertThat(e.getMessage(), containsString("property [name] is missing in"));
 
-        PluginTestUtil.writeProperties(pluginDir, "name", "");
+        PluginTestUtil.writePluginProperties(pluginDir, "name", "");
         e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir));
         assertThat(e.getMessage(), containsString("property [name] is missing in"));
     }
 
     public void testReadFromPropertiesDescriptionMissing() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir, "name", "fake-plugin");
+        PluginTestUtil.writePluginProperties(pluginDir, "name", "fake-plugin");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir));
         assertThat(e.getMessage(), containsString("[description] is missing"));
     }
 
     public void testReadFromPropertiesVersionMissing() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(
+        PluginTestUtil.writePluginProperties(
                 pluginDir, "description", "fake desc", "name", "fake-plugin");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir));
         assertThat(e.getMessage(), containsString("[version] is missing"));
@@ -84,7 +84,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testReadFromPropertiesElasticsearchVersionMissing() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", "my_plugin",
             "version", "1.0");
@@ -94,7 +94,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testReadFromPropertiesJavaVersionMissing() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", "my_plugin",
             "elasticsearch.version", Version.CURRENT.toString(),
@@ -106,7 +106,7 @@ public class PluginInfoTests extends ESTestCase {
     public void testReadFromPropertiesJavaVersionIncompatible() throws Exception {
         String pluginName = "fake-plugin";
         Path pluginDir = createTempDir().resolve(pluginName);
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", pluginName,
             "elasticsearch.version", Version.CURRENT.toString(),
@@ -120,7 +120,7 @@ public class PluginInfoTests extends ESTestCase {
     public void testReadFromPropertiesBadJavaVersionFormat() throws Exception {
         String pluginName = "fake-plugin";
         Path pluginDir = createTempDir().resolve(pluginName);
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
                 "description", "fake desc",
                 "name", pluginName,
                 "elasticsearch.version", Version.CURRENT.toString(),
@@ -134,7 +134,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testReadFromPropertiesBogusElasticsearchVersion() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "version", "1.0",
             "name", "my_plugin",
@@ -145,7 +145,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testReadFromPropertiesOldElasticsearchVersion() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", "my_plugin",
             "version", "1.0",
@@ -156,7 +156,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testReadFromPropertiesJvmMissingClassname() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", "my_plugin",
             "version", "1.0",
@@ -168,7 +168,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testExtendedPluginsSingleExtension() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", "my_plugin",
             "version", "1.0",
@@ -182,7 +182,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testExtendedPluginsMultipleExtensions() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", "my_plugin",
             "version", "1.0",
@@ -196,7 +196,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testExtendedPluginsEmpty() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "description", "fake desc",
             "name", "my_plugin",
             "version", "1.0",
@@ -224,7 +224,7 @@ public class PluginInfoTests extends ESTestCase {
         List<PluginInfo> plugins = new ArrayList<>();
         plugins.add(new PluginInfo("c", "foo", "dummy", "dummyclass", Collections.emptyList(), randomBoolean(), randomBoolean()));
         plugins.add(new PluginInfo("b", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
-        plugins.add(new PluginInfo("e", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
+        plugins.add(new PluginInfo( "e", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
         plugins.add(new PluginInfo("a", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
         plugins.add(new PluginInfo("d", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
         PluginsAndModules pluginsInfo = new PluginsAndModules(plugins, Collections.emptyList());
@@ -236,7 +236,7 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testUnknownProperties() throws Exception {
         Path pluginDir = createTempDir().resolve("fake-plugin");
-        PluginTestUtil.writeProperties(pluginDir,
+        PluginTestUtil.writePluginProperties(pluginDir,
             "extra", "property",
             "unknown", "property",
             "description", "fake desc",

core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java

@@ -180,7 +180,7 @@ public class PluginsServiceTests extends ESTestCase {
         Files.createFile(fake.resolve("plugin.jar"));
         final Path removing = home.resolve("plugins").resolve(".removing-fake");
         Files.createFile(removing);
-        PluginTestUtil.writeProperties(
+        PluginTestUtil.writePluginProperties(
                 fake,
                 "description", "fake",
                 "name", "fake",
@@ -541,7 +541,7 @@ public class PluginsServiceTests extends ESTestCase {
         Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), homeDir).build();
         Path pluginsDir = homeDir.resolve("plugins");
         Path mypluginDir = pluginsDir.resolve("myplugin");
-        PluginTestUtil.writeProperties(
+        PluginTestUtil.writePluginProperties(
             mypluginDir,
             "description", "whatever",
             "name", "myplugin",
@@ -554,7 +554,7 @@ public class PluginsServiceTests extends ESTestCase {
             Files.copy(jar, mypluginDir.resolve("plugin.jar"));
         }
         Path nonextensibleDir = pluginsDir.resolve("nonextensible");
-        PluginTestUtil.writeProperties(
+        PluginTestUtil.writePluginProperties(
             nonextensibleDir,
             "description", "whatever",
             "name", "nonextensible",

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -30,6 +30,7 @@ import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.hash.MessageDigests;
@@ -86,8 +87,8 @@ import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
  * <li>A URL to a plugin zip</li>
  * </ul>
  *
- * Plugins are packaged as zip files. Each packaged plugin must contain a
+ * Plugins are packaged as zip files. Each packaged plugin must contain a plugin properties file
- * plugin properties file. See {@link PluginInfo}.
+ * or a meta plugin properties file. See {@link PluginInfo} and {@link MetaPluginInfo}, respectively.
  * <p>
  * The installation process first extracts the plugin files into a temporary
  * directory in order to verify the plugin satisfies the following requirements:
@@ -105,6 +106,11 @@ import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
  * files specific to the plugin. The config files be installed into a subdirectory of the
  * elasticsearch config directory, using the name of the plugin. If any files to be installed
  * already exist, they will be skipped.
+ * <p>
+ * If the plugin is a meta plugin, the installation process installs each plugin separately
+ * inside the meta plugin directory. The {@code bin} and {@code config} directory are also moved
+ * inside the meta plugin directory.
+ * </p>
  */
 class InstallPluginCommand extends EnvironmentAwareCommand {
 
@@ -527,22 +533,44 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         return Files.createTempDirectory(pluginsDir, ".installing-");
     }
 
-    /** Load information about the plugin, and verify it can be installed with no errors. */
+    // checking for existing version of the plugin
-    private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception {
+    private void verifyPluginName(Path pluginPath, String pluginName, Path candidateDir) throws UserException, IOException {
-        // read and validate the plugin descriptor
+        final Path destination = pluginPath.resolve(pluginName);
-        PluginInfo info = PluginInfo.readFromProperties(pluginRoot);
-
-        // checking for existing version of the plugin
-        final Path destination = env.pluginsFile().resolve(info.getName());
         if (Files.exists(destination)) {
             final String message = String.format(
                 Locale.ROOT,
                 "plugin directory [%s] already exists; if you need to update the plugin, " +
                     "uninstall it first using command 'remove %s'",
                 destination.toAbsolutePath(),
-                info.getName());
+                pluginName);
             throw new UserException(PLUGIN_EXISTS, message);
         }
+        // checks meta plugins too
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginPath)) {
+            for (Path plugin : stream) {
+                if (candidateDir.equals(plugin.resolve(pluginName))) {
+                    continue;
+                }
+                if (MetaPluginInfo.isMetaPlugin(plugin) && Files.exists(plugin.resolve(pluginName))) {
+                    final MetaPluginInfo info = MetaPluginInfo.readFromProperties(plugin);
+                    final String message = String.format(
+                        Locale.ROOT,
+                        "plugin name [%s] already exists in a meta plugin; if you need to update the meta plugin, " +
+                            "uninstall it first using command 'remove %s'",
+                        plugin.resolve(pluginName).toAbsolutePath(),
+                        info.getName());
+                    throw new UserException(PLUGIN_EXISTS, message);
+                }
+            }
+        }
+    }
+
+    /** Load information about the plugin, and verify it can be installed with no errors. */
+    private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception {
+        final PluginInfo info = PluginInfo.readFromProperties(pluginRoot);
+
+        // checking for existing version of the plugin
+        verifyPluginName(env.pluginsFile(), info.getName(), pluginRoot);
 
         PluginsService.checkForFailedPluginRemovals(env.pluginsFile());
 
@@ -569,14 +597,15 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
     }
 
     /** check a candidate plugin for jar hell before installing it */
-    void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
+    void jarHellCheck(PluginInfo candidateInfo, Path candidateDir, Path pluginsDir, Path modulesDir) throws Exception {
         // create list of current jars in classpath
         final Set<URL> jars = new HashSet<>(JarHell.parseClassPath());
 
+
         // read existing bundles. this does some checks on the installation too.
         Set<PluginsService.Bundle> bundles = new HashSet<>(PluginsService.getPluginBundles(pluginsDir));
         bundles.addAll(PluginsService.getModuleBundles(modulesDir));
-        bundles.add(new PluginsService.Bundle(info, candidate));
+        bundles.add(new PluginsService.Bundle(candidateInfo, candidateDir));
         List<PluginsService.Bundle> sortedBundles = PluginsService.sortBundles(bundles);
 
         // check jarhell of all plugins so we know this plugin and anything depending on it are ok together
@@ -590,62 +619,15 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         // TODO: verify the classname exists in one of the jars!
     }
 
-    /**
-     * Installs the plugin from {@code tmpRoot} into the plugins dir.
-     * If the plugin has a bin dir and/or a config dir, those are copied.
-     */
     private void install(Terminal terminal, boolean isBatch, Path tmpRoot, Environment env) throws Exception {
         List<Path> deleteOnFailure = new ArrayList<>();
         deleteOnFailure.add(tmpRoot);
-
         try {
-            PluginInfo info = verify(terminal, tmpRoot, isBatch, env);
+            if (MetaPluginInfo.isMetaPlugin(tmpRoot)) {
-            final Path destination = env.pluginsFile().resolve(info.getName());
+                installMetaPlugin(terminal, isBatch, tmpRoot, env, deleteOnFailure);
-
+            } else {
-            Path tmpBinDir = tmpRoot.resolve("bin");
+                installPlugin(terminal, isBatch, tmpRoot, env, deleteOnFailure);
-            if (Files.exists(tmpBinDir)) {
-                Path destBinDir = env.binFile().resolve(info.getName());
-                deleteOnFailure.add(destBinDir);
-                installBin(info, tmpBinDir, destBinDir);
             }
-
-            Path tmpConfigDir = tmpRoot.resolve("config");
-            if (Files.exists(tmpConfigDir)) {
-                // some files may already exist, and we don't remove plugin config files on plugin removal,
-                // so any installed config files are left on failure too
-                installConfig(info, tmpConfigDir, env.configFile().resolve(info.getName()));
-            }
-
-            Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE);
-            Files.walkFileTree(destination, new SimpleFileVisitor<Path>() {
-                @Override
-                public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
-                    if ("bin".equals(file.getParent().getFileName().toString())) {
-                        setFileAttributes(file, BIN_FILES_PERMS);
-                    } else {
-                        setFileAttributes(file, PLUGIN_FILES_PERMS);
-                    }
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
-                    setFileAttributes(dir, PLUGIN_DIR_PERMS);
-                    return FileVisitResult.CONTINUE;
-                }
-            });
-
-            if (info.requiresKeystore()) {
-                KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
-                if (keystore == null) {
-                    terminal.println("Elasticsearch keystore is required by plugin [" + info.getName() + "], creating...");
-                    keystore = KeyStoreWrapper.create(new char[0]);
-                    keystore.save(env.configFile());
-                }
-            }
-
-            terminal.println("-> Installed " + info.getName());
-
         } catch (Exception installProblem) {
             try {
                 IOUtils.rm(deleteOnFailure.toArray(new Path[0]));
@@ -656,12 +638,119 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         }
     }
 
+    /**
+     * Installs the meta plugin and all the bundled plugins from {@code tmpRoot} into the plugins dir.
+     * If a bundled plugin has a bin dir and/or a config dir, those are copied.
+     */
+    private void installMetaPlugin(Terminal terminal, boolean isBatch, Path tmpRoot,
+                                   Environment env, List<Path> deleteOnFailure) throws Exception {
+        final MetaPluginInfo metaInfo = MetaPluginInfo.readFromProperties(tmpRoot);
+        verifyPluginName(env.pluginsFile(), metaInfo.getName(), tmpRoot);
+        final Path destination = env.pluginsFile().resolve(metaInfo.getName());
+        deleteOnFailure.add(destination);
+        terminal.println(VERBOSE, metaInfo.toString());
+        final List<Path> pluginPaths = new ArrayList<>();
+        try (DirectoryStream<Path> paths = Files.newDirectoryStream(tmpRoot)) {
+            // Extract bundled plugins path and validate plugin names
+            for (Path plugin : paths) {
+                if (MetaPluginInfo.isPropertiesFile(plugin)) {
+                    continue;
+                }
+                final PluginInfo info = PluginInfo.readFromProperties(plugin);
+                verifyPluginName(env.pluginsFile(), info.getName(), plugin);
+                pluginPaths.add(plugin);
+            }
+        }
+        final List<PluginInfo> pluginInfos = new ArrayList<>();
+        for (Path plugin : pluginPaths) {
+            final PluginInfo info = verify(terminal, plugin, isBatch, env);
+            pluginInfos.add(info);
+            Path tmpBinDir = plugin.resolve("bin");
+            if (Files.exists(tmpBinDir)) {
+                Path destBinDir = env.binFile().resolve(metaInfo.getName()).resolve(info.getName());
+                deleteOnFailure.add(destBinDir);
+                installBin(info, tmpBinDir, destBinDir);
+            }
+
+            Path tmpConfigDir = plugin.resolve("config");
+            if (Files.exists(tmpConfigDir)) {
+                // some files may already exist, and we don't remove plugin config files on plugin removal,
+                // so any installed config files are left on failure too
+                Path destConfigDir = env.configFile().resolve(metaInfo.getName()).resolve(info.getName());
+                installConfig(info, tmpConfigDir, destConfigDir);
+            }
+        }
+        movePlugin(tmpRoot, destination);
+        for (PluginInfo info : pluginInfos) {
+            if (info.requiresKeystore()) {
+                createKeystoreIfNeeded(terminal, env, info);
+                break;
+            }
+        }
+        String[] plugins = pluginInfos.stream().map(PluginInfo::getName).toArray(String[]::new);
+        terminal.println("-> Installed " + metaInfo.getName() + " with: " + Strings.arrayToCommaDelimitedString(plugins));
+    }
+
+    /**
+     * Installs the plugin from {@code tmpRoot} into the plugins dir.
+     * If the plugin has a bin dir and/or a config dir, those are copied.
+     */
+    private void installPlugin(Terminal terminal, boolean isBatch, Path tmpRoot,
+                               Environment env, List<Path> deleteOnFailure) throws Exception {
+        final PluginInfo info = verify(terminal, tmpRoot, isBatch, env);
+        final Path destination = env.pluginsFile().resolve(info.getName());
+        deleteOnFailure.add(destination);
+
+        Path tmpBinDir = tmpRoot.resolve("bin");
+        if (Files.exists(tmpBinDir)) {
+            Path destBinDir = env.binFile().resolve(info.getName());
+            deleteOnFailure.add(destBinDir);
+            installBin(info, tmpBinDir, destBinDir);
+        }
+
+        Path tmpConfigDir = tmpRoot.resolve("config");
+        if (Files.exists(tmpConfigDir)) {
+            // some files may already exist, and we don't remove plugin config files on plugin removal,
+            // so any installed config files are left on failure too
+            Path destConfigDir = env.configFile().resolve(info.getName());
+            installConfig(info, tmpConfigDir, destConfigDir);
+        }
+        movePlugin(tmpRoot, destination);
+        if (info.requiresKeystore()) {
+            createKeystoreIfNeeded(terminal, env, info);
+        }
+        terminal.println("-> Installed " + info.getName());
+    }
+
+    /** Moves the plugin directory into its final destination. **/
+    private void movePlugin(Path tmpRoot, Path destination) throws IOException {
+        Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE);
+        Files.walkFileTree(destination, new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
+                if ("bin".equals(file.getParent().getFileName().toString())) {
+                    setFileAttributes(file, BIN_FILES_PERMS);
+                } else {
+                    setFileAttributes(file, PLUGIN_FILES_PERMS);
+                }
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
+                setFileAttributes(dir, PLUGIN_DIR_PERMS);
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+
     /** Copies the files from {@code tmpBinDir} into {@code destBinDir}, along with permissions from dest dirs parent. */
     private void installBin(PluginInfo info, Path tmpBinDir, Path destBinDir) throws Exception {
         if (Files.isDirectory(tmpBinDir) == false) {
             throw new UserException(PLUGIN_MALFORMED, "bin in plugin " + info.getName() + " is not a directory");
         }
-        Files.createDirectory(destBinDir);
+        Files.createDirectories(destBinDir);
         setFileAttributes(destBinDir, BIN_DIR_PERMS);
 
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpBinDir)) {
@@ -719,6 +808,15 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         IOUtils.rm(tmpConfigDir); // clean up what we just copied
     }
 
+    private void createKeystoreIfNeeded(Terminal terminal, Environment env, PluginInfo info) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            terminal.println("Elasticsearch keystore is required by plugin [" + info.getName() + "], creating...");
+            keystore = KeyStoreWrapper.create(new char[0]);
+            keystore.save(env.configFile());
+        }
+    }
+
     private static void setOwnerGroup(final Path path, final PosixFileAttributes attributes) throws IOException {
         Objects.requireNonNull(attributes);
         PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class);

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java

@@ -22,6 +22,7 @@ package org.elasticsearch.plugins;
 import joptsimple.OptionSet;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.env.Environment;
 
 import java.io.IOException;
@@ -29,8 +30,11 @@ import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * A command for the plugin cli to list plugins installed in elasticsearch.
@@ -56,16 +60,38 @@ class ListPluginsCommand extends EnvironmentAwareCommand {
         }
         Collections.sort(plugins);
         for (final Path plugin : plugins) {
-            terminal.println(Terminal.Verbosity.SILENT, plugin.getFileName().toString());
+            if (MetaPluginInfo.isMetaPlugin(plugin)) {
-            try {
+                MetaPluginInfo metaInfo = MetaPluginInfo.readFromProperties(plugin);
-                PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath()));
+                List<Path> subPluginPaths = new ArrayList<>();
-                terminal.println(Terminal.Verbosity.VERBOSE, info.toString());
+                try (DirectoryStream<Path> subPaths = Files.newDirectoryStream(plugin)) {
-            } catch (IllegalArgumentException e) {
+                    for (Path subPlugin : subPaths) {
-                if (e.getMessage().contains("incompatible with version")) {
+                        if (MetaPluginInfo.isPropertiesFile(subPlugin)) {
-                    terminal.println("WARNING: " + e.getMessage());
+                            continue;
-                } else {
+                        }
-                    throw e;
+                        subPluginPaths.add(subPlugin);
+                    }
                 }
+                Collections.sort(subPluginPaths);
+                terminal.println(Terminal.Verbosity.SILENT, metaInfo.getName());
+                for (Path subPlugin : subPluginPaths) {
+                    printPlugin(env, terminal, subPlugin, "\t");
+                }
+            } else {
+                printPlugin(env, terminal, plugin, "");
+            }
+        }
+    }
+
+    private void printPlugin(Environment env, Terminal terminal, Path plugin, String prefix) throws IOException {
+        terminal.println(Terminal.Verbosity.SILENT, prefix + plugin.getFileName().toString());
+        try {
+            PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath()));
+            terminal.println(Terminal.Verbosity.VERBOSE, info.toString(prefix));
+        } catch (IllegalArgumentException e) {
+            if (e.getMessage().contains("incompatible with version")) {
+                terminal.println("WARNING: " + e.getMessage());
+            } else {
+                throw e;
             }
         }
     }

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -115,7 +115,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         super.setUp();
         skipJarHellCommand = new InstallPluginCommand() {
             @Override
-            void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
+            void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
                 // no jarhell check
             }
         };
@@ -214,7 +214,19 @@ public class InstallPluginCommandTests extends ESTestCase {
         return createPlugin(name, structure, false, additionalProps).toUri().toURL().toString();
     }
 
-    static Path createPlugin(String name, Path structure, boolean createSecurityPolicyFile, String... additionalProps) throws IOException {
+    /** creates an meta plugin .zip and returns the url for testing */
+    static String createMetaPluginUrl(String name, Path structure) throws IOException {
+        return createMetaPlugin(name, structure).toUri().toURL().toString();
+    }
+
+    static void writeMetaPlugin(String name, Path structure) throws IOException {
+        PluginTestUtil.writeMetaPluginProperties(structure,
+            "description", "fake desc",
+            "name", name
+        );
+    }
+
+    static void writePlugin(String name, Path structure, boolean createSecurityPolicyFile, String... additionalProps) throws IOException {
         String[] properties = Stream.concat(Stream.of(
             "description", "fake desc",
             "name", name,
@@ -223,12 +235,22 @@ public class InstallPluginCommandTests extends ESTestCase {
             "java.version", System.getProperty("java.specification.version"),
             "classname", "FakePlugin"
         ), Arrays.stream(additionalProps)).toArray(String[]::new);
-        PluginTestUtil.writeProperties(structure, properties);
+        PluginTestUtil.writePluginProperties(structure, properties);
         if (createSecurityPolicyFile) {
             String securityPolicyContent = "grant {\n  permission java.lang.RuntimePermission \"setFactory\";\n};\n";
             Files.write(structure.resolve("plugin-security.policy"), securityPolicyContent.getBytes(StandardCharsets.UTF_8));
         }
-        writeJar(structure.resolve("plugin.jar"), "FakePlugin");
+        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
+        writeJar(structure.resolve("plugin.jar"), className);
+    }
+
+    static Path createPlugin(String name, Path structure, boolean createSecurityPolicyFile, String... additionalProps) throws IOException {
+        writePlugin(name, structure, createSecurityPolicyFile, additionalProps);
+        return writeZip(structure, "elasticsearch");
+    }
+
+    static Path createMetaPlugin(String name, Path structure) throws IOException {
+        writeMetaPlugin(name, structure);
         return writeZip(structure, "elasticsearch");
     }
 
@@ -243,8 +265,20 @@ public class InstallPluginCommandTests extends ESTestCase {
         return terminal;
     }
 
+    void assertMetaPlugin(String metaPlugin, String name, Path original, Environment env) throws IOException {
+        assertPluginInternal(name, original, env,
+            env.pluginsFile().resolve(metaPlugin), env.configFile().resolve(metaPlugin), env.binFile().resolve(metaPlugin));
+    }
+
+
     void assertPlugin(String name, Path original, Environment env) throws IOException {
-        Path got = env.pluginsFile().resolve(name);
+        assertPluginInternal(name, original, env,
+            env.pluginsFile(), env.configFile(), env.binFile());
+    }
+
+    void assertPluginInternal(String name, Path original, Environment env,
+                              Path pluginsFile, Path configFile, Path binFile) throws IOException {
+        Path got = pluginsFile.resolve(name);
         assertTrue("dir " + name + " exists", Files.exists(got));
 
         if (isPosix) {
@@ -265,12 +299,12 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertFalse("bin was not copied", Files.exists(got.resolve("bin")));
         assertFalse("config was not copied", Files.exists(got.resolve("config")));
         if (Files.exists(original.resolve("bin"))) {
-            Path binDir = env.binFile().resolve(name);
+            Path binDir = binFile.resolve(name);
             assertTrue("bin dir exists", Files.exists(binDir));
             assertTrue("bin is a dir", Files.isDirectory(binDir));
             PosixFileAttributes binAttributes = null;
             if (isPosix) {
-                binAttributes = Files.readAttributes(env.binFile(), PosixFileAttributes.class);
+                binAttributes = Files.readAttributes(binFile, PosixFileAttributes.class);
             }
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(binDir)) {
                 for (Path file : stream) {
@@ -283,7 +317,7 @@ public class InstallPluginCommandTests extends ESTestCase {
             }
         }
         if (Files.exists(original.resolve("config"))) {
-            Path configDir = env.configFile().resolve(name);
+            Path configDir = configFile.resolve(name);
             assertTrue("config dir exists", Files.exists(configDir));
             assertTrue("config is a dir", Files.isDirectory(configDir));
 
@@ -292,7 +326,7 @@ public class InstallPluginCommandTests extends ESTestCase {
 
             if (isPosix) {
                 PosixFileAttributes configAttributes =
-                        Files.getFileAttributeView(env.configFile(), PosixFileAttributeView.class).readAttributes();
+                        Files.getFileAttributeView(configFile, PosixFileAttributeView.class).readAttributes();
                 user = configAttributes.owner();
                 group = configAttributes.group();
 
@@ -344,9 +378,23 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertPlugin("fake", pluginDir, env.v2());
     }
 
-    public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception {
+    public void testWithMetaPlugin() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
+        Files.createDirectory(pluginDir.resolve("fake1"));
+        writePlugin("fake1", pluginDir.resolve("fake1"), false);
+        Files.createDirectory(pluginDir.resolve("fake2"));
+        writePlugin("fake2", pluginDir.resolve("fake2"), false);
+        String pluginZip = createMetaPluginUrl("my_plugins", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        assertMetaPlugin("my_plugins", "fake1", pluginDir, env.v2());
+        assertMetaPlugin("my_plugins", "fake2", pluginDir, env.v2());
+    }
+
+    public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
         String pluginZip = createPluginUrl("fake", pluginDir);
         final Path removing = env.v2().pluginsFile().resolve(".removing-failed");
         Files.createDirectory(removing);
@@ -356,6 +404,11 @@ public class InstallPluginCommandTests extends ESTestCase {
                 "found file [%s] from a failed attempt to remove the plugin [failed]; execute [elasticsearch-plugin remove failed]",
                 removing);
         assertThat(e, hasToString(containsString(expected)));
+
+        // test with meta plugin
+        String metaZip = createMetaPluginUrl("my_plugins", metaDir);
+        final IllegalStateException e1 = expectThrows(IllegalStateException.class, () -> installPlugin(metaZip, env.v1()));
+        assertThat(e1, hasToString(containsString(expected)));
     }
 
     public void testSpaceInUrl() throws Exception {
@@ -418,6 +471,23 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertInstallCleaned(environment.v2());
     }
 
+    public void testJarHellInMetaPlugin() throws Exception {
+        // jar hell test needs a real filesystem
+        assumeTrue("real filesystem", isReal);
+        Tuple<Path, Environment> environment = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Files.createDirectory(pluginDir.resolve("fake1"));
+        writePlugin("fake1", pluginDir.resolve("fake1"), false);
+        Files.createDirectory(pluginDir.resolve("fake2"));
+        writePlugin("fake2", pluginDir.resolve("fake2"), false); // adds plugin.jar with Fake2Plugin
+        writeJar(pluginDir.resolve("fake2").resolve("other.jar"), "Fake2Plugin");
+        String pluginZip = createMetaPluginUrl("my_plugins", pluginDir);
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+            () -> installPlugin(pluginZip, environment.v1(), defaultCommand));
+        assertTrue(e.getMessage(), e.getMessage().contains("jar hell"));
+        assertInstallCleaned(environment.v2());
+    }
+
     public void testIsolatedPlugins() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         // these both share the same FakePlugin class
@@ -441,6 +511,23 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertInstallCleaned(env.v2());
     }
 
+    public void testExistingMetaPlugin() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaZip = createPluginDir(temp);
+        Path pluginDir = metaZip.resolve("fake");
+        Files.createDirectory(pluginDir);
+        String pluginZip = createPluginUrl("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("already exists"));
+        assertInstallCleaned(env.v2());
+
+        String anotherZip = createMetaPluginUrl("another_plugins", metaZip);
+        e = expectThrows(UserException.class, () -> installPlugin(anotherZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("already exists"));
+        assertInstallCleaned(env.v2());
+    }
+
     public void testBin() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
@@ -452,20 +539,43 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertPlugin("fake", pluginDir, env.v2());
     }
 
+    public void testMetaBin() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
+        Files.createDirectory(pluginDir);
+        writePlugin("fake", pluginDir, false);
+        Path binDir = pluginDir.resolve("bin");
+        Files.createDirectory(binDir);
+        Files.createFile(binDir.resolve("somescript"));
+        String pluginZip = createMetaPluginUrl("my_plugins", metaDir);
+        installPlugin(pluginZip, env.v1());
+        assertMetaPlugin("my_plugins","fake", pluginDir, env.v2());
+    }
+
     public void testBinNotDir() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
+        Files.createDirectory(pluginDir);
         Path binDir = pluginDir.resolve("bin");
         Files.createFile(binDir);
         String pluginZip = createPluginUrl("fake", pluginDir);
         UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
         assertInstallCleaned(env.v2());
+
+        String metaZip = createMetaPluginUrl("my_plugins", metaDir);
+        e = expectThrows(UserException.class, () -> installPlugin(metaZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
+        assertInstallCleaned(env.v2());
     }
 
     public void testBinContainsDir() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
+        Files.createDirectory(pluginDir);
         Path dirInBinDir = pluginDir.resolve("bin").resolve("foo");
         Files.createDirectories(dirInBinDir);
         Files.createFile(dirInBinDir.resolve("somescript"));
@@ -473,11 +583,16 @@ public class InstallPluginCommandTests extends ESTestCase {
         UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin"));
         assertInstallCleaned(env.v2());
+
+        String metaZip = createMetaPluginUrl("my_plugins", metaDir);
+        e = expectThrows(UserException.class, () -> installPlugin(metaZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin"));
+        assertInstallCleaned(env.v2());
     }
 
     public void testBinConflict() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
+        Path pluginDir =  createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createDirectory(binDir);
         Files.createFile(binDir.resolve("somescript"));
@@ -505,6 +620,27 @@ public class InstallPluginCommandTests extends ESTestCase {
         }
     }
 
+    public void testMetaBinPermissions() throws Exception {
+        assumeTrue("posix filesystem", isPosix);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
+        Files.createDirectory(pluginDir);
+        writePlugin("fake", pluginDir, false);
+        Path binDir = pluginDir.resolve("bin");
+        Files.createDirectory(binDir);
+        Files.createFile(binDir.resolve("somescript"));
+        String pluginZip = createMetaPluginUrl("my_plugins", metaDir);
+        try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binFile())) {
+            Set<PosixFilePermission> perms = binAttrs.getCopyPermissions();
+            // make sure at least one execute perm is missing, so we know we forced it during installation
+            perms.remove(PosixFilePermission.GROUP_EXECUTE);
+            binAttrs.setPermissions(perms);
+            installPlugin(pluginZip, env.v1());
+            assertMetaPlugin("my_plugins", "fake", pluginDir, env.v2());
+        }
+    }
+
     public void testPluginPermissions() throws Exception {
         assumeTrue("posix filesystem", isPosix);
 
@@ -596,15 +732,44 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertTrue(Files.exists(envConfigDir.resolve("other.yml")));
     }
 
+    public void testExistingMetaConfig() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path envConfigDir = env.v2().configFile().resolve("my_plugins").resolve("fake");
+        Files.createDirectories(envConfigDir);
+        Files.write(envConfigDir.resolve("custom.yml"), "existing config".getBytes(StandardCharsets.UTF_8));
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
+        Files.createDirectory(pluginDir);
+        writePlugin("fake", pluginDir, false);
+        Path configDir = pluginDir.resolve("config");
+        Files.createDirectory(configDir);
+        Files.write(configDir.resolve("custom.yml"), "new config".getBytes(StandardCharsets.UTF_8));
+        Files.createFile(configDir.resolve("other.yml"));
+        String pluginZip = createMetaPluginUrl("my_plugins", metaDir);
+        installPlugin(pluginZip, env.v1());
+        assertMetaPlugin("my_plugins", "fake", pluginDir, env.v2());
+        List<String> configLines = Files.readAllLines(envConfigDir.resolve("custom.yml"), StandardCharsets.UTF_8);
+        assertEquals(1, configLines.size());
+        assertEquals("existing config", configLines.get(0));
+        assertTrue(Files.exists(envConfigDir.resolve("other.yml")));
+    }
+
     public void testConfigNotDir() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
+        Files.createDirectories(pluginDir);
         Path configDir = pluginDir.resolve("config");
         Files.createFile(configDir);
         String pluginZip = createPluginUrl("fake", pluginDir);
         UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
         assertInstallCleaned(env.v2());
+
+        String metaZip = createMetaPluginUrl("my_plugins", metaDir);
+        e = expectThrows(UserException.class, () -> installPlugin(metaZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
+        assertInstallCleaned(env.v2());
     }
 
     public void testConfigContainsDir() throws Exception {
@@ -619,26 +784,21 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertInstallCleaned(env.v2());
     }
 
-    public void testConfigConflict() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        Path configDir = pluginDir.resolve("config");
-        Files.createDirectory(configDir);
-        Files.createFile(configDir.resolve("myconfig.yml"));
-        String pluginZip = createPluginUrl("elasticsearch.yml", pluginDir);
-        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains(env.v2().configFile().resolve("elasticsearch.yml").toString()));
-        assertInstallCleaned(env.v2());
-    }
-
     public void testMissingDescriptor() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
+        Files.createDirectory(pluginDir);
         Files.createFile(pluginDir.resolve("fake.yml"));
         String pluginZip = writeZip(pluginDir, "elasticsearch").toUri().toURL().toString();
         NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties"));
         assertInstallCleaned(env.v2());
+
+        String metaZip = createMetaPluginUrl("my_plugins", metaDir);
+        e = expectThrows(NoSuchFileException.class, () -> installPlugin(metaZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties"));
+        assertInstallCleaned(env.v2());
     }
 
     public void testMissingDirectory() throws Exception {
@@ -651,6 +811,16 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertInstallCleaned(env.v2());
     }
 
+    public void testMissingDirectoryMeta() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        Files.createFile(pluginDir.resolve(MetaPluginInfo.ES_META_PLUGIN_PROPERTIES));
+        String pluginZip = writeZip(pluginDir, null).toUri().toURL().toString();
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("`elasticsearch` directory is missing in the plugin zip"));
+        assertInstallCleaned(env.v2());
+    }
+
     public void testZipRelativeOutsideEntryName() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path zip = createTempDir().resolve("broken.zip");
@@ -748,6 +918,29 @@ public class InstallPluginCommandTests extends ESTestCase {
                 "if you need to update the plugin, uninstall it first using command 'remove fake'"));
     }
 
+    public void testMetaPluginAlreadyInstalled() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        {
+            // install fake plugin
+            Path pluginDir = createPluginDir(temp);
+            String pluginZip = createPluginUrl("fake", pluginDir);
+            installPlugin(pluginZip, env.v1());
+        }
+
+        Path pluginDir = createPluginDir(temp);
+        Files.createDirectory(pluginDir.resolve("fake"));
+        writePlugin("fake", pluginDir.resolve("fake"), false);
+        Files.createDirectory(pluginDir.resolve("other"));
+        writePlugin("other", pluginDir.resolve("other"), false);
+        String metaZip = createMetaPluginUrl("meta", pluginDir);
+        final UserException e = expectThrows(UserException.class,
+            () -> installPlugin(metaZip, env.v1(), randomFrom(skipJarHellCommand, defaultCommand)));
+        assertThat(
+            e.getMessage(),
+            equalTo("plugin directory [" + env.v2().pluginsFile().resolve("fake") + "] already exists; " +
+                "if you need to update the plugin, uninstall it first using command 'remove fake'"));
+    }
+
     private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
@@ -791,7 +984,7 @@ public class InstallPluginCommandTests extends ESTestCase {
                 return stagingHash;
             }
             @Override
-            void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
+            void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
                 // no jarhell check
             }
         };
@@ -951,6 +1144,17 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertTrue(Files.exists(KeyStoreWrapper.keystorePath(env.v2().configFile())));
     }
 
+    public void testKeystoreRequiredCreatedWithMetaPlugin() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaDir = createPluginDir(temp);
+        Path pluginDir = metaDir.resolve("fake");
+        Files.createDirectory(pluginDir);
+        writePlugin("fake", pluginDir, false, "requires.keystore", "true");
+        String metaZip = createMetaPluginUrl("my_plugins", metaDir);
+        MockTerminal terminal = installPlugin(metaZip, env.v1());
+        assertTrue(Files.exists(KeyStoreWrapper.keystorePath(env.v2().configFile())));
+    }
+
     private Function<byte[], String> checksum(final MessageDigest digest) {
         return checksumAndString(digest, "");
     }

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java

@@ -33,7 +33,6 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.MockTerminal;
-import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -94,18 +93,39 @@ public class ListPluginsCommandTests extends ESTestCase {
             final String description,
             final String name,
             final String classname) throws IOException {
-        buildFakePlugin(env, description, name, classname, false, false);
+        buildFakePlugin(env, null, description, name, classname, false, false);
     }
 
     private static void buildFakePlugin(
             final Environment env,
+            final String metaPlugin,
+            final String description,
+            final String name,
+            final String classname) throws IOException {
+        buildFakePlugin(env, metaPlugin, description, name, classname, false, false);
+    }
+
+    private static void buildFakePlugin(
+        final Environment env,
+        final String description,
+        final String name,
+        final String classname,
+        final boolean hasNativeController,
+        final boolean requiresKeystore) throws IOException {
+        buildFakePlugin(env, null, description, name, classname, hasNativeController, requiresKeystore);
+    }
+
+    private static void buildFakePlugin(
+            final Environment env,
+            final String metaPlugin,
             final String description,
             final String name,
             final String classname,
             final boolean hasNativeController,
             final boolean requiresKeystore) throws IOException {
-        PluginTestUtil.writeProperties(
+        Path dest = metaPlugin != null ? env.pluginsFile().resolve(metaPlugin) : env.pluginsFile();
-                env.pluginsFile().resolve(name),
+        PluginTestUtil.writePluginProperties(
+                dest.resolve(name),
                 "description", description,
                 "name", name,
                 "version", "1.0",
@@ -116,6 +136,16 @@ public class ListPluginsCommandTests extends ESTestCase {
                 "requires.keystore", Boolean.toString(requiresKeystore));
     }
 
+    private static void buildFakeMetaPlugin(
+        final Environment env,
+        final String description,
+        final String name) throws IOException {
+        PluginTestUtil.writeMetaPluginProperties(
+            env.pluginsFile().resolve(name),
+            "description", description,
+            "name", name);
+    }
+
     public void testPluginsDirMissing() throws Exception {
         Files.delete(env.pluginsFile());
         IOException e = expectThrows(IOException.class, () -> listPlugins(home));
@@ -140,6 +170,16 @@ public class ListPluginsCommandTests extends ESTestCase {
         assertEquals(buildMultiline("fake1", "fake2"), terminal.getOutput());
     }
 
+    public void testMetaPlugin() throws Exception {
+        buildFakeMetaPlugin(env, "fake meta desc", "meta_plugin");
+        buildFakePlugin(env, "meta_plugin", "fake desc", "fake1", "org.fake1");
+        buildFakePlugin(env, "meta_plugin",  "fake desc 2", "fake2", "org.fake2");
+        buildFakePlugin(env, "fake desc 3", "fake3", "org.fake3");
+        buildFakePlugin(env, "fake desc 4", "fake4", "org.fake4");
+        MockTerminal terminal = listPlugins(home);
+        assertEquals(buildMultiline("fake3", "fake4", "meta_plugin", "\tfake1", "\tfake2"), terminal.getOutput());
+    }
+
     public void testPluginWithVerbose() throws Exception {
         buildFakePlugin(env, "fake desc", "fake_plugin", "org.fake");
         String[] params = { "-v" };
@@ -226,6 +266,37 @@ public class ListPluginsCommandTests extends ESTestCase {
                 terminal.getOutput());
     }
 
+    public void testPluginWithVerboseMetaPlugins() throws Exception {
+        buildFakeMetaPlugin(env, "fake meta desc", "meta_plugin");
+        buildFakePlugin(env, "meta_plugin", "fake desc 1", "fake_plugin1", "org.fake");
+        buildFakePlugin(env, "meta_plugin",  "fake desc 2", "fake_plugin2", "org.fake2");
+        String[] params = { "-v" };
+        MockTerminal terminal = listPlugins(home, params);
+        assertEquals(
+            buildMultiline(
+                "Plugins directory: " + env.pluginsFile(),
+                "meta_plugin",
+                "\tfake_plugin1",
+                "\t- Plugin information:",
+                "\tName: fake_plugin1",
+                "\tDescription: fake desc 1",
+                "\tVersion: 1.0",
+                "\tNative Controller: false",
+                "\tRequires Keystore: false",
+                "\tExtended Plugins: []",
+                "\t * Classname: org.fake",
+                "\tfake_plugin2",
+                "\t- Plugin information:",
+                "\tName: fake_plugin2",
+                "\tDescription: fake desc 2",
+                "\tVersion: 1.0",
+                "\tNative Controller: false",
+                "\tRequires Keystore: false",
+                "\tExtended Plugins: []",
+                "\t * Classname: org.fake2"),
+            terminal.getOutput());
+    }
+
     public void testPluginWithoutVerboseMultiplePlugins() throws Exception {
         buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
         buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
@@ -243,7 +314,7 @@ public class ListPluginsCommandTests extends ESTestCase {
 
     public void testPluginWithWrongDescriptorFile() throws Exception{
         final Path pluginDir = env.pluginsFile().resolve("fake1");
-        PluginTestUtil.writeProperties(pluginDir, "description", "fake desc");
+        PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc");
         IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
                 () -> listPlugins(home));
@@ -253,8 +324,21 @@ public class ListPluginsCommandTests extends ESTestCase {
                 e.getMessage());
     }
 
+    public void testMetaPluginWithWrongDescriptorFile() throws Exception{
+        buildFakeMetaPlugin(env, "fake meta desc", "meta_plugin");
+        final Path pluginDir = env.pluginsFile().resolve("meta_plugin").resolve("fake_plugin1");
+        PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc");
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> listPlugins(home));
+        final Path descriptorPath = pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES);
+        assertEquals(
+            "property [name] is missing in [" + descriptorPath.toString() + "]",
+            e.getMessage());
+    }
+
     public void testExistingIncompatiblePlugin() throws Exception {
-        PluginTestUtil.writeProperties(env.pluginsFile().resolve("fake_plugin1"),
+        PluginTestUtil.writePluginProperties(env.pluginsFile().resolve("fake_plugin1"),
             "description", "fake desc 1",
             "name", "fake_plugin1",
             "version", "1.0",
@@ -278,4 +362,30 @@ public class ListPluginsCommandTests extends ESTestCase {
         assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput());
     }
 
+    public void testExistingIncompatibleMetaPlugin() throws Exception {
+        buildFakeMetaPlugin(env, "fake meta desc", "meta_plugin");
+        PluginTestUtil.writePluginProperties(env.pluginsFile().resolve("meta_plugin").resolve("fake_plugin1"),
+            "description", "fake desc 1",
+            "name", "fake_plugin1",
+            "version", "1.0",
+            "elasticsearch.version", Version.fromString("1.0.0").toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "org.fake1");
+        buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
+
+        MockTerminal terminal = listPlugins(home);
+        final String message = String.format(Locale.ROOT,
+            "plugin [%s] is incompatible with version [%s]; was designed for version [%s]",
+            "fake_plugin1",
+            Version.CURRENT.toString(),
+            "1.0.0");
+        assertEquals(
+            "fake_plugin2\nmeta_plugin\n\tfake_plugin1\n" + "WARNING: " + message + "\n",
+            terminal.getOutput());
+
+        String[] params = {"-s"};
+        terminal = listPlugins(home, params);
+        assertEquals("fake_plugin2\nmeta_plugin\n\tfake_plugin1\n", terminal.getOutput());
+    }
+
 }

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java

@@ -79,8 +79,12 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     void createPlugin(String name) throws Exception {
-        PluginTestUtil.writeProperties(
+        createPlugin(env.pluginsFile(), name);
-            env.pluginsFile().resolve(name),
+    }
+
+    void createPlugin(Path path, String name) throws Exception {
+        PluginTestUtil.writePluginProperties(
+            path.resolve(name),
             "description", "dummy",
             "name", name,
             "version", "1.0",
@@ -89,6 +93,16 @@ public class RemovePluginCommandTests extends ESTestCase {
             "classname", "SomeClass");
     }
 
+    void createMetaPlugin(String name, String... plugins) throws Exception {
+        PluginTestUtil.writeMetaPluginProperties(
+            env.pluginsFile().resolve(name),
+            "description", "dummy",
+            "name", name);
+        for (String plugin : plugins) {
+            createPlugin(env.pluginsFile().resolve(name), plugin);
+        }
+    }
+
     static MockTerminal removePlugin(String name, Path home, boolean purge) throws Exception {
         Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", home).build());
         MockTerminal terminal = new MockTerminal();
@@ -123,6 +137,19 @@ public class RemovePluginCommandTests extends ESTestCase {
         assertRemoveCleaned(env);
     }
 
+    public void testBasicMeta() throws Exception {
+        createMetaPlugin("meta", "fake1");
+        createPlugin("other");
+        removePlugin("meta", home, randomBoolean());
+        assertFalse(Files.exists(env.pluginsFile().resolve("meta")));
+        assertTrue(Files.exists(env.pluginsFile().resolve("other")));
+        assertRemoveCleaned(env);
+
+        UserException exc =
+            expectThrows(UserException.class, () -> removePlugin("fake1", home, randomBoolean()));
+        assertThat(exc.getMessage(), containsString("plugin [fake1] not found"));
+    }
+
     public void testBin() throws Exception {
         createPlugin("fake");
         Path binDir = env.binFile().resolve("fake");

docs/plugins/authors.asciidoc

@@ -1,10 +1,18 @@
 [[plugin-authors]]
 == Help for plugin authors
 
+:plugin-properties-files: {docdir}/../../buildSrc/src/main/resources
+
 The Elasticsearch repository contains examples of:
 
 * a https://github.com/elastic/elasticsearch/tree/master/plugins/jvm-example[Java plugin]
   which contains Java code.
+* a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/rescore[Java plugin]
+  which contains a rescore plugin.
+* a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/script-expert-scoring[Java plugin]
+  which contains a script plugin.
+* a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/meta-plugin[Java plugin]
+  which contains a meta plugin.
 
 These examples provide the bare bones needed to get started.  For more
 information about how to write a plugin, we recommend looking at the plugins
@@ -18,10 +26,13 @@ All plugin files must be contained in a directory called `elasticsearch`.
 [float]
 === Plugin descriptor file
 
-All plugins must contain a file called `plugin-descriptor.properties` in the folder named `elasticsearch`. The format
+All plugins must contain a file called `plugin-descriptor.properties` in the folder named `elasticsearch`.
-for this file is described in detail here:
+The format for this file is described in detail in this example:
 
-https://github.com/elastic/elasticsearch/blob/master/buildSrc/src/main/resources/plugin-descriptor.properties[`/buildSrc/src/main/resources/plugin-descriptor.properties`].
+["source","properties",subs="attributes"]
+--------------------------------------------------
+include-tagged::{plugin-properties-files}/plugin-descriptor.properties[plugin-descriptor.properties]
+--------------------------------------------------
 
 Either fill in this template yourself or, if you are using Elasticsearch's Gradle build system, you
 can fill in the necessary values in the `build.gradle` file for your plugin.
@@ -112,3 +123,19 @@ AccessController.doPrivileged(
 
 See http://www.oracle.com/technetwork/java/seccodeguide-139067.html[Secure Coding Guidelines for Java SE]
 for more information.
+
+[float]
+=== Meta Plugin
+
+It is also possible to bundle multiple plugins into a meta plugin.
+A directory for each sub-plugin must be contained in a directory called `elasticsearch.
+The meta plugin must also contain a file called `meta-plugin-descriptor.properties` in the directory named
+`elasticsearch`.
+The format for this file is described in detail in this example:
+
+["source","properties",subs="attributes"]
+--------------------------------------------------
+include-tagged::{plugin-properties-files}/meta-plugin-descriptor.properties[meta-plugin-descriptor.properties]
+--------------------------------------------------
+
+A meta plugin can be installed/removed like a normal plugin with the `bin/elasticsearch-plugin` command.

plugins/examples/meta-plugin/build.gradle

@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A meta plugin packaging example that bundles multiple plugins in a single zip.
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+File plugins = new File(buildDir, 'plugins-unzip')
+subprojects {
+    // unzip the subproject plugins
+    task unzip(type:Copy, dependsOn: "${project.path}:bundlePlugin") {
+        File dest = new File(plugins, project.name)
+        from { zipTree(project(project.path).bundlePlugin.outputs.files.singleFile) }
+        eachFile { f -> f.path = f.path.replaceFirst('elasticsearch', '') }
+        into dest
+    }
+}
+
+// Build the meta plugin zip from the subproject plugins (unzipped)
+task buildZip(type:Zip) {
+    subprojects.each { dependsOn("${it.name}:unzip") }
+    from plugins
+    from 'src/main/resources/meta-plugin-descriptor.properties'
+    into 'elasticsearch'
+    includeEmptyDirs false
+}
+
+integTestCluster {
+    dependsOn buildZip
+
+    // This is important, so that all the modules are available too.
+    // There are index templates that use token filters that are in analysis-module and
+    // processors are being used that are in ingest-common module.
+    distribution = 'zip'
+
+    // Install the meta plugin before start.
+    setupCommand 'installMetaPlugin',
+            'bin/elasticsearch-plugin', 'install', 'file:' + buildZip.archivePath
+}
+check.dependsOn integTest

plugins/examples/meta-plugin/dummy-plugin1/build.gradle

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'dummy-plugin1'
+  description 'A dummy plugin'
+  classname 'org.elasticsearch.example.DummyPlugin1'
+}
+
+test.enabled = false
+integTestRunner.enabled = false

plugins/examples/meta-plugin/dummy-plugin1/src/main/java/org/elasticsearch/example/DummyPlugin1.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+
+public class DummyPlugin1 extends Plugin {}

plugins/examples/meta-plugin/dummy-plugin2/build.gradle

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'dummy-plugin2'
+  description 'Another dummy plugin'
+  classname 'org.elasticsearch.example.DummyPlugin2'
+}
+
+test.enabled = false
+integTestRunner.enabled = false

plugins/examples/meta-plugin/dummy-plugin2/src/main/java/org/elasticsearch/example/DummyPlugin2.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+
+public class DummyPlugin2 extends Plugin {}

plugins/examples/meta-plugin/src/main/resources/meta-plugin-descriptor.properties

@@ -0,0 +1,4 @@
+# The name of the meta plugin
+name=my_meta_plugin
+# The description of the meta plugin
+description=A meta plugin example

plugins/examples/meta-plugin/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.smoketest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class SmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public SmokeTestPluginsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}
+

plugins/examples/meta-plugin/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yml

@@ -0,0 +1,14 @@
+# Integration tests for testing meta plugins
+#
+"Check meta plugin install":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.plugins.0.name: dummy-plugin1  }
+    - match:  { nodes.$master.plugins.1.name: dummy-plugin2  }

qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java

@@ -78,7 +78,7 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
         // This plugin will NOT have a controller daemon
         Path plugin = environment.pluginsFile().resolve("a_plugin");
         Files.createDirectories(plugin);
-        PluginTestUtil.writeProperties(
+        PluginTestUtil.writePluginProperties(
                 plugin,
                 "description", "a_plugin",
                 "version", Version.CURRENT.toString(),
@@ -114,7 +114,7 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
         // this plugin will have a controller daemon
         Path plugin = environment.pluginsFile().resolve("test_plugin");
         Files.createDirectories(plugin);
-        PluginTestUtil.writeProperties(
+        PluginTestUtil.writePluginProperties(
                 plugin,
                 "description", "test_plugin",
                 "version", Version.CURRENT.toString(),
@@ -129,7 +129,7 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
         // this plugin will not have a controller daemon
         Path otherPlugin = environment.pluginsFile().resolve("other_plugin");
         Files.createDirectories(otherPlugin);
-        PluginTestUtil.writeProperties(
+        PluginTestUtil.writePluginProperties(
                 otherPlugin,
                 "description", "other_plugin",
                 "version", Version.CURRENT.toString(),
@@ -163,6 +163,84 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
         }
     }
 
+    /**
+     * Two plugins in a meta plugin - one with a controller daemon and one without.
+     */
+    public void testControllerSpawnMetaPlugin() throws IOException, InterruptedException {
+        /*
+         * On Windows you can not directly run a batch file - you have to run cmd.exe with the batch
+         * file as an argument and that's out of the remit of the controller daemon process spawner.
+         */
+        assumeFalse("This test does not work on Windows", Constants.WINDOWS);
+
+        Path esHome = createTempDir().resolve("esHome");
+        Settings.Builder settingsBuilder = Settings.builder();
+        settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.toString());
+        Settings settings = settingsBuilder.build();
+
+        Environment environment = TestEnvironment.newEnvironment(settings);
+
+        Path metaPlugin = environment.pluginsFile().resolve("meta_plugin");
+        Files.createDirectories(metaPlugin);
+        PluginTestUtil.writeMetaPluginProperties(
+            metaPlugin,
+            "description", "test_plugin",
+            "name", "meta_plugin",
+            "plugins", "test_plugin,other_plugin");
+
+        // this plugin will have a controller daemon
+        Path plugin = metaPlugin.resolve("test_plugin");
+
+        Files.createDirectories(plugin);
+        PluginTestUtil.writePluginProperties(
+            plugin,
+            "description", "test_plugin",
+            "version", Version.CURRENT.toString(),
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "name", "test_plugin",
+            "java.version", "1.8",
+            "classname", "TestPlugin",
+            "has.native.controller", "true");
+        Path controllerProgram = Platforms.nativeControllerPath(plugin);
+        createControllerProgram(controllerProgram);
+
+        // this plugin will not have a controller daemon
+        Path otherPlugin = metaPlugin.resolve("other_plugin");
+        Files.createDirectories(otherPlugin);
+        PluginTestUtil.writePluginProperties(
+            otherPlugin,
+            "description", "other_plugin",
+            "version", Version.CURRENT.toString(),
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "name", "other_plugin",
+            "java.version", "1.8",
+            "classname", "OtherPlugin",
+            "has.native.controller", "false");
+
+        Spawner spawner = new Spawner();
+        spawner.spawnNativePluginControllers(environment);
+
+        List<Process> processes = spawner.getProcesses();
+        /*
+         * As there should only be a reference in the list for the plugin that had the controller
+         * daemon, we expect one here.
+         */
+        assertThat(processes, hasSize(1));
+        Process process = processes.get(0);
+        final InputStreamReader in =
+            new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8);
+        try (BufferedReader stdoutReader = new BufferedReader(in)) {
+            String line = stdoutReader.readLine();
+            assertEquals("I am alive", line);
+            spawner.close();
+            /*
+             * Fail if the process does not die within one second; usually it will be even quicker
+             * but it depends on OS scheduling.
+             */
+            assertTrue(process.waitFor(1, TimeUnit.SECONDS));
+        }
+    }
+
     public void testControllerSpawnWithIncorrectDescriptor() throws IOException {
         // this plugin will have a controller daemon
         Path esHome = createTempDir().resolve("esHome");
@@ -174,7 +252,7 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
 
         Path plugin = environment.pluginsFile().resolve("test_plugin");
         Files.createDirectories(plugin);
-        PluginTestUtil.writeProperties(
+        PluginTestUtil.writePluginProperties(
                 plugin,
                 "description", "test_plugin",
                 "version", Version.CURRENT.toString(),

settings.gradle

@@ -90,15 +90,6 @@ List projects = [
   'qa:query-builder-bwc'
 ]
 
-File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples')
-List<String> examplePlugins = []
-for (File example : examplePluginsDir.listFiles()) {
-  if (example.isDirectory() == false) continue;
-  if (example.name.startsWith('build') || example.name.startsWith('.')) continue;
-  projects.add("example-plugins:${example.name}".toString())
-  examplePlugins.add(example.name)
-}
-
 projects.add("libs")
 File libsDir = new File(rootProject.projectDir, 'libs')
 for (File libDir : new File(rootProject.projectDir, 'libs').listFiles()) {
@@ -124,11 +115,6 @@ if (isEclipse) {
 include projects.toArray(new String[0])
 
 project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
-project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples')
-
-for (String example : examplePlugins) {
-  project(":example-plugins:${example}").projectDir = new File(rootProject.projectDir, "plugins/examples/${example}")
-}
 
 /* The BWC snapshot projects share the same build directory and build file,
  * but apply to different backwards compatibility branches. */
@@ -170,7 +156,7 @@ void addSubProjects(String path, File dir, List<String> projects, List<String> b
     }
     // TODO do we want to assert that there's nothing else in the bwc directory?
   } else {
-    if (path.isEmpty()) {
+    if (path.isEmpty() || path.startsWith(':example-plugins')) {
       project(projectName).projectDir = dir
     }
     for (File subdir : dir.listFiles()) {
@@ -179,6 +165,15 @@ void addSubProjects(String path, File dir, List<String> projects, List<String> b
   }
 }
 
+// include example plugins
+File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples')
+for (File example : examplePluginsDir.listFiles()) {
+  if (example.isDirectory() == false) continue;
+  if (example.name.startsWith('build') || example.name.startsWith('.')) continue;
+  addSubProjects(':example-plugins', example, projects, [])
+}
+project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples')
+
 // look for extra plugins for elasticsearch
 File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra")
 if (extraProjects.exists()) {

test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java

@@ -27,12 +27,18 @@ import java.util.Properties;
 
 /** Utility methods for testing plugins */
 public class PluginTestUtil {
-    
+    public static void writeMetaPluginProperties(Path pluginDir, String... stringProps) throws IOException {
+        writeProperties(pluginDir.resolve(MetaPluginInfo.ES_META_PLUGIN_PROPERTIES), stringProps);
+    }
+
+    public static void writePluginProperties(Path pluginDir, String... stringProps) throws IOException {
+        writeProperties(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES), stringProps);
+    }
+
     /** convenience method to write a plugin properties file */
-    public static void writeProperties(Path pluginDir, String... stringProps) throws IOException {
+    private static void writeProperties(Path propertiesFile, String... stringProps) throws IOException {
         assert stringProps.length % 2 == 0;
-        Files.createDirectories(pluginDir);
+        Files.createDirectories(propertiesFile.getParent());
-        Path propertiesFile = pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES);
         Properties properties =  new Properties();
         for (int i = 0; i < stringProps.length; i += 2) {
             properties.put(stringProps[i], stringProps[i + 1]);