Plugins: Add plugin extension capabilities (#27881)

This commit adds the infrastructure to plugin building and loading to
allow one plugin to extend another. That is, one plugin may extend
another by the "parent" plugin allowing itself to be extended through
java SPI. When all plugins extending a plugin are finished loading, the
"parent" plugin has a callback (through the ExtensiblePlugin interface)
allowing it to reload SPI.

This commit also adds an example plugin which uses as-yet implemented
extensibility (adding to the painless whitelist).

build.gradle

@@ -139,7 +139,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 allprojects {
-  ext.bwc_tests_enabled = true
+  // TODO: re-enable after https://github.com/elastic/elasticsearch/pull/27881 is backported
+  ext.bwc_tests_enabled = false
 }
 
 task verifyBwcTestsEnabled {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -39,6 +39,10 @@ class PluginPropertiesExtension {
     @Input
     String classname
 
+    /** Other plugins this plugin extends through SPI */
+    @Input
+    List<String> extendedPlugins = []
+
     @Input
     boolean hasNativeController = false
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -80,6 +80,7 @@ class PluginPropertiesTask extends Copy {
             'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
+            'extendedPlugins': extension.extendedPlugins.join(','),
             'hasNativeController': extension.hasNativeController,
             'requiresKeystore': extension.requiresKeystore
         ]

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -40,6 +40,9 @@ java.version=${javaVersion}
 elasticsearch.version=${elasticsearchVersion}
 ### optional elements for plugins:
 #
+#  'extended.plugins': other plugins this plugin extends through SPI
+extended.plugins=${extendedPlugins}
+#
 # 'has.native.controller': whether or not the plugin has a native controller
 has.native.controller=${hasNativeController}
 #

core/build.gradle

@@ -38,6 +38,9 @@ archivesBaseName = 'elasticsearch'
 
 dependencies {
 
+  compileOnly project(':libs:plugin-classloader')
+  testRuntime project(':libs:plugin-classloader')
+
   // lucene
   compile "org.apache.lucene:lucene-core:${versions.lucene}"
   compile "org.apache.lucene:lucene-analyzers-common:${versions.lucene}"

core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

@@ -33,6 +33,7 @@ import java.security.Policy;
 import java.security.ProtectionDomain;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Predicate;
 
 /** custom policy for union of static and dynamic permissions */
@@ -49,9 +50,9 @@ final class ESPolicy extends Policy {
     final PermissionCollection dynamic;
     final Map<String,Policy> plugins;
 
-    ESPolicy(PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
+    ESPolicy(Map<String, URL> codebases, PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
-        this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), JarHell.parseClassPath());
+        this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), codebases);
-        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptySet());
+        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptyMap());
         if (filterBadDefaults) {
             this.system = new SystemPolicy(Policy.getPolicy());
         } else {

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -48,10 +48,13 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath;
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addSingleFilePath;
@@ -116,7 +119,8 @@ final class Security {
     static void configure(Environment environment, boolean filterBadDefaults) throws IOException, NoSuchAlgorithmException {
 
         // enable security policy: union of template and environment-based paths, and possibly plugin permissions
-        Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
+        Map<String, URL> codebases = getCodebaseJarMap(JarHell.parseClassPath());
+        Policy.setPolicy(new ESPolicy(codebases, createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
 
         // enable security manager
         final String[] classesThatCanExit =
@@ -130,6 +134,27 @@ final class Security {
         selfTest();
     }
 
+    /**
+     * Return a map from codebase name to codebase url of jar codebases used by ES core.
+     */
+    @SuppressForbidden(reason = "find URL path")
+    static Map<String, URL> getCodebaseJarMap(Set<URL> urls) {
+        Map<String, URL> codebases = new LinkedHashMap<>(); // maintain order
+        for (URL url : urls) {
+            try {
+                String fileName = PathUtils.get(url.toURI()).getFileName().toString();
+                if (fileName.endsWith(".jar") == false) {
+                    // tests :(
+                    continue;
+                }
+                codebases.put(fileName, url);
+            } catch (URISyntaxException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return codebases;
+    }
+
     /**
      * Sets properties (codebase URLs) for policy files.
      * we look for matching plugins and set URLs to fit
@@ -174,7 +199,7 @@ final class Security {
                 }
 
                 // parse the plugin's policy file into a set of permissions
-                Policy policy = readPolicy(policyFile.toUri().toURL(), codebases);
+                Policy policy = readPolicy(policyFile.toUri().toURL(), getCodebaseJarMap(codebases));
 
                 // consult this policy for each of the plugin's jars:
                 for (URL url : codebases) {
@@ -197,21 +222,20 @@ final class Security {
      * would map to full URL.
      */
     @SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
-    static Policy readPolicy(URL policyFile, Set<URL> codebases) {
+    static Policy readPolicy(URL policyFile, Map<String, URL> codebases) {
         try {
             List<String> propertiesSet = new ArrayList<>();
             try {
                 // set codebase properties
-                for (URL url : codebases) {
+                for (Map.Entry<String,URL> codebase : codebases.entrySet()) {
-                    String fileName = PathUtils.get(url.toURI()).getFileName().toString();
+                    String name = codebase.getKey();
-                    if (fileName.endsWith(".jar") == false) {
+                    URL url = codebase.getValue();
-                        continue; // tests :(
+
-                    }
                     // We attempt to use a versionless identifier for each codebase. This assumes a specific version
                     // format in the jar filename. While we cannot ensure all jars in all plugins use this format, nonconformity
                     // only means policy grants would need to include the entire jar filename as they always have before.
-                    String property = "codebase." + fileName;
+                    String property = "codebase." + name;
-                    String aliasProperty = "codebase." + fileName.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
+                    String aliasProperty = "codebase." + name.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
                     if (aliasProperty.equals(property) == false) {
                         propertiesSet.add(aliasProperty);
                         String previous = System.setProperty(aliasProperty, url.toString());

core/src/main/java/org/elasticsearch/plugins/ExtensiblePlugin.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+/**
+ * An extension point for {@link Plugin} implementations to be themselves extensible.
+ *
+ * This class provides a callback for extensible plugins to be informed of other plugins
+ * which extend them.
+ */
+public interface ExtensiblePlugin {
+
+    /**
+     * Reload any SPI implementations from the given classloader.
+     */
+    default void reloadSPI(ClassLoader loader) {}
+}

core/src/main/java/org/elasticsearch/plugins/PluginInfo.java

@@ -22,10 +22,10 @@ package org.elasticsearch.plugins;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -33,6 +33,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
@@ -51,6 +54,7 @@ public class PluginInfo implements Writeable, ToXContentObject {
     private final String description;
     private final String version;
     private final String classname;
+    private final List<String> extendedPlugins;
     private final boolean hasNativeController;
     private final boolean requiresKeystore;
 
@@ -61,15 +65,17 @@ public class PluginInfo implements Writeable, ToXContentObject {
      * @param description         a description of the plugin
      * @param version             the version of Elasticsearch the plugin is built for
      * @param classname           the entry point to the plugin
+     * @param extendedPlugins     other plugins this plugin extends through SPI
      * @param hasNativeController whether or not the plugin has a native controller
      * @param requiresKeystore    whether or not the plugin requires the elasticsearch keystore to be created
      */
     public PluginInfo(String name, String description, String version, String classname,
-                      boolean hasNativeController, boolean requiresKeystore) {
+                      List<String> extendedPlugins, boolean hasNativeController, boolean requiresKeystore) {
         this.name = name;
         this.description = description;
         this.version = version;
         this.classname = classname;
+        this.extendedPlugins = Collections.unmodifiableList(extendedPlugins);
         this.hasNativeController = hasNativeController;
         this.requiresKeystore = requiresKeystore;
     }
@@ -85,6 +91,11 @@ public class PluginInfo implements Writeable, ToXContentObject {
         this.description = in.readString();
         this.version = in.readString();
         this.classname = in.readString();
+        if (in.getVersion().onOrAfter(Version.V_6_2_0)) {
+            extendedPlugins = in.readList(StreamInput::readString);
+        } else {
+            extendedPlugins = Collections.emptyList();
+        }
         if (in.getVersion().onOrAfter(Version.V_5_4_0)) {
             hasNativeController = in.readBoolean();
         } else {
@@ -103,6 +114,9 @@ public class PluginInfo implements Writeable, ToXContentObject {
         out.writeString(description);
         out.writeString(version);
         out.writeString(classname);
+        if (out.getVersion().onOrAfter(Version.V_6_2_0)) {
+            out.writeStringList(extendedPlugins);
+        }
         if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
             out.writeBoolean(hasNativeController);
         }
@@ -176,6 +190,14 @@ public class PluginInfo implements Writeable, ToXContentObject {
                     "property [classname] is missing for plugin [" + name + "]");
         }
 
+        final String extendedString = propsMap.remove("extended.plugins");
+        final List<String> extendedPlugins;
+        if (extendedString == null) {
+            extendedPlugins = Collections.emptyList();
+        } else {
+            extendedPlugins = Arrays.asList(Strings.delimitedListToStringArray(extendedString, ","));
+        }
+
         final String hasNativeControllerValue = propsMap.remove("has.native.controller");
         final boolean hasNativeController;
         if (hasNativeControllerValue == null) {
@@ -216,7 +238,7 @@ public class PluginInfo implements Writeable, ToXContentObject {
             throw new IllegalArgumentException("Unknown properties in plugin descriptor: " + propsMap.keySet());
         }
 
-        return new PluginInfo(name, description, version, classname, hasNativeController, requiresKeystore);
+        return new PluginInfo(name, description, version, classname, extendedPlugins, hasNativeController, requiresKeystore);
     }
 
     /**
@@ -246,6 +268,15 @@ public class PluginInfo implements Writeable, ToXContentObject {
         return classname;
     }
 
+    /**
+     * Other plugins this plugin extends through SPI.
+     *
+     * @return the names of the plugins extended
+     */
+    public List<String> getExtendedPlugins() {
+        return extendedPlugins;
+    }
+
     /**
      * The version of Elasticsearch the plugin was built for.
      *
@@ -281,6 +312,7 @@ public class PluginInfo implements Writeable, ToXContentObject {
             builder.field("version", version);
             builder.field("description", description);
             builder.field("classname", classname);
+            builder.field("extended_plugins", extendedPlugins);
             builder.field("has_native_controller", hasNativeController);
             builder.field("requires_keystore", requiresKeystore);
         }
@@ -316,6 +348,7 @@ public class PluginInfo implements Writeable, ToXContentObject {
                 .append("Version: ").append(version).append("\n")
                 .append("Native Controller: ").append(hasNativeController).append("\n")
                 .append("Requires Keystore: ").append(requiresKeystore).append("\n")
+                .append("Extended Plugins: ").append(extendedPlugins).append("\n")
                 .append(" * Classname: ").append(classname);
         return information.toString();
     }

core/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java

@@ -16,18 +16,18 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.plugins;
 
-public class DummyPluginInfo extends PluginInfo {
+import java.util.List;
 
-    private DummyPluginInfo(String name, String description, String version, String classname) {
+/**
-        super(name, description, version, classname, false, false);
+ * This class exists solely as an intermediate layer to avoid causing PluginsService
-    }
+ * to load ExtendedPluginsClassLoader when used in the transport client.
+ */
+class PluginLoaderIndirection {
 
-    public static final DummyPluginInfo INSTANCE =
+    static ClassLoader createLoader(ClassLoader parent, List<ClassLoader> extendedLoaders) {
-            new DummyPluginInfo(
+        return ExtendedPluginsClassLoader.create(parent, extendedLoaders);
-                    "dummy_plugin_name",
+    }
-                    "dummy plugin description",
-                    "dummy_plugin_version",
-                    "DummyPluginName");
 }

core/src/main/java/org/elasticsearch/plugins/PluginsService.java

@@ -103,7 +103,8 @@ public class PluginsService extends AbstractComponent {
         // first we load plugins that are on the classpath. this is for tests and transport clients
         for (Class<? extends Plugin> pluginClass : classpathPlugins) {
             Plugin plugin = loadPlugin(pluginClass, settings, configPath);
-            PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA", pluginClass.getName(), false, false);
+            PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA",
+                                                   pluginClass.getName(), Collections.emptyList(), false, false);
             if (logger.isTraceEnabled()) {
                 logger.trace("plugin loaded from classpath [{}]", pluginInfo);
             }
@@ -129,11 +130,15 @@ public class PluginsService extends AbstractComponent {
         // now, find all the ones that are in plugins/
         if (pluginsDirectory != null) {
             try {
+                // TODO: remove this leniency, but tests bogusly rely on it
+                if (isAccessibleDirectory(pluginsDirectory, logger)) {
+                    checkForFailedPluginRemovals(pluginsDirectory);
                     Set<Bundle> plugins = getPluginBundles(pluginsDirectory);
                     for (Bundle bundle : plugins) {
                         pluginsList.add(bundle.plugin);
                     }
                     seenBundles.addAll(plugins);
+                }
             } catch (IOException ex) {
                 throw new IllegalStateException("Unable to initialize plugins", ex);
             }
@@ -243,8 +248,19 @@ public class PluginsService extends AbstractComponent {
         final PluginInfo plugin;
         final Set<URL> urls;
 
-        Bundle(PluginInfo plugin, Set<URL> urls) {
+        Bundle(PluginInfo plugin, Path dir) throws IOException {
             this.plugin = Objects.requireNonNull(plugin);
+            Set<URL> urls = new LinkedHashSet<>();
+            // gather urls for jar files
+            try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(dir, "*.jar")) {
+                for (Path jar : jarStream) {
+                    // normalize with toRealPath to get symlinks out of our hair
+                    URL url = jar.toRealPath().toUri().toURL();
+                    if (urls.add(url) == false) {
+                        throw new IllegalStateException("duplicate codebase: " + url);
+                    }
+                }
+            }
             this.urls = Objects.requireNonNull(urls);
         }
 
@@ -273,18 +289,7 @@ public class PluginsService extends AbstractComponent {
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(modulesDirectory)) {
             for (Path module : stream) {
                 PluginInfo info = PluginInfo.readFromProperties(module);
-                Set<URL> urls = new LinkedHashSet<>();
+                if (bundles.add(new Bundle(info, module)) == false) {
-                // gather urls for jar files
-                try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(module, "*.jar")) {
-                    for (Path jar : jarStream) {
-                        // normalize with toRealPath to get symlinks out of our hair
-                        URL url = jar.toRealPath().toUri().toURL();
-                            if (urls.add(url) == false) {
-                                throw new IllegalStateException("duplicate codebase: " + url);
-                            }
-                    }
-                }
-                if (bundles.add(new Bundle(info, urls)) == false) {
                     throw new IllegalStateException("duplicate module: " + info);
                 }
             }
@@ -315,21 +320,16 @@ public class PluginsService extends AbstractComponent {
 
     static Set<Bundle> getPluginBundles(Path pluginsDirectory) throws IOException {
         Logger logger = Loggers.getLogger(PluginsService.class);
-
-        // TODO: remove this leniency, but tests bogusly rely on it
-        if (!isAccessibleDirectory(pluginsDirectory, logger)) {
-            return Collections.emptySet();
-        }
-
         Set<Bundle> bundles = new LinkedHashSet<>();
 
-        checkForFailedPluginRemovals(pluginsDirectory);
-
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
             for (Path plugin : stream) {
                 if (FileSystemUtils.isDesktopServicesStore(plugin)) {
                     continue;
                 }
+                if (plugin.getFileName().toString().startsWith(".removing-")) {
+                    continue;
+                }
                 logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
                 final PluginInfo info;
                 try {
@@ -339,17 +339,7 @@ public class PluginsService extends AbstractComponent {
                         + plugin.getFileName() + "]. Was the plugin built before 2.0?", e);
                 }
 
-                Set<URL> urls = new LinkedHashSet<>();
+                if (bundles.add(new Bundle(info, plugin)) == false) {
-                try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
-                    for (Path jar : jarStream) {
-                        // normalize with toRealPath to get symlinks out of our hair
-                        URL url = jar.toRealPath().toUri().toURL();
-                        if (urls.add(url) == false) {
-                            throw new IllegalStateException("duplicate codebase: " + url);
-                        }
-                    }
-                }
-                if (bundles.add(new Bundle(info, urls)) == false) {
                     throw new IllegalStateException("duplicate plugin: " + info);
                 }
             }
@@ -358,42 +348,153 @@ public class PluginsService extends AbstractComponent {
         return bundles;
     }
 
+    /**
+     * Return the given bundles, sorted in dependency loading order.
+     *
+     * This sort is stable, so that if two plugins do not have any interdependency,
+     * their relative order from iteration of the provided set will not change.
+     *
+     * @throws IllegalStateException if a dependency cycle is found
+     */
+    // pkg private for tests
+    static List<Bundle> sortBundles(Set<Bundle> bundles) {
+        Map<String, Bundle> namedBundles = bundles.stream().collect(Collectors.toMap(b -> b.plugin.getName(), Function.identity()));
+        LinkedHashSet<Bundle> sortedBundles = new LinkedHashSet<>();
+        LinkedHashSet<String> dependencyStack = new LinkedHashSet<>();
+        for (Bundle bundle : bundles) {
+            addSortedBundle(bundle, namedBundles, sortedBundles, dependencyStack);
+        }
+        return new ArrayList<>(sortedBundles);
+    }
+
+    // add the given bundle to the sorted bundles, first adding dependencies
+    private static void addSortedBundle(Bundle bundle, Map<String, Bundle> bundles, LinkedHashSet<Bundle> sortedBundles,
+                                        LinkedHashSet<String> dependencyStack) {
+
+        String name = bundle.plugin.getName();
+        if (dependencyStack.contains(name)) {
+            StringBuilder msg = new StringBuilder("Cycle found in plugin dependencies: ");
+            dependencyStack.forEach(s -> {
+                msg.append(s);
+                msg.append(" -> ");
+            });
+            msg.append(name);
+            throw new IllegalStateException(msg.toString());
+        }
+        if (sortedBundles.contains(bundle)) {
+            // already added this plugin, via a dependency
+            return;
+        }
+
+        dependencyStack.add(name);
+        for (String dependency : bundle.plugin.getExtendedPlugins()) {
+            Bundle depBundle = bundles.get(dependency);
+            if (depBundle == null) {
+                throw new IllegalArgumentException("Missing plugin [" + dependency + "], dependency of [" + name + "]");
+            }
+            addSortedBundle(depBundle, bundles, sortedBundles, dependencyStack);
+            assert sortedBundles.contains(depBundle);
+        }
+        dependencyStack.remove(name);
+
+        sortedBundles.add(bundle);
+    }
+
     private List<Tuple<PluginInfo,Plugin>> loadBundles(Set<Bundle> bundles) {
         List<Tuple<PluginInfo, Plugin>> plugins = new ArrayList<>();
+        Map<String, Plugin> loaded = new HashMap<>();
+        Map<String, Set<URL>> transitiveUrls = new HashMap<>();
+        List<Bundle> sortedBundles = sortBundles(bundles);
+
+        for (Bundle bundle : sortedBundles) {
+            checkBundleJarHell(bundle, transitiveUrls);
+
+            final Plugin plugin = loadBundle(bundle, loaded);
+            plugins.add(new Tuple<>(bundle.plugin, plugin));
+        }
+
+        return Collections.unmodifiableList(plugins);
+    }
+
+    // jar-hell check the bundle against the parent classloader and extended plugins
+    // the plugin cli does it, but we do it again, in case lusers mess with jar files manually
+    static void checkBundleJarHell(Bundle bundle, Map<String, Set<URL>> transitiveUrls) {
+        // invariant: any plugins this plugin bundle extends have already been added to transitiveUrls
+        List<String> exts = bundle.plugin.getExtendedPlugins();
 
-        for (Bundle bundle : bundles) {
-            // jar-hell check the bundle against the parent classloader
-            // pluginmanager does it, but we do it again, in case lusers mess with jar files manually
         try {
+            Set<URL> urls = new HashSet<>();
+            for (String extendedPlugin : exts) {
+                Set<URL> pluginUrls = transitiveUrls.get(extendedPlugin);
+                assert pluginUrls != null : "transitive urls should have already been set for " + extendedPlugin;
+
+                Set<URL> intersection = new HashSet<>(urls);
+                intersection.retainAll(pluginUrls);
+                if (intersection.isEmpty() == false) {
+                    throw new IllegalStateException("jar hell! extended plugins " + exts +
+                                                    " have duplicate codebases with each other: " + intersection);
+                }
+
+                intersection = new HashSet<>(bundle.urls);
+                intersection.retainAll(pluginUrls);
+                if (intersection.isEmpty() == false) {
+                    throw new IllegalStateException("jar hell! duplicate codebases with extended plugin [" +
+                                                    extendedPlugin + "]: " + intersection);
+                }
+
+                urls.addAll(pluginUrls);
+                JarHell.checkJarHell(urls); // check jarhell as we add each extended plugin's urls
+            }
+
+            urls.addAll(bundle.urls);
+            JarHell.checkJarHell(urls); // check jarhell of each extended plugin against this plugin
+            transitiveUrls.put(bundle.plugin.getName(), urls);
+
             Set<URL> classpath = JarHell.parseClassPath();
-                // check we don't have conflicting codebases
+            // check we don't have conflicting codebases with core
             Set<URL> intersection = new HashSet<>(classpath);
             intersection.retainAll(bundle.urls);
             if (intersection.isEmpty() == false) {
-                    throw new IllegalStateException("jar hell! duplicate codebases between" +
+                throw new IllegalStateException("jar hell! duplicate codebases between plugin and core: " + intersection);
-                                                    " plugin and core: " + intersection);
             }
             // check we don't have conflicting classes
             Set<URL> union = new HashSet<>(classpath);
             union.addAll(bundle.urls);
             JarHell.checkJarHell(union);
         } catch (Exception e) {
-                throw new IllegalStateException("failed to load plugin " + bundle.plugin +
+            throw new IllegalStateException("failed to load plugin " + bundle.plugin.getName() + " due to jar hell", e);
-                                                " due to jar hell", e);
+        }
+    }
+
+    private Plugin loadBundle(Bundle bundle, Map<String, Plugin> loaded) {
+        String name = bundle.plugin.getName();
+
+        // collect loaders of extended plugins
+        List<ClassLoader> extendedLoaders = new ArrayList<>();
+        for (String extendedPluginName : bundle.plugin.getExtendedPlugins()) {
+            Plugin extendedPlugin = loaded.get(extendedPluginName);
+            assert extendedPlugin != null;
+            if (ExtensiblePlugin.class.isInstance(extendedPlugin) == false) {
+                throw new IllegalStateException("Plugin [" + name + "] cannot extend non-extensible plugin [" + extendedPluginName + "]");
+            }
+            extendedLoaders.add(extendedPlugin.getClass().getClassLoader());
         }
 
         // create a child to load the plugin in this bundle
-            ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]),
+        ClassLoader parentLoader = PluginLoaderIndirection.createLoader(getClass().getClassLoader(), extendedLoaders);
-                                                            getClass().getClassLoader());
+        ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), parentLoader);
-            // reload lucene SPI with any new services from the plugin
+
+        // reload SPI with any new services from the plugin
         reloadLuceneSPI(loader);
-            final Class<? extends Plugin> pluginClass =
+        for (String extendedPluginName : bundle.plugin.getExtendedPlugins()) {
-                loadPluginClass(bundle.plugin.getClassname(), loader);
+            // note: already asserted above that extended plugins are loaded and extensible
-            final Plugin plugin = loadPlugin(pluginClass, settings, configPath);
+            ExtensiblePlugin.class.cast(loaded.get(extendedPluginName)).reloadSPI(loader);
-            plugins.add(new Tuple<>(bundle.plugin, plugin));
         }
 
-        return Collections.unmodifiableList(plugins);
+        Class<? extends Plugin> pluginClass = loadPluginClass(bundle.plugin.getClassname(), loader);
+        Plugin plugin = loadPlugin(pluginClass, settings, configPath);
+        loaded.put(name, plugin);
+        return plugin;
     }
 
     /**

core/src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -47,6 +47,11 @@ grant codeBase "${codebase.lucene-misc}" {
   permission java.nio.file.LinkPermission "hard";
 };
 
+grant codeBase "${codebase.plugin-classloader}" {
+  // needed to create the classloader which allows plugins to extend other plugins
+  permission java.lang.RuntimePermission "createClassLoader";
+};
+
 //// Everything else:
 
 grant {

core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java

@@ -46,6 +46,7 @@ import org.elasticsearch.transport.TransportInfo;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -143,13 +144,15 @@ public class NodeInfoStreamingTests extends ESTestCase {
             List<PluginInfo> plugins = new ArrayList<>();
             for (int i = 0; i < numPlugins; i++) {
                 plugins.add(new PluginInfo(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10),
-                        randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), randomBoolean(), randomBoolean()));
+                        randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), Collections.emptyList(),
+                    randomBoolean(), randomBoolean()));
             }
             int numModules = randomIntBetween(0, 5);
             List<PluginInfo> modules = new ArrayList<>();
             for (int i = 0; i < numModules; i++) {
                 modules.add(new PluginInfo(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10),
-                        randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), randomBoolean(), randomBoolean()));
+                        randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), Collections.emptyList(),
+                    randomBoolean(), randomBoolean()));
             }
             pluginsAndModules = new PluginsAndModules(plugins, modules);
         }

core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java

@@ -21,8 +21,11 @@ package org.elasticsearch.plugins;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
+import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.test.ESTestCase;
 
+import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -30,6 +33,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -49,6 +53,7 @@ public class PluginInfoTests extends ESTestCase {
         assertEquals("fake desc", info.getDescription());
         assertEquals("1.0", info.getVersion());
         assertEquals("FakePlugin", info.getClassname());
+        assertThat(info.getExtendedPlugins(), empty());
     }
 
     public void testReadFromPropertiesNameMissing() throws Exception {
@@ -161,13 +166,67 @@ public class PluginInfoTests extends ESTestCase {
         assertThat(e.getMessage(), containsString("property [classname] is missing"));
     }
 
+    public void testExtendedPluginsSingleExtension() throws Exception {
+        Path pluginDir = createTempDir().resolve("fake-plugin");
+        PluginTestUtil.writeProperties(pluginDir,
+            "description", "fake desc",
+            "name", "my_plugin",
+            "version", "1.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "FakePlugin",
+            "extended.plugins", "foo");
+        PluginInfo info = PluginInfo.readFromProperties(pluginDir);
+        assertThat(info.getExtendedPlugins(), contains("foo"));
+    }
+
+    public void testExtendedPluginsMultipleExtensions() throws Exception {
+        Path pluginDir = createTempDir().resolve("fake-plugin");
+        PluginTestUtil.writeProperties(pluginDir,
+            "description", "fake desc",
+            "name", "my_plugin",
+            "version", "1.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "FakePlugin",
+            "extended.plugins", "foo,bar,baz");
+        PluginInfo info = PluginInfo.readFromProperties(pluginDir);
+        assertThat(info.getExtendedPlugins(), contains("foo", "bar", "baz"));
+    }
+
+    public void testExtendedPluginsEmpty() throws Exception {
+        Path pluginDir = createTempDir().resolve("fake-plugin");
+        PluginTestUtil.writeProperties(pluginDir,
+            "description", "fake desc",
+            "name", "my_plugin",
+            "version", "1.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "FakePlugin",
+            "extended.plugins", "");
+        PluginInfo info = PluginInfo.readFromProperties(pluginDir);
+        assertThat(info.getExtendedPlugins(), empty());
+    }
+
+    public void testSerialize() throws Exception {
+        PluginInfo info = new PluginInfo("c", "foo", "dummy", "dummyclass",
+                                         Collections.singletonList("foo"), randomBoolean(), randomBoolean());
+        BytesStreamOutput output = new BytesStreamOutput();
+        info.writeTo(output);
+        ByteBuffer buffer = ByteBuffer.wrap(output.bytes().toBytesRef().bytes);
+        ByteBufferStreamInput input = new ByteBufferStreamInput(buffer);
+        PluginInfo info2 = new PluginInfo(input);
+        assertThat(info2.toString(), equalTo(info.toString()));
+
+    }
+
     public void testPluginListSorted() {
         List<PluginInfo> plugins = new ArrayList<>();
-        plugins.add(new PluginInfo("c", "foo", "dummy", "dummyclass", randomBoolean(), randomBoolean()));
+        plugins.add(new PluginInfo("c", "foo", "dummy", "dummyclass", Collections.emptyList(), randomBoolean(), randomBoolean()));
-        plugins.add(new PluginInfo("b", "foo", "dummy", "dummyclass", randomBoolean(), randomBoolean()));
+        plugins.add(new PluginInfo("b", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
-        plugins.add(new PluginInfo("e", "foo", "dummy", "dummyclass", randomBoolean(), randomBoolean()));
+        plugins.add(new PluginInfo("e", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
-        plugins.add(new PluginInfo("a", "foo", "dummy", "dummyclass", randomBoolean(), randomBoolean()));
+        plugins.add(new PluginInfo("a", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
-        plugins.add(new PluginInfo("d", "foo", "dummy", "dummyclass", randomBoolean(), randomBoolean()));
+        plugins.add(new PluginInfo("d", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean()));
         PluginsAndModules pluginsInfo = new PluginsAndModules(plugins, Collections.emptyList());
 
         final List<PluginInfo> infos = pluginsInfo.getPluginInfos();

core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java

@@ -19,25 +19,39 @@
 
 package org.elasticsearch.plugins;
 
+import org.apache.log4j.Level;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
 import java.nio.file.FileSystemException;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
 
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
@@ -269,4 +283,283 @@ public class PluginsServiceTests extends ESTestCase {
         }
     }
 
+    public void testSortBundlesCycleSelfReference() throws Exception {
+        Path pluginDir = createTempDir();
+        PluginInfo info = new PluginInfo("foo", "desc", "1.0", "MyPlugin", Collections.singletonList("foo"), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info, pluginDir);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () ->
+            PluginsService.sortBundles(Collections.singleton(bundle))
+        );
+        assertEquals("Cycle found in plugin dependencies: foo -> foo", e.getMessage());
+    }
+
+    public void testSortBundlesCycle() throws Exception {
+        Path pluginDir = createTempDir();
+        Set<PluginsService.Bundle> bundles = new LinkedHashSet<>(); // control iteration order, so we get know the beginning of the cycle
+        PluginInfo info = new PluginInfo("foo", "desc", "1.0", "MyPlugin", Arrays.asList("bar", "other"), false, false);
+        bundles.add(new PluginsService.Bundle(info, pluginDir));
+        PluginInfo info2 = new PluginInfo("bar", "desc", "1.0", "MyPlugin", Collections.singletonList("baz"), false, false);
+        bundles.add(new PluginsService.Bundle(info2, pluginDir));
+        PluginInfo info3 = new PluginInfo("baz", "desc", "1.0", "MyPlugin", Collections.singletonList("foo"), false, false);
+        bundles.add(new PluginsService.Bundle(info3, pluginDir));
+        PluginInfo info4 = new PluginInfo("other", "desc", "1.0", "MyPlugin", Collections.emptyList(), false, false);
+        bundles.add(new PluginsService.Bundle(info4, pluginDir));
+
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> PluginsService.sortBundles(bundles));
+        assertEquals("Cycle found in plugin dependencies: foo -> bar -> baz -> foo", e.getMessage());
+    }
+
+    public void testSortBundlesSingle() throws Exception {
+        Path pluginDir = createTempDir();
+        PluginInfo info = new PluginInfo("foo", "desc", "1.0", "MyPlugin", Collections.emptyList(), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info, pluginDir);
+        List<PluginsService.Bundle> sortedBundles = PluginsService.sortBundles(Collections.singleton(bundle));
+        assertThat(sortedBundles, Matchers.contains(bundle));
+    }
+
+    public void testSortBundlesNoDeps() throws Exception {
+        Path pluginDir = createTempDir();
+        Set<PluginsService.Bundle> bundles = new LinkedHashSet<>(); // control iteration order
+        PluginInfo info1 = new PluginInfo("foo", "desc", "1.0", "MyPlugin", Collections.emptyList(), false, false);
+        PluginsService.Bundle bundle1 = new PluginsService.Bundle(info1, pluginDir);
+        bundles.add(bundle1);
+        PluginInfo info2 = new PluginInfo("bar", "desc", "1.0", "MyPlugin", Collections.emptyList(), false, false);
+        PluginsService.Bundle bundle2 = new PluginsService.Bundle(info2, pluginDir);
+        bundles.add(bundle2);
+        PluginInfo info3 = new PluginInfo("baz", "desc", "1.0", "MyPlugin", Collections.emptyList(), false, false);
+        PluginsService.Bundle bundle3 = new PluginsService.Bundle(info3, pluginDir);
+        bundles.add(bundle3);
+        List<PluginsService.Bundle> sortedBundles = PluginsService.sortBundles(bundles);
+        assertThat(sortedBundles, Matchers.contains(bundle1, bundle2, bundle3));
+    }
+
+    public void testSortBundlesMissingDep() throws Exception {
+        Path pluginDir = createTempDir();
+        PluginInfo info = new PluginInfo("foo", "desc", "1.0", "MyPlugin", Collections.singletonList("dne"), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info, pluginDir);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            PluginsService.sortBundles(Collections.singleton(bundle))
+        );
+        assertEquals("Missing plugin [dne], dependency of [foo]", e.getMessage());
+    }
+
+    public void testSortBundlesCommonDep() throws Exception {
+        Path pluginDir = createTempDir();
+        Set<PluginsService.Bundle> bundles = new LinkedHashSet<>(); // control iteration order
+        PluginInfo info1 = new PluginInfo("grandparent", "desc", "1.0", "MyPlugin", Collections.emptyList(), false, false);
+        PluginsService.Bundle bundle1 = new PluginsService.Bundle(info1, pluginDir);
+        bundles.add(bundle1);
+        PluginInfo info2 = new PluginInfo("foo", "desc", "1.0", "MyPlugin", Collections.singletonList("common"), false, false);
+        PluginsService.Bundle bundle2 = new PluginsService.Bundle(info2, pluginDir);
+        bundles.add(bundle2);
+        PluginInfo info3 = new PluginInfo("bar", "desc", "1.0", "MyPlugin", Collections.singletonList("common"), false, false);
+        PluginsService.Bundle bundle3 = new PluginsService.Bundle(info3, pluginDir);
+        bundles.add(bundle3);
+        PluginInfo info4 = new PluginInfo("common", "desc", "1.0", "MyPlugin", Collections.singletonList("grandparent"), false, false);
+        PluginsService.Bundle bundle4 = new PluginsService.Bundle(info4, pluginDir);
+        bundles.add(bundle4);
+        List<PluginsService.Bundle> sortedBundles = PluginsService.sortBundles(bundles);
+        assertThat(sortedBundles, Matchers.contains(bundle1, bundle4, bundle2, bundle3));
+    }
+
+    public void testSortBundlesAlreadyOrdered() throws Exception {
+        Path pluginDir = createTempDir();
+        Set<PluginsService.Bundle> bundles = new LinkedHashSet<>(); // control iteration order
+        PluginInfo info1 = new PluginInfo("dep", "desc", "1.0", "MyPlugin", Collections.emptyList(), false, false);
+        PluginsService.Bundle bundle1 = new PluginsService.Bundle(info1, pluginDir);
+        bundles.add(bundle1);
+        PluginInfo info2 = new PluginInfo("myplugin", "desc", "1.0", "MyPlugin", Collections.singletonList("dep"), false, false);
+        PluginsService.Bundle bundle2 = new PluginsService.Bundle(info2, pluginDir);
+        bundles.add(bundle2);
+        List<PluginsService.Bundle> sortedBundles = PluginsService.sortBundles(bundles);
+        assertThat(sortedBundles, Matchers.contains(bundle1, bundle2));
+    }
+
+    public static class DummyClass1 {}
+
+    public static class DummyClass2 {}
+
+    public static class DummyClass3 {}
+
+    void makeJar(Path jarFile, Class... classes) throws Exception {
+        try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(jarFile))) {
+            for (Class clazz : classes) {
+                String relativePath = clazz.getCanonicalName().replaceAll("\\.", "/") + ".class";
+                if (relativePath.contains(PluginsServiceTests.class.getSimpleName())) {
+                    // static inner class of this test
+                    relativePath = relativePath.replace("/" + clazz.getSimpleName(), "$" + clazz.getSimpleName());
+                }
+
+                Path codebase = PathUtils.get(clazz.getProtectionDomain().getCodeSource().getLocation().toURI());
+                if (codebase.toString().endsWith(".jar")) {
+                    // copy from jar, exactly as is
+                    out.putNextEntry(new ZipEntry(relativePath));
+                    try (ZipInputStream in = new ZipInputStream(Files.newInputStream(codebase))) {
+                        ZipEntry entry = in.getNextEntry();
+                        while (entry != null) {
+                            if (entry.getName().equals(relativePath)) {
+                                byte[] buffer = new byte[10*1024];
+                                int read = in.read(buffer);
+                                while (read != -1) {
+                                    out.write(buffer, 0, read);
+                                    read = in.read(buffer);
+                                }
+                                break;
+                            }
+                            in.closeEntry();
+                            entry = in.getNextEntry();
+                        }
+                    }
+                } else {
+                    // copy from dir, and use a different canonical path to not conflict with test classpath
+                    out.putNextEntry(new ZipEntry("test/" + clazz.getSimpleName() + ".class"));
+                    Files.copy(codebase.resolve(relativePath), out);
+                }
+                out.closeEntry();
+            }
+        }
+    }
+
+    public void testJarHellDuplicateCodebaseWithDep() throws Exception {
+        Path pluginDir = createTempDir();
+        Path dupJar = pluginDir.resolve("dup.jar");
+        makeJar(dupJar);
+        Map<String, Set<URL>> transitiveDeps = new HashMap<>();
+        transitiveDeps.put("dep", Collections.singleton(dupJar.toUri().toURL()));
+        PluginInfo info1 = new PluginInfo("myplugin", "desc", "1.0", "MyPlugin", Collections.singletonList("dep"), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info1, pluginDir);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () ->
+            PluginsService.checkBundleJarHell(bundle, transitiveDeps));
+        assertEquals("failed to load plugin myplugin due to jar hell", e.getMessage());
+        assertThat(e.getCause().getMessage(), containsString("jar hell! duplicate codebases with extended plugin"));
+    }
+
+    public void testJarHellDuplicateCodebaseAcrossDeps() throws Exception {
+        Path pluginDir = createTempDir();
+        Path pluginJar = pluginDir.resolve("plugin.jar");
+        makeJar(pluginJar, DummyClass1.class);
+        Path otherDir = createTempDir();
+        Path dupJar = otherDir.resolve("dup.jar");
+        makeJar(dupJar, DummyClass2.class);
+        Map<String, Set<URL>> transitiveDeps = new HashMap<>();
+        transitiveDeps.put("dep1", Collections.singleton(dupJar.toUri().toURL()));
+        transitiveDeps.put("dep2", Collections.singleton(dupJar.toUri().toURL()));
+        PluginInfo info1 = new PluginInfo("myplugin", "desc", "1.0", "MyPlugin", Arrays.asList("dep1", "dep2"), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info1, pluginDir);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () ->
+            PluginsService.checkBundleJarHell(bundle, transitiveDeps));
+        assertEquals("failed to load plugin myplugin due to jar hell", e.getMessage());
+        assertThat(e.getCause().getMessage(), containsString("jar hell!"));
+        assertThat(e.getCause().getMessage(), containsString("duplicate codebases"));
+    }
+
+    // Note: testing dup codebase with core is difficult because it requires a symlink, but we have mock filesystems and security manager
+
+    public void testJarHellDuplicateClassWithCore() throws Exception {
+        // need a jar file of core dep, use log4j here
+        Path pluginDir = createTempDir();
+        Path pluginJar = pluginDir.resolve("plugin.jar");
+        makeJar(pluginJar, Level.class);
+        PluginInfo info1 = new PluginInfo("myplugin", "desc", "1.0", "MyPlugin", Collections.emptyList(), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info1, pluginDir);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () ->
+            PluginsService.checkBundleJarHell(bundle, new HashMap<>()));
+        assertEquals("failed to load plugin myplugin due to jar hell", e.getMessage());
+        assertThat(e.getCause().getMessage(), containsString("jar hell!"));
+        assertThat(e.getCause().getMessage(), containsString("Level"));
+    }
+
+    public void testJarHellDuplicateClassWithDep() throws Exception {
+        Path pluginDir = createTempDir();
+        Path pluginJar = pluginDir.resolve("plugin.jar");
+        makeJar(pluginJar, DummyClass1.class);
+        Path depDir = createTempDir();
+        Path depJar = depDir.resolve("dep.jar");
+        makeJar(depJar, DummyClass1.class);
+        Map<String, Set<URL>> transitiveDeps = new HashMap<>();
+        transitiveDeps.put("dep", Collections.singleton(depJar.toUri().toURL()));
+        PluginInfo info1 = new PluginInfo("myplugin", "desc", "1.0", "MyPlugin", Collections.singletonList("dep"), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info1, pluginDir);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () ->
+            PluginsService.checkBundleJarHell(bundle, transitiveDeps));
+        assertEquals("failed to load plugin myplugin due to jar hell", e.getMessage());
+        assertThat(e.getCause().getMessage(), containsString("jar hell!"));
+        assertThat(e.getCause().getMessage(), containsString("DummyClass1"));
+    }
+
+    public void testJarHellDuplicateClassAcrossDeps() throws Exception {
+        Path pluginDir = createTempDir();
+        Path pluginJar = pluginDir.resolve("plugin.jar");
+        makeJar(pluginJar, DummyClass1.class);
+        Path dep1Dir = createTempDir();
+        Path dep1Jar = dep1Dir.resolve("dep1.jar");
+        makeJar(dep1Jar, DummyClass2.class);
+        Path dep2Dir = createTempDir();
+        Path dep2Jar = dep2Dir.resolve("dep2.jar");
+        makeJar(dep2Jar, DummyClass2.class);
+        Map<String, Set<URL>> transitiveDeps = new HashMap<>();
+        transitiveDeps.put("dep1", Collections.singleton(dep1Jar.toUri().toURL()));
+        transitiveDeps.put("dep2", Collections.singleton(dep2Jar.toUri().toURL()));
+        PluginInfo info1 = new PluginInfo("myplugin", "desc", "1.0", "MyPlugin", Arrays.asList("dep1", "dep2"), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info1, pluginDir);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () ->
+            PluginsService.checkBundleJarHell(bundle, transitiveDeps));
+        assertEquals("failed to load plugin myplugin due to jar hell", e.getMessage());
+        assertThat(e.getCause().getMessage(), containsString("jar hell!"));
+        assertThat(e.getCause().getMessage(), containsString("DummyClass2"));
+    }
+
+    public void testJarHellTransitiveMap() throws Exception {
+        Path pluginDir = createTempDir();
+        Path pluginJar = pluginDir.resolve("plugin.jar");
+        makeJar(pluginJar, DummyClass1.class);
+        Path dep1Dir = createTempDir();
+        Path dep1Jar = dep1Dir.resolve("dep1.jar");
+        makeJar(dep1Jar, DummyClass2.class);
+        Path dep2Dir = createTempDir();
+        Path dep2Jar = dep2Dir.resolve("dep2.jar");
+        makeJar(dep2Jar, DummyClass3.class);
+        Map<String, Set<URL>> transitiveDeps = new HashMap<>();
+        transitiveDeps.put("dep1", Collections.singleton(dep1Jar.toUri().toURL()));
+        transitiveDeps.put("dep2", Collections.singleton(dep2Jar.toUri().toURL()));
+        PluginInfo info1 = new PluginInfo("myplugin", "desc", "1.0", "MyPlugin", Arrays.asList("dep1", "dep2"), false, false);
+        PluginsService.Bundle bundle = new PluginsService.Bundle(info1, pluginDir);
+        PluginsService.checkBundleJarHell(bundle, transitiveDeps);
+        Set<URL> deps = transitiveDeps.get("myplugin");
+        assertNotNull(deps);
+        assertThat(deps, containsInAnyOrder(pluginJar.toUri().toURL(), dep1Jar.toUri().toURL(), dep2Jar.toUri().toURL()));
+    }
+
+    public void testNonExtensibleDep() throws Exception {
+        Path homeDir = createTempDir();
+        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), homeDir).build();
+        Path pluginsDir = homeDir.resolve("plugins");
+        Path mypluginDir = pluginsDir.resolve("myplugin");
+        PluginTestUtil.writeProperties(
+            mypluginDir,
+            "description", "whatever",
+            "name", "myplugin",
+            "version", "1.0.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "extended.plugins", "nonextensible",
+            "classname", "test.DummyPlugin");
+        try (InputStream jar = PluginsServiceTests.class.getResourceAsStream("dummy-plugin.jar")) {
+            Files.copy(jar, mypluginDir.resolve("plugin.jar"));
+        }
+        Path nonextensibleDir = pluginsDir.resolve("nonextensible");
+        PluginTestUtil.writeProperties(
+            nonextensibleDir,
+            "description", "whatever",
+            "name", "nonextensible",
+            "version", "1.0.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "test.NonExtensiblePlugin");
+        try (InputStream jar = PluginsServiceTests.class.getResourceAsStream("non-extensible-plugin.jar")) {
+            Files.copy(jar, nonextensibleDir.resolve("plugin.jar"));
+        }
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> newPluginsService(settings));
+        assertEquals("Plugin [myplugin] cannot extend non-extensible plugin [nonextensible]", e.getMessage());
+    }
 }

distribution/build.gradle

@@ -175,6 +175,7 @@ configure(distributions) {
       into 'lib'
       from project(':core').jar
       from project(':core').configurations.runtime
+      from { project(':libs:plugin-classloader').jar }
       // delay add tools using closures, since they have not yet been configured, so no jar task exists yet
       from { project(':distribution:tools:launchers').jar }
       from { project(':distribution:tools:plugin-cli').jar }

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -33,7 +33,6 @@ import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.env.Environment;
 
@@ -63,9 +62,11 @@ import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
@@ -555,7 +556,7 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         }
 
         // check for jar hell before any copying
-        jarHellCheck(pluginRoot, env.pluginsFile());
+        jarHellCheck(info, pluginRoot, env.pluginsFile(), env.modulesFile());
 
         // read optional security policy (extra permissions)
         // if it exists, confirm or warn the user
@@ -568,25 +569,25 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
     }
 
     /** check a candidate plugin for jar hell before installing it */
-    void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+    void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
         // create list of current jars in classpath
         final Set<URL> jars = new HashSet<>(JarHell.parseClassPath());
 
         // read existing bundles. this does some checks on the installation too.
-        PluginsService.getPluginBundles(pluginsDir);
+        Set<PluginsService.Bundle> bundles = new HashSet<>(PluginsService.getPluginBundles(pluginsDir));
+        bundles.addAll(PluginsService.getModuleBundles(modulesDir));
+        bundles.add(new PluginsService.Bundle(info, candidate));
+        List<PluginsService.Bundle> sortedBundles = PluginsService.sortBundles(bundles);
 
-        // add plugin jars to the list
+        // check jarhell of all plugins so we know this plugin and anything depending on it are ok together
-        Path pluginJars[] = FileSystemUtils.files(candidate, "*.jar");
+        // TODO: optimize to skip any bundles not connected to the candidate plugin?
-        for (Path jar : pluginJars) {
+        Map<String, Set<URL>> transitiveUrls = new HashMap<>();
-            if (jars.add(jar.toUri().toURL()) == false) {
+        for (PluginsService.Bundle bundle : sortedBundles) {
-                throw new IllegalStateException("jar hell! duplicate plugin jar: " + jar);
+            PluginsService.checkBundleJarHell(bundle, transitiveUrls);
-            }
         }
+
         // TODO: no jars should be an error
         // TODO: verify the classname exists in one of the jars!
-
-        // check combined (current classpath + new jars to-be-added)
-        JarHell.checkJarHell(jars);
     }
 
     /**

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java

@@ -36,6 +36,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -46,6 +47,10 @@ import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
  */
 class RemovePluginCommand extends EnvironmentAwareCommand {
 
+    // exit codes for remove
+    /** A plugin cannot be removed because it is extended by another plugin. */
+    static final int PLUGIN_STILL_USED = 11;
+
     private final OptionSpec<Void> purgeOption;
     private final OptionSpec<String> arguments;
 
@@ -74,20 +79,31 @@ class RemovePluginCommand extends EnvironmentAwareCommand {
      * @throws UserException if plugin directory does not exist
      * @throws UserException if the plugin bin directory is not a directory
      */
-    void execute(
+    void execute(Terminal terminal, Environment env, String pluginName, boolean purge) throws IOException, UserException {
-            final Terminal terminal,
-            final Environment env,
-            final String pluginName,
-            final boolean purge) throws IOException, UserException {
         if (pluginName == null) {
             throw new UserException(ExitCodes.USAGE, "plugin name is required");
         }
 
-        terminal.println("-> removing [" + pluginName + "]...");
+        // first make sure nothing extends this plugin
+        List<String> usedBy = new ArrayList<>();
+        Set<PluginsService.Bundle> bundles = PluginsService.getPluginBundles(env.pluginsFile());
+        for (PluginsService.Bundle bundle : bundles) {
+            for (String extendedPlugin : bundle.plugin.getExtendedPlugins()) {
+                if (extendedPlugin.equals(pluginName)) {
+                    usedBy.add(bundle.plugin.getName());
+                }
+            }
+        }
+        if (usedBy.isEmpty() == false) {
+            throw new UserException(PLUGIN_STILL_USED, "plugin [" + pluginName + "] cannot be removed" +
+                " because it is extended by other plugins: " + usedBy);
+        }
 
         final Path pluginDir = env.pluginsFile().resolve(pluginName);
         final Path pluginConfigDir = env.configFile().resolve(pluginName);
         final Path removing = env.pluginsFile().resolve(".removing-" + pluginName);
+
+        terminal.println("-> removing [" + pluginName + "]...");
         /*
          * If the plugin does not exist and the plugin config does not exist, fail to the user that the plugin is not found, unless there's
          * a marker file left from a previously failed attempt in which case we proceed to clean up the marker file. Or, if the plugin does

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -115,7 +115,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         super.setUp();
         skipJarHellCommand = new InstallPluginCommand() {
             @Override
-            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+            void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
                 // no jarhell check
             }
         };
@@ -791,7 +791,7 @@ public class InstallPluginCommandTests extends ESTestCase {
                 return stagingHash;
             }
             @Override
-            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+            void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
                 // no jarhell check
             }
         };

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java

@@ -154,6 +154,7 @@ public class ListPluginsCommandTests extends ESTestCase {
                         "Version: 1.0",
                         "Native Controller: false",
                         "Requires Keystore: false",
+                        "Extended Plugins: []",
                         " * Classname: org.fake"),
                 terminal.getOutput());
     }
@@ -172,6 +173,7 @@ public class ListPluginsCommandTests extends ESTestCase {
                 "Version: 1.0",
                 "Native Controller: true",
                 "Requires Keystore: false",
+                "Extended Plugins: []",
                 " * Classname: org.fake"),
             terminal.getOutput());
     }
@@ -190,6 +192,7 @@ public class ListPluginsCommandTests extends ESTestCase {
                 "Version: 1.0",
                 "Native Controller: false",
                 "Requires Keystore: true",
+                "Extended Plugins: []",
                 " * Classname: org.fake"),
             terminal.getOutput());
     }
@@ -209,6 +212,7 @@ public class ListPluginsCommandTests extends ESTestCase {
                         "Version: 1.0",
                         "Native Controller: false",
                         "Requires Keystore: false",
+                        "Extended Plugins: []",
                         " * Classname: org.fake",
                         "fake_plugin2",
                         "- Plugin information:",
@@ -217,6 +221,7 @@ public class ListPluginsCommandTests extends ESTestCase {
                         "Version: 1.0",
                         "Native Controller: false",
                         "Requires Keystore: false",
+                        "Extended Plugins: []",
                         " * Classname: org.fake2"),
                 terminal.getOutput());
     }

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.plugins;
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cli.Terminal;
@@ -77,6 +78,17 @@ public class RemovePluginCommandTests extends ESTestCase {
         env = TestEnvironment.newEnvironment(settings);
     }
 
+    void createPlugin(String name) throws Exception {
+        PluginTestUtil.writeProperties(
+            env.pluginsFile().resolve(name),
+            "description", "dummy",
+            "name", name,
+            "version", "1.0",
+            "elasticsearch.version", Version.CURRENT.toString(),
+            "java.version", System.getProperty("java.specification.version"),
+            "classname", "SomeClass");
+    }
+
     static MockTerminal removePlugin(String name, Path home, boolean purge) throws Exception {
         Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", home).build());
         MockTerminal terminal = new MockTerminal();
@@ -101,10 +113,10 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testBasic() throws Exception {
-        Files.createDirectory(env.pluginsFile().resolve("fake"));
+        createPlugin("fake");
         Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
         Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir"));
-        Files.createDirectory(env.pluginsFile().resolve("other"));
+        createPlugin("other");
         removePlugin("fake", home, randomBoolean());
         assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
         assertTrue(Files.exists(env.pluginsFile().resolve("other")));
@@ -112,7 +124,7 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testBin() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("fake"));
+        createPlugin("fake");
         Path binDir = env.binFile().resolve("fake");
         Files.createDirectories(binDir);
         Files.createFile(binDir.resolve("somescript"));
@@ -124,16 +136,17 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testBinNotDir() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("elasticsearch"));
+        createPlugin("fake");
-        UserException e = expectThrows(UserException.class, () -> removePlugin("elasticsearch", home, randomBoolean()));
+        Files.createFile(env.binFile().resolve("fake"));
+        UserException e = expectThrows(UserException.class, () -> removePlugin("fake", home, randomBoolean()));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
-        assertTrue(Files.exists(env.pluginsFile().resolve("elasticsearch"))); // did not remove
+        assertTrue(Files.exists(env.pluginsFile().resolve("fake"))); // did not remove
-        assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));
+        assertTrue(Files.exists(env.binFile().resolve("fake")));
         assertRemoveCleaned(env);
     }
 
     public void testConfigDirPreserved() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("fake"));
+        createPlugin("fake");
         final Path configDir = env.configFile().resolve("fake");
         Files.createDirectories(configDir);
         Files.createFile(configDir.resolve("fake.yml"));
@@ -144,7 +157,7 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testPurgePluginExists() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("fake"));
+        createPlugin("fake");
         final Path configDir = env.configFile().resolve("fake");
         if (randomBoolean()) {
             Files.createDirectories(configDir);
@@ -181,7 +194,7 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testNoConfigDirPreserved() throws Exception {
-        Files.createDirectories(env.pluginsFile().resolve("fake"));
+        createPlugin("fake");
         final Path configDir = env.configFile().resolve("fake");
         final MockTerminal terminal = removePlugin("fake", home, randomBoolean());
         assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
@@ -214,7 +227,7 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testRemoveWhenRemovingMarker() throws Exception {
-        Files.createDirectory(env.pluginsFile().resolve("fake"));
+        createPlugin("fake");
         Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
         Files.createFile(env.pluginsFile().resolve(".removing-fake"));
         removePlugin("fake", home, randomBoolean());

libs/plugin-classloader/build.gradle

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.build'
+
+test.enabled = false
+
+// test depend on ES core...
+forbiddenApisMain.enabled = false
+jarHell.enabled = false

libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/ExtendedPluginsClassLoader.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A classloader that is a union over the parent core classloader and classloaders of extended plugins.
+ */
+public class ExtendedPluginsClassLoader extends ClassLoader {
+
+    /** Loaders of plugins extended by a plugin. */
+    private final List<ClassLoader> extendedLoaders;
+
+    private ExtendedPluginsClassLoader(ClassLoader parent, List<ClassLoader> extendedLoaders) {
+        super(parent);
+        this.extendedLoaders = Collections.unmodifiableList(extendedLoaders);
+    }
+
+    @Override
+    protected Class<?> findClass(String name) throws ClassNotFoundException {
+        for (ClassLoader loader : extendedLoaders) {
+            try {
+                return loader.loadClass(name);
+            } catch (ClassNotFoundException e) {
+                // continue
+            }
+        }
+        throw new ClassNotFoundException(name);
+    }
+
+    /**
+     * Return a new classloader across the parent and extended loaders.
+     */
+    public static ExtendedPluginsClassLoader create(ClassLoader parent, List<ClassLoader> extendedLoaders) {
+        return AccessController.doPrivileged((PrivilegedAction<ExtendedPluginsClassLoader>)
+            () -> new ExtendedPluginsClassLoader(parent, extendedLoaders));
+    }
+}

modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java

@@ -22,6 +22,7 @@ package org.elasticsearch.painless;
 
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptContext;
@@ -34,7 +35,7 @@ import java.util.List;
 /**
  * Registers Painless as a plugin.
  */
-public final class PainlessPlugin extends Plugin implements ScriptPlugin {
+public final class PainlessPlugin extends Plugin implements ScriptPlugin, ExtensiblePlugin {
 
     // force to parse our definition at startup (not on the user's first script)
     static {

plugins/examples/painless-whitelist/build.gradle

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'painless-whitelist'
+  description 'An example whitelisting additional classes and methods in painless'
+  classname 'org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin'
+  extendedPlugins = ['lang-painless']
+}
+
+integTestCluster {
+  distribution = 'zip'
+}
+
+test.enabled = false

plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/MyWhitelistPlugin.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.painlesswhitelist;
+
+import org.elasticsearch.plugins.Plugin;
+
+public class MyWhitelistPlugin extends Plugin {
+}

plugins/examples/painless-whitelist/src/test/java/org/elasticsearch/example/painlesswhitelist/PainlessWhitelistClientYamlTestSuiteIT.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.painlesswhitelist;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class PainlessWhitelistClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public PainlessWhitelistClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}
+

plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/10_basic.yml

@@ -0,0 +1,13 @@
+# Integration tests for the painless whitelist example plugin
+#
+"Plugin loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.plugins.0.name: painless-whitelist  }

qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java

@@ -52,7 +52,7 @@ public class ESPolicyUnitTests extends ESTestCase {
         Permission all = new AllPermission();
         PermissionCollection allCollection = all.newPermissionCollection();
         allCollection.add(all);
-        ESPolicy policy = new ESPolicy(allCollection, Collections.emptyMap(), true);
+        ESPolicy policy = new ESPolicy(Collections.emptyMap(), allCollection, Collections.emptyMap(), true);
         // restrict ourselves to NoPermission
         PermissionCollection noPermissions = new Permissions();
         assertFalse(policy.implies(new ProtectionDomain(null, noPermissions), new FilePermission("foo", "read")));
@@ -67,7 +67,7 @@ public class ESPolicyUnitTests extends ESTestCase {
     public void testNullLocation() throws Exception {
         assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
         PermissionCollection noPermissions = new Permissions();
-        ESPolicy policy = new ESPolicy(noPermissions, Collections.emptyMap(), true);
+        ESPolicy policy = new ESPolicy(Collections.emptyMap(), noPermissions, Collections.emptyMap(), true);
         assertFalse(policy.implies(new ProtectionDomain(new CodeSource(null, (Certificate[]) null), noPermissions),
                 new FilePermission("foo", "read")));
     }
@@ -75,7 +75,7 @@ public class ESPolicyUnitTests extends ESTestCase {
     public void testListen() {
         assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
         final PermissionCollection noPermissions = new Permissions();
-        final ESPolicy policy = new ESPolicy(noPermissions, Collections.emptyMap(), true);
+        final ESPolicy policy = new ESPolicy(Collections.emptyMap(), noPermissions, Collections.emptyMap(), true);
         assertFalse(
             policy.implies(
                 new ProtectionDomain(ESPolicyUnitTests.class.getProtectionDomain().getCodeSource(), noPermissions),

qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java

@@ -48,7 +48,7 @@ public class PluginSecurityTests extends ESTestCase {
                 "test cannot run with security manager enabled",
                 System.getSecurityManager() == null);
         final PluginInfo info =
-                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", true, false);
+                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", Collections.emptyList(), true, false);
         final MockTerminal terminal = new MockTerminal();
         terminal.addTextInput("y");
         terminal.addTextInput("y");
@@ -63,7 +63,7 @@ public class PluginSecurityTests extends ESTestCase {
                 "test cannot run with security manager enabled",
                 System.getSecurityManager() == null);
         final PluginInfo info =
-                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", true, false);
+                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", Collections.emptyList(), true, false);
         final MockTerminal terminal = new MockTerminal();
         terminal.addTextInput("y");
         terminal.addTextInput("n");
@@ -79,7 +79,7 @@ public class PluginSecurityTests extends ESTestCase {
                 "test cannot run with security manager enabled",
                 System.getSecurityManager() == null);
         final PluginInfo info =
-                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", false, false);
+                new PluginInfo("fake", "fake", Version.CURRENT.toString(), "Fake", Collections.emptyList(), false, false);
         final MockTerminal terminal = new MockTerminal();
         terminal.addTextInput("y");
         final Path policyFile = this.getDataPath("security/simple-plugin-security.policy");

qa/mixed-cluster/build.gradle

@@ -65,9 +65,11 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr
 
 // basic integ tests includes testing bwc against the most recent version
 task integTest {
+  if (project.bwc_tests_enabled) {
     for (final def version : versionCollection.basicIntegrationTestVersions) {
       dependsOn "v${version}#bwcTest"
     }
   }
+}
 
 check.dependsOn(integTest)

settings.gradle

@@ -98,6 +98,14 @@ for (File example : examplePluginsDir.listFiles()) {
   examplePlugins.add(example.name)
 }
 
+projects.add("libs")
+File libsDir = new File(rootProject.projectDir, 'libs')
+for (File libDir : new File(rootProject.projectDir, 'libs').listFiles()) {
+  if (libDir.isDirectory() == false) continue;
+  if (libDir.name.startsWith('build') || libDir.name.startsWith('.')) continue;
+  projects.add("libs:${libDir.name}".toString())
+}
+
 /* Create projects for building BWC snapshot distributions from the heads of other branches */
 final List<String> branches = ['5.6', '6.0', '6.1', '6.x']
 for (final String branch : branches) {

test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

@@ -131,8 +131,13 @@ public class BootstrapForTesting {
                 perms.add(new SocketPermission("localhost:1024-", "listen,resolve"));
 
                 // read test-framework permissions
-                final Policy testFramework = Security.readPolicy(Bootstrap.class.getResource("test-framework.policy"), JarHell.parseClassPath());
+                Map<String, URL> codebases = Security.getCodebaseJarMap(JarHell.parseClassPath());
-                final Policy esPolicy = new ESPolicy(perms, getPluginPermissions(), true);
+                if (System.getProperty("tests.gradle") == null) {
+                    // intellij and eclipse don't package our internal libs, so we need to set the codebases for them manually
+                    addClassCodebase(codebases,"plugin-classloader", "org.elasticsearch.plugins.ExtendedPluginsClassLoader");
+                }
+                final Policy testFramework = Security.readPolicy(Bootstrap.class.getResource("test-framework.policy"), codebases);
+                final Policy esPolicy = new ESPolicy(codebases, perms, getPluginPermissions(), true);
                 Policy.setPolicy(new Policy() {
                     @Override
                     public boolean implies(ProtectionDomain domain, Permission permission) {
@@ -161,6 +166,19 @@ public class BootstrapForTesting {
         }
     }
 
+    /** Add the codebase url of the given classname to the codebases map, if the class exists. */
+    private static void addClassCodebase(Map<String, URL> codebases, String name, String classname) {
+        try {
+            Class clazz = BootstrapForTesting.class.getClassLoader().loadClass(classname);
+            if (codebases.put(name, clazz.getProtectionDomain().getCodeSource().getLocation()) != null) {
+                throw new IllegalStateException("Already added " + name + " codebase for testing");
+            }
+        } catch (ClassNotFoundException e) {
+            // no class, fall through to not add. this can happen for any tests that do not include
+            // the given class. eg only core tests include plugin-classloader
+        }
+    }
+
     /**
      * we don't know which codesources belong to which plugin, so just remove the permission from key codebases
      * like core, test-framework, etc. this way tests fail if accesscontroller blocks are missing.
@@ -191,7 +209,7 @@ public class BootstrapForTesting {
         // parse each policy file, with codebase substitution from the classpath
         final List<Policy> policies = new ArrayList<>(pluginPolicies.size());
         for (URL policyFile : pluginPolicies) {
-            policies.add(Security.readPolicy(policyFile, codebases));
+            policies.add(Security.readPolicy(policyFile, Security.getCodebaseJarMap(codebases)));
         }
 
         // consult each policy file for those codebases