From f9598030fdbcf5879c3563ce19fe2d2cf55365ff Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 29 Jun 2015 12:11:04 -0400 Subject: [PATCH] Load plugins into classpath in bootstrap --- .../elasticsearch/bootstrap/Bootstrap.java | 94 +++++++++++++ .../org/elasticsearch/bootstrap/Security.java | 3 - .../elasticsearch/plugins/PluginsService.java | 70 ---------- .../elasticsearch/bootstrap/security.policy | 2 - .../bootstrap/BootstrapTests.java | 47 +++++++ .../plugins/PluginLuceneCheckerTests.java | 85 ------------ .../plugins/PluginServiceTests.java | 123 ------------------ 7 files changed, 141 insertions(+), 283 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java delete mode 100644 core/src/test/java/org/elasticsearch/plugins/PluginLuceneCheckerTests.java delete mode 100644 core/src/test/java/org/elasticsearch/plugins/PluginServiceTests.java diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index 912d0800533..14042c5ecd5 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -19,6 +19,9 @@ package org.elasticsearch.bootstrap; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; + import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Constants; import org.elasticsearch.ExceptionsHelper; @@ -29,6 +32,7 @@ import org.elasticsearch.common.cli.Terminal; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.CreationException; import org.elasticsearch.common.inject.spi.Message; +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; @@ -42,10 +46,20 @@ import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.hyperic.sigar.Sigar; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URL; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import java.util.Set; import java.util.concurrent.CountDownLatch; +import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; import static com.google.common.collect.Sets.newHashSet; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; @@ -162,6 +176,9 @@ public class Bootstrap { }); } + // install any plugins into classpath + setupPlugins(environment); + // install SM after natives, shutdown hooks, etc. setupSecurity(settings, environment); @@ -348,4 +365,81 @@ public class Bootstrap { } return errorMessage.toString(); } + + static final String PLUGIN_LIB_PATTERN = "glob:**.{jar,zip}"; + private static void setupPlugins(Environment environment) throws IOException { + ESLogger logger = Loggers.getLogger(Bootstrap.class); + + Path pluginsDirectory = environment.pluginsFile(); + if (!isAccessibleDirectory(pluginsDirectory, logger)) { + return; + } + + ClassLoader classLoader = ClassLoader.getSystemClassLoader(); + Class classLoaderClass = classLoader.getClass(); + Method addURL = null; + try { + while (!classLoaderClass.equals(Object.class)) { + try { + addURL = classLoaderClass.getDeclaredMethod("addURL", URL.class); + addURL.setAccessible(true); + break; + } catch (NoSuchMethodException e) { + // no method, try the parent + classLoaderClass = classLoaderClass.getSuperclass(); + } + } + + if (addURL == null) { + logger.debug("failed to find addURL method on classLoader [" + classLoader + "] to add methods"); + return; + } + + try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { + + for (Path plugin : stream) { + // We check that subdirs are directories and readable + if (!isAccessibleDirectory(plugin, logger)) { + continue; + } + + logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); + + try { + // add the root + addURL.invoke(classLoader, plugin.toUri().toURL()); + // gather files to add + List libFiles = Lists.newArrayList(); + libFiles.addAll(Arrays.asList(files(plugin))); + Path libLocation = plugin.resolve("lib"); + if (Files.isDirectory(libLocation)) { + libFiles.addAll(Arrays.asList(files(libLocation))); + } + + PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(PLUGIN_LIB_PATTERN); + + // if there are jars in it, add it as well + for (Path libFile : libFiles) { + if (!matcher.matches(libFile)) { + continue; + } + addURL.invoke(classLoader, libFile.toUri().toURL()); + } + } catch (Throwable e) { + logger.warn("failed to add plugin [" + plugin + "]", e); + } + } + } + } finally { + if (addURL != null) { + addURL.setAccessible(false); + } + } + } + + private static Path[] files(Path from) throws IOException { + try (DirectoryStream stream = Files.newDirectoryStream(from)) { + return Iterators.toArray(stream.iterator(), Path.class); + } + } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java index 194c5693982..496ff7b1b6a 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -87,9 +87,6 @@ final class Security { for (Map.Entry e : SPECIAL_JARS.entrySet()) { if (e.getKey().matcher(url.getPath()).matches()) { String prop = e.getValue(); - if (System.getProperty(prop) != null) { - throw new IllegalStateException("property: " + prop + " is unexpectedly set"); - } System.setProperty(prop, url.toString()); } } diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index 5929de5f092..e9110da424b 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -61,7 +61,6 @@ public class PluginsService extends AbstractComponent { public static final String ES_PLUGIN_PROPERTIES = "es-plugin.properties"; public static final String LOAD_PLUGIN_FROM_CLASSPATH = "plugins.load_classpath_plugins"; - static final String PLUGIN_LIB_PATTERN = "glob:**.{jar,zip}"; public static final String PLUGINS_CHECK_LUCENE_KEY = "plugins.check_lucene"; public static final String PLUGINS_INFO_REFRESH_INTERVAL_KEY = "plugins.info_refresh_interval"; @@ -118,11 +117,6 @@ public class PluginsService extends AbstractComponent { } // now, find all the ones that are in the classpath - try { - loadPluginsIntoClassLoader(); - } catch (IOException ex) { - throw new IllegalStateException("Can't load plugins into classloader", ex); - } if (loadClasspathPlugins) { tupleBuilder.addAll(loadPluginsFromClasspath(settings)); } @@ -349,71 +343,7 @@ public class PluginsService extends AbstractComponent { return cachedPluginsInfo; } - private void loadPluginsIntoClassLoader() throws IOException { - Path pluginsDirectory = environment.pluginsFile(); - if (!isAccessibleDirectory(pluginsDirectory, logger)) { - return; - } - ClassLoader classLoader = settings.getClassLoader(); - Class classLoaderClass = classLoader.getClass(); - Method addURL = null; - while (!classLoaderClass.equals(Object.class)) { - try { - addURL = classLoaderClass.getDeclaredMethod("addURL", URL.class); - addURL.setAccessible(true); - break; - } catch (NoSuchMethodException e) { - // no method, try the parent - classLoaderClass = classLoaderClass.getSuperclass(); - } - } - if (addURL == null) { - logger.debug("failed to find addURL method on classLoader [" + classLoader + "] to add methods"); - return; - } - try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { - - for (Path plugin : stream) { - // We check that subdirs are directories and readable - if (!isAccessibleDirectory(plugin, logger)) { - continue; - } - - logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); - - try { - // add the root - addURL.invoke(classLoader, plugin.toUri().toURL()); - // gather files to add - List libFiles = Lists.newArrayList(); - libFiles.addAll(Arrays.asList(files(plugin))); - Path libLocation = plugin.resolve("lib"); - if (Files.isDirectory(libLocation)) { - libFiles.addAll(Arrays.asList(files(libLocation))); - } - - PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(PLUGIN_LIB_PATTERN); - - // if there are jars in it, add it as well - for (Path libFile : libFiles) { - if (!matcher.matches(libFile)) { - continue; - } - addURL.invoke(classLoader, libFile.toUri().toURL()); - } - } catch (Throwable e) { - logger.warn("failed to add plugin [" + plugin + "]", e); - } - } - } - } - - private Path[] files(Path from) throws IOException { - try (DirectoryStream stream = Files.newDirectoryStream(from)) { - return Iterators.toArray(stream.iterator(), Path.class); - } - } private List> loadPluginsFromClasspath(Settings settings) { ImmutableList.Builder> plugins = ImmutableList.builder(); diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index 260ca5edafe..0434bf5be31 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -86,8 +86,6 @@ grant { permission java.lang.RuntimePermission "getProtectionDomain"; // reflection hacks: - // needed by pluginmanager to CHANGE THE CLASSLOADER (!) - permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; // needed for mock filesystems in tests (to capture implCloseChannel) permission java.lang.RuntimePermission "accessClassInPackage.sun.nio.ch"; // needed by groovy engine diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java new file mode 100644 index 00000000000..c07995e8d35 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import java.nio.file.Path; +import java.nio.file.PathMatcher; + +public class BootstrapTests extends ElasticsearchTestCase { + + @Test + public void testHasLibExtension() { + PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(Bootstrap.PLUGIN_LIB_PATTERN); + + Path p = PathUtils.get("path", "to", "plugin.jar"); + assertTrue(matcher.matches(p)); + + p = PathUtils.get("path", "to", "plugin.zip"); + assertTrue(matcher.matches(p)); + + p = PathUtils.get("path", "to", "plugin.tar.gz"); + assertFalse(matcher.matches(p)); + + p = PathUtils.get("path", "to", "plugin"); + assertFalse(matcher.matches(p)); + } +} diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginLuceneCheckerTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginLuceneCheckerTests.java deleted file mode 100644 index 1a44b1192eb..00000000000 --- a/core/src/test/java/org/elasticsearch/plugins/PluginLuceneCheckerTests.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.plugins; - -import com.google.common.collect.Lists; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.junit.Test; - -import java.net.URISyntaxException; -import java.util.Collections; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; - -/** - * - */ -@ClusterScope(scope= ElasticsearchIntegrationTest.Scope.TEST, numDataNodes=0, transportClientRatio = 0) -public class PluginLuceneCheckerTests extends PluginTestCase { - - /** - * We check that no Lucene version checking is done - * when we set `"plugins.check_lucene":false` - */ - @Test - public void testDisableLuceneVersionCheckingPlugin() throws URISyntaxException { - String serverNodeId = startNodeWithPlugins( - settingsBuilder().put(PluginsService.PLUGINS_CHECK_LUCENE_KEY, false) - .put(PluginsService.ES_PLUGIN_PROPERTIES_FILE_KEY, "es-plugin-test.properties") - .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true).build(), - "/org/elasticsearch/plugins/lucene/"); - logger.info("--> server {} started" + serverNodeId); - - NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setPlugins(true).execute().actionGet(); - logger.info("--> full json answer, status " + response.toString()); - - ElasticsearchAssertions.assertNodeContainsPlugins(response, serverNodeId, - Lists.newArrayList("old-lucene"), Lists.newArrayList("old"), Lists.newArrayList("1.0.0"), // JVM Plugin - Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST);// No Site Plugin - } - - /** - * We check that with an old plugin (built on an old Lucene version) - * plugin is not loaded - * We check that with a recent plugin (built on current Lucene version) - * plugin is loaded - * We check that with a too recent plugin (built on an unknown Lucene version) - * plugin is not loaded - */ - @Test - public void testEnableLuceneVersionCheckingPlugin() throws URISyntaxException { - String serverNodeId = startNodeWithPlugins( - settingsBuilder().put(PluginsService.PLUGINS_CHECK_LUCENE_KEY, true) - .put(PluginsService.ES_PLUGIN_PROPERTIES_FILE_KEY, "es-plugin-test.properties") - .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true).build(), - "/org/elasticsearch/plugins/lucene/"); - logger.info("--> server {} started" + serverNodeId); - - NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setPlugins(true).execute().actionGet(); - logger.info("--> full json answer, status " + response.toString()); - - ElasticsearchAssertions.assertNodeContainsPlugins(response, serverNodeId, - Lists.newArrayList("current-lucene"), Lists.newArrayList("current"), Lists.newArrayList("2.0.0"), // JVM Plugin - Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST);// No Site Plugin - } -} diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginServiceTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginServiceTests.java deleted file mode 100644 index 18f826033c4..00000000000 --- a/core/src/test/java/org/elasticsearch/plugins/PluginServiceTests.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.plugins; - -import com.google.common.collect.ImmutableList; - -import org.elasticsearch.action.admin.cluster.node.info.PluginInfo; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.loading.classpath.InClassPathPlugin; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; -import org.junit.Test; - -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.nio.file.PathMatcher; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.instanceOf; - -@ClusterScope(scope= ElasticsearchIntegrationTest.Scope.TEST, numDataNodes=0, numClientNodes = 1, transportClientRatio = 0) -public class PluginServiceTests extends PluginTestCase { - - @Test - public void testPluginLoadingFromClassName() throws URISyntaxException { - Settings settings = settingsBuilder() - // Defines a plugin in classpath - .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) - .put(PluginsService.ES_PLUGIN_PROPERTIES_FILE_KEY, "es-plugin-test.properties") - // Defines a plugin in settings - .put("plugin.types", InSettingsPlugin.class.getName()) - .build(); - - startNodeWithPlugins(settings, "/org/elasticsearch/plugins/loading/"); - - Plugin plugin = getPlugin("in-settings-plugin"); - assertNotNull("InSettingsPlugin (defined below in this class) must be loaded", plugin); - assertThat(plugin, instanceOf(InSettingsPlugin.class)); - - plugin = getPlugin("in-classpath-plugin"); - assertNotNull("InClassPathPlugin (defined in package ) must be loaded", plugin); - assertThat(plugin, instanceOf(InClassPathPlugin.class)); - - plugin = getPlugin("in-jar-plugin"); - assertNotNull("InJarPlugin (packaged as a JAR file in a plugins directory) must be loaded", plugin); - assertThat(plugin.getClass().getName(), endsWith("InJarPlugin")); - - plugin = getPlugin("in-zip-plugin"); - assertNotNull("InZipPlugin (packaged as a Zipped file in a plugins directory) must be loaded", plugin); - assertThat(plugin.getClass().getName(), endsWith("InZipPlugin")); - } - - @Test - public void testHasLibExtension() { - PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(PluginsService.PLUGIN_LIB_PATTERN); - - Path p = PathUtils.get("path", "to", "plugin.jar"); - assertTrue(matcher.matches(p)); - - p = PathUtils.get("path", "to", "plugin.zip"); - assertTrue(matcher.matches(p)); - - p = PathUtils.get("path", "to", "plugin.tar.gz"); - assertFalse(matcher.matches(p)); - - p = PathUtils.get("path", "to", "plugin"); - assertFalse(matcher.matches(p)); - } - - private Plugin getPlugin(String pluginName) { - assertNotNull("cannot check plugin existence with a null plugin's name", pluginName); - PluginsService pluginsService = internalCluster().getInstance(PluginsService.class); - ImmutableList> plugins = pluginsService.plugins(); - - if ((plugins != null) && (!plugins.isEmpty())) { - for (Tuple plugin:plugins) { - if (pluginName.equals(plugin.v1().getName())) { - return plugin.v2(); - } - } - } - return null; - } - - static class InSettingsPlugin extends AbstractPlugin { - - private final Settings settings; - - public InSettingsPlugin(Settings settings) { - this.settings = settings; - } - - @Override - public String name() { - return "in-settings-plugin"; - } - - @Override - public String description() { - return "A plugin defined in settings"; - } - } -}