honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Load plugins into classpath in bootstrap

This commit is contained in:
Robert Muir 2015-06-29 12:11:04 -04:00
parent 55c33b29f6
commit f9598030fd
7 changed files with 141 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
View File

@ -19,6 +19,9 @@
package org.elasticsearch.bootstrap;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.Constants;
import org.elasticsearch.ExceptionsHelper;
@ -29,6 +32,7 @@ import org.elasticsearch.common.cli.Terminal;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.CreationException;
import org.elasticsearch.common.inject.spi.Message;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
@ -42,10 +46,20 @@ import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
import org.hyperic.sigar.Sigar;
import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URL;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
import static com.google.common.collect.Sets.newHashSet;
import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
@ -162,6 +176,9 @@ public class Bootstrap {
});
}
// install any plugins into classpath
setupPlugins(environment);
// install SM after natives, shutdown hooks, etc.
setupSecurity(settings, environment);
@ -348,4 +365,81 @@ public class Bootstrap {
}
return errorMessage.toString();
}
static final String PLUGIN_LIB_PATTERN = "glob:**.{jar,zip}";
private static void setupPlugins(Environment environment) throws IOException {
ESLogger logger = Loggers.getLogger(Bootstrap.class);
Path pluginsDirectory = environment.pluginsFile();
if (!isAccessibleDirectory(pluginsDirectory, logger)) {
return;
}
ClassLoader classLoader = ClassLoader.getSystemClassLoader();
Class<?> classLoaderClass = classLoader.getClass();
Method addURL = null;
try {
while (!classLoaderClass.equals(Object.class)) {
try {
addURL = classLoaderClass.getDeclaredMethod("addURL", URL.class);
addURL.setAccessible(true);
break;
} catch (NoSuchMethodException e) {
// no method, try the parent
classLoaderClass = classLoaderClass.getSuperclass();
}
}
if (addURL == null) {
logger.debug("failed to find addURL method on classLoader [" + classLoader + "] to add methods");
return;
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
for (Path plugin : stream) {
// We check that subdirs are directories and readable
if (!isAccessibleDirectory(plugin, logger)) {
continue;
}
logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
try {
// add the root
addURL.invoke(classLoader, plugin.toUri().toURL());
// gather files to add
List<Path> libFiles = Lists.newArrayList();
libFiles.addAll(Arrays.asList(files(plugin)));
Path libLocation = plugin.resolve("lib");
if (Files.isDirectory(libLocation)) {
libFiles.addAll(Arrays.asList(files(libLocation)));
}
PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(PLUGIN_LIB_PATTERN);
// if there are jars in it, add it as well
for (Path libFile : libFiles) {
if (!matcher.matches(libFile)) {
continue;
}
addURL.invoke(classLoader, libFile.toUri().toURL());
}
} catch (Throwable e) {
logger.warn("failed to add plugin [" + plugin + "]", e);
}
}
}
} finally {
if (addURL != null) {
addURL.setAccessible(false);
}
}
}
private static Path[] files(Path from) throws IOException {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(from)) {
return Iterators.toArray(stream.iterator(), Path.class);
}
}
}

3
core/src/main/java/org/elasticsearch/bootstrap/Security.java
View File

@ -87,9 +87,6 @@ final class Security {
for (Map.Entry<Pattern,String> e : SPECIAL_JARS.entrySet()) {
if (e.getKey().matcher(url.getPath()).matches()) {
String prop = e.getValue();
if (System.getProperty(prop) != null) {
throw new IllegalStateException("property: " + prop + " is unexpectedly set");
}
System.setProperty(prop, url.toString());
}
}

70
core/src/main/java/org/elasticsearch/plugins/PluginsService.java
View File

@ -61,7 +61,6 @@ public class PluginsService extends AbstractComponent {
public static final String ES_PLUGIN_PROPERTIES = "es-plugin.properties";
public static final String LOAD_PLUGIN_FROM_CLASSPATH = "plugins.load_classpath_plugins";
static final String PLUGIN_LIB_PATTERN = "glob:**.{jar,zip}";
public static final String PLUGINS_CHECK_LUCENE_KEY = "plugins.check_lucene";
public static final String PLUGINS_INFO_REFRESH_INTERVAL_KEY = "plugins.info_refresh_interval";
@ -118,11 +117,6 @@ public class PluginsService extends AbstractComponent {
}
// now, find all the ones that are in the classpath
try {
loadPluginsIntoClassLoader();
} catch (IOException ex) {
throw new IllegalStateException("Can't load plugins into classloader", ex);
}
if (loadClasspathPlugins) {
tupleBuilder.addAll(loadPluginsFromClasspath(settings));
}
@ -349,71 +343,7 @@ public class PluginsService extends AbstractComponent {
return cachedPluginsInfo;
}
private void loadPluginsIntoClassLoader() throws IOException {
Path pluginsDirectory = environment.pluginsFile();
if (!isAccessibleDirectory(pluginsDirectory, logger)) {
return;
}
ClassLoader classLoader = settings.getClassLoader();
Class classLoaderClass = classLoader.getClass();
Method addURL = null;
while (!classLoaderClass.equals(Object.class)) {
try {
addURL = classLoaderClass.getDeclaredMethod("addURL", URL.class);
addURL.setAccessible(true);
break;
} catch (NoSuchMethodException e) {
// no method, try the parent
classLoaderClass = classLoaderClass.getSuperclass();
}
}
if (addURL == null) {
logger.debug("failed to find addURL method on classLoader [" + classLoader + "] to add methods");
return;
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
for (Path plugin : stream) {
// We check that subdirs are directories and readable
if (!isAccessibleDirectory(plugin, logger)) {
continue;
}
logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
try {
// add the root
addURL.invoke(classLoader, plugin.toUri().toURL());
// gather files to add
List<Path> libFiles = Lists.newArrayList();
libFiles.addAll(Arrays.asList(files(plugin)));
Path libLocation = plugin.resolve("lib");
if (Files.isDirectory(libLocation)) {
libFiles.addAll(Arrays.asList(files(libLocation)));
}
PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(PLUGIN_LIB_PATTERN);
// if there are jars in it, add it as well
for (Path libFile : libFiles) {
if (!matcher.matches(libFile)) {
continue;
}
addURL.invoke(classLoader, libFile.toUri().toURL());
}
} catch (Throwable e) {
logger.warn("failed to add plugin [" + plugin + "]", e);
}
}
}
}
private Path[] files(Path from) throws IOException {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(from)) {
return Iterators.toArray(stream.iterator(), Path.class);
}
}
private List<Tuple<PluginInfo,Plugin>> loadPluginsFromClasspath(Settings settings) {
ImmutableList.Builder<Tuple<PluginInfo, Plugin>> plugins = ImmutableList.builder();

2
core/src/main/resources/org/elasticsearch/bootstrap/security.policy
View File

@ -86,8 +86,6 @@ grant {
permission java.lang.RuntimePermission "getProtectionDomain";
// reflection hacks:
// needed by pluginmanager to CHANGE THE CLASSLOADER (!)
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
// needed for mock filesystems in tests (to capture implCloseChannel)
permission java.lang.RuntimePermission "accessClassInPackage.sun.nio.ch";
// needed by groovy engine

47
core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.bootstrap;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
public class BootstrapTests extends ElasticsearchTestCase {
@Test
public void testHasLibExtension() {
PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(Bootstrap.PLUGIN_LIB_PATTERN);
Path p = PathUtils.get("path", "to", "plugin.jar");
assertTrue(matcher.matches(p));
p = PathUtils.get("path", "to", "plugin.zip");
assertTrue(matcher.matches(p));
p = PathUtils.get("path", "to", "plugin.tar.gz");
assertFalse(matcher.matches(p));
p = PathUtils.get("path", "to", "plugin");
assertFalse(matcher.matches(p));
}
}

85
core/src/test/java/org/elasticsearch/plugins/PluginLuceneCheckerTests.java
View File

@ -1,85 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import com.google.common.collect.Lists;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import org.junit.Test;
import java.net.URISyntaxException;
import java.util.Collections;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
/**
*
*/
@ClusterScope(scope= ElasticsearchIntegrationTest.Scope.TEST, numDataNodes=0, transportClientRatio = 0)
public class PluginLuceneCheckerTests extends PluginTestCase {
/**
* We check that no Lucene version checking is done
* when we set `"plugins.check_lucene":false`
*/
@Test
public void testDisableLuceneVersionCheckingPlugin() throws URISyntaxException {
String serverNodeId = startNodeWithPlugins(
settingsBuilder().put(PluginsService.PLUGINS_CHECK_LUCENE_KEY, false)
.put(PluginsService.ES_PLUGIN_PROPERTIES_FILE_KEY, "es-plugin-test.properties")
.put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true).build(),
"/org/elasticsearch/plugins/lucene/");
logger.info("--> server {} started" + serverNodeId);
NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setPlugins(true).execute().actionGet();
logger.info("--> full json answer, status " + response.toString());
ElasticsearchAssertions.assertNodeContainsPlugins(response, serverNodeId,
Lists.newArrayList("old-lucene"), Lists.newArrayList("old"), Lists.newArrayList("1.0.0"), // JVM Plugin
Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST);// No Site Plugin
}
/**
* We check that with an old plugin (built on an old Lucene version)
* plugin is not loaded
* We check that with a recent plugin (built on current Lucene version)
* plugin is loaded
* We check that with a too recent plugin (built on an unknown Lucene version)
* plugin is not loaded
*/
@Test
public void testEnableLuceneVersionCheckingPlugin() throws URISyntaxException {
String serverNodeId = startNodeWithPlugins(
settingsBuilder().put(PluginsService.PLUGINS_CHECK_LUCENE_KEY, true)
.put(PluginsService.ES_PLUGIN_PROPERTIES_FILE_KEY, "es-plugin-test.properties")
.put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true).build(),
"/org/elasticsearch/plugins/lucene/");
logger.info("--> server {} started" + serverNodeId);
NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setPlugins(true).execute().actionGet();
logger.info("--> full json answer, status " + response.toString());
ElasticsearchAssertions.assertNodeContainsPlugins(response, serverNodeId,
Lists.newArrayList("current-lucene"), Lists.newArrayList("current"), Lists.newArrayList("2.0.0"), // JVM Plugin
Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST);// No Site Plugin
}
}

123
core/src/test/java/org/elasticsearch/plugins/PluginServiceTests.java
View File

@ -1,123 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.action.admin.cluster.node.info.PluginInfo;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.plugins.loading.classpath.InClassPathPlugin;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.junit.Test;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.instanceOf;
@ClusterScope(scope= ElasticsearchIntegrationTest.Scope.TEST, numDataNodes=0, numClientNodes = 1, transportClientRatio = 0)
public class PluginServiceTests extends PluginTestCase {
@Test
public void testPluginLoadingFromClassName() throws URISyntaxException {
Settings settings = settingsBuilder()
// Defines a plugin in classpath
.put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true)
.put(PluginsService.ES_PLUGIN_PROPERTIES_FILE_KEY, "es-plugin-test.properties")
// Defines a plugin in settings
.put("plugin.types", InSettingsPlugin.class.getName())
.build();
startNodeWithPlugins(settings, "/org/elasticsearch/plugins/loading/");
Plugin plugin = getPlugin("in-settings-plugin");
assertNotNull("InSettingsPlugin (defined below in this class) must be loaded", plugin);
assertThat(plugin, instanceOf(InSettingsPlugin.class));
plugin = getPlugin("in-classpath-plugin");
assertNotNull("InClassPathPlugin (defined in package ) must be loaded", plugin);
assertThat(plugin, instanceOf(InClassPathPlugin.class));
plugin = getPlugin("in-jar-plugin");
assertNotNull("InJarPlugin (packaged as a JAR file in a plugins directory) must be loaded", plugin);
assertThat(plugin.getClass().getName(), endsWith("InJarPlugin"));
plugin = getPlugin("in-zip-plugin");
assertNotNull("InZipPlugin (packaged as a Zipped file in a plugins directory) must be loaded", plugin);
assertThat(plugin.getClass().getName(), endsWith("InZipPlugin"));
}
@Test
public void testHasLibExtension() {
PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(PluginsService.PLUGIN_LIB_PATTERN);
Path p = PathUtils.get("path", "to", "plugin.jar");
assertTrue(matcher.matches(p));
p = PathUtils.get("path", "to", "plugin.zip");
assertTrue(matcher.matches(p));
p = PathUtils.get("path", "to", "plugin.tar.gz");
assertFalse(matcher.matches(p));
p = PathUtils.get("path", "to", "plugin");
assertFalse(matcher.matches(p));
}
private Plugin getPlugin(String pluginName) {
assertNotNull("cannot check plugin existence with a null plugin's name", pluginName);
PluginsService pluginsService = internalCluster().getInstance(PluginsService.class);
ImmutableList<Tuple<PluginInfo, Plugin>> plugins = pluginsService.plugins();
if ((plugins != null) && (!plugins.isEmpty())) {
for (Tuple<PluginInfo, Plugin> plugin:plugins) {
if (pluginName.equals(plugin.v1().getName())) {
return plugin.v2();
}
}
}
return null;
}
static class InSettingsPlugin extends AbstractPlugin {
private final Settings settings;
public InSettingsPlugin(Settings settings) {
this.settings = settings;
}
@Override
public String name() {
return "in-settings-plugin";
}
@Override
public String description() {
return "A plugin defined in settings";
}
}
}