From 9c6aa6353eee1b42d97bc5567cb0ba45c95794a9 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Mon, 7 Mar 2016 18:00:30 +0100 Subject: [PATCH] Make xpack extensible: * Add XPackExtension: an api class (like Plugin in core) for what a x-pack extension can do. * Add XPackExtensionCli: a cli tool for adding, removing and listing extensions for x-pack. * Add XPackExtensionService: loading of jars from pluginsdir/x-pack/extensions, into child classloader. * Add bin/x-pack/extension script, similar to plugin cli, which installs an extension into pluginsdir/x-pack/extensions. * Add XPack extension integration test. Fixed elastic/elasticsearch#1515 Original commit: elastic/x-pack-elasticsearch@130ba03270b8c59faafc0b049a735005998a8392 --- .../qa/shield-example-realm/build.gradle | 42 +++- ...Plugin.java => ExampleRealmExtension.java} | 5 +- .../xpack-extension-descriptor.properties | 6 + .../example/realm/CustomRealmIT.java | 3 + elasticsearch/x-pack/bin/xpack/extension | 116 +++++++++++ elasticsearch/x-pack/bin/xpack/extension.bat | 9 + .../org/elasticsearch/xpack/XPackClient.java | 2 - .../org/elasticsearch/xpack/XPackPlugin.java | 28 +++ .../InstallXPackExtensionCommand.java | 190 ++++++++++++++++++ .../extensions/ListXPackExtensionCommand.java | 46 +++++ .../RemoveXPackExtensionCommand.java | 70 +++++++ .../xpack/extensions/XPackExtension.java | 29 +++ .../xpack/extensions/XPackExtensionCli.java | 33 +++ .../xpack/extensions/XPackExtensionInfo.java | 124 ++++++++++++ .../extensions/XPackExtensionsService.java | 186 +++++++++++++++++ .../InstallXPackExtensionCommandTests.java | 185 +++++++++++++++++ .../ListXPackExtensionCommandTests.java | 78 +++++++ .../RemoveXPackExtensionCommandTests.java | 75 +++++++ .../extensions/XPackExtensionInfoTests.java | 161 +++++++++++++++ .../extensions/XPackExtensionTestUtil.java | 31 +++ .../XPackExtensionsServiceTests.java | 23 +++ 21 files changed, 1427 insertions(+), 15 deletions(-) rename elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/{ExampleRealmPlugin.java => ExampleRealmExtension.java} (89%) create mode 100644 elasticsearch/qa/shield-example-realm/src/main/resources/xpack-extension-descriptor.properties create mode 100755 elasticsearch/x-pack/bin/xpack/extension create mode 100644 elasticsearch/x-pack/bin/xpack/extension.bat create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtension.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionInfo.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionsService.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionInfoTests.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionTestUtil.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionsServiceTests.java diff --git a/elasticsearch/qa/shield-example-realm/build.gradle b/elasticsearch/qa/shield-example-realm/build.gradle index b6e65102ac4..924133c3baf 100644 --- a/elasticsearch/qa/shield-example-realm/build.gradle +++ b/elasticsearch/qa/shield-example-realm/build.gradle @@ -1,19 +1,38 @@ -apply plugin: 'elasticsearch.esplugin' +import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.VersionProperties -esplugin { - description 'a very basic implementation of a custom realm to validate it works' - classname 'org.elasticsearch.example.ExampleRealmPlugin' - isolated false -} +apply plugin: 'elasticsearch.build' dependencies { + provided "org.elasticsearch:elasticsearch:${versions.elasticsearch}" + testCompile "org.elasticsearch.test:framework:${project.versions.elasticsearch}" provided project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime') } -compileJava.options.compilerArgs << "-Xlint:-rawtypes" -//compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" +Map generateSubstitutions() { + def stringSnap = { version -> + if (version.endsWith("-SNAPSHOT")) { + return version.substring(0, version.length() - 9) + } + return version + } + return [ + 'version': stringSnap(version), + 'xpack.version': stringSnap(VersionProperties.elasticsearch), + 'java.version': targetCompatibility as String + ] +} -integTest { +processResources { + MavenFilteringHack.filter(it, generateSubstitutions()) +} + +task buildZip(type:Zip, dependsOn: [jar]) { + from 'build/resources/main/xpack-extension-descriptor.properties' + from project.jar +} + +task integTest(type: org.elasticsearch.gradle.test.RestIntegTestTask, dependsOn: buildZip) { cluster { plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') // TODO: these should be settings? @@ -24,6 +43,8 @@ integTest { setupCommand 'setupDummyUser', 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' + setupCommand 'installExtension', + 'bin/xpack/extension', 'install', 'file:' + buildZip.archivePath waitCondition = { node, ant -> File tmpFile = new File(node.cwd, 'wait.success') ant.get(src: "http://${node.httpUri()}", @@ -36,4 +57,5 @@ integTest { } } } - +check.dependsOn integTest +integTest.mustRunAfter precommit \ No newline at end of file diff --git a/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java b/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmExtension.java similarity index 89% rename from elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java rename to elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmExtension.java index da194a919cf..752e0a74b32 100644 --- a/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java +++ b/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmExtension.java @@ -8,11 +8,10 @@ package org.elasticsearch.example; import org.elasticsearch.example.realm.CustomAuthenticationFailureHandler; import org.elasticsearch.example.realm.CustomRealm; import org.elasticsearch.example.realm.CustomRealmFactory; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.shield.authc.AuthenticationModule; +import org.elasticsearch.xpack.extensions.XPackExtension; -public class ExampleRealmPlugin extends Plugin { - +public class ExampleRealmExtension extends XPackExtension { @Override public String name() { return "custom realm example"; diff --git a/elasticsearch/qa/shield-example-realm/src/main/resources/xpack-extension-descriptor.properties b/elasticsearch/qa/shield-example-realm/src/main/resources/xpack-extension-descriptor.properties new file mode 100644 index 00000000000..faa674a8b49 --- /dev/null +++ b/elasticsearch/qa/shield-example-realm/src/main/resources/xpack-extension-descriptor.properties @@ -0,0 +1,6 @@ +description=Custom Realm Extension +version=${version} +name=examplerealm +classname=org.elasticsearch.example.ExampleRealmExtension +java.version=${java.version} +xpack.version=${xpack.version} \ No newline at end of file diff --git a/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java b/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java index 2bd0c252abe..2c6b60b7b8f 100644 --- a/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java +++ b/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.rest.client.http.HttpResponse; @@ -64,6 +65,7 @@ public class CustomRealmIT extends ESIntegTestCase { Settings settings = Settings.builder() .put("cluster.name", clusterName) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER) .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW) .build(); @@ -83,6 +85,7 @@ public class CustomRealmIT extends ESIntegTestCase { Settings settings = Settings.builder() .put("cluster.name", clusterName) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER + randomAsciiOfLength(1)) .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW) .build(); diff --git a/elasticsearch/x-pack/bin/xpack/extension b/elasticsearch/x-pack/bin/xpack/extension new file mode 100755 index 00000000000..35e061c7deb --- /dev/null +++ b/elasticsearch/x-pack/bin/xpack/extension @@ -0,0 +1,116 @@ +#!/bin/sh + +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +SCRIPT="$0" + +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# determine elasticsearch home +ES_HOME=`dirname "$SCRIPT"`/../.. + +# make ELASTICSEARCH_HOME absolute +ES_HOME=`cd "$ES_HOME"; pwd` + +# If an include wasn't specified in the environment, then search for one... +if [ "x$ES_INCLUDE" = "x" ]; then + # Locations (in order) to use when searching for an include file. + for include in /usr/share/elasticsearch/elasticsearch.in.sh \ + /usr/local/share/elasticsearch/elasticsearch.in.sh \ + /opt/elasticsearch/elasticsearch.in.sh \ + ~/.elasticsearch.in.sh \ + "`dirname "$0"`"/../elasticsearch.in.sh \ + "$ES_HOME/bin/elasticsearch.in.sh"; do + if [ -r "$include" ]; then + . "$include" + break + fi + done +# ...otherwise, source the specified include. +elif [ -r "$ES_INCLUDE" ]; then + . "$ES_INCLUDE" +fi + +if [ -x "$JAVA_HOME/bin/java" ]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA=`which java` +fi + +if [ ! -x "$JAVA" ]; then + echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME" + exit 1 +fi + +if [ -z "$ES_CLASSPATH" ]; then + echo "You must set the ES_CLASSPATH var" >&2 + exit 1 +fi + +# Try to read package config files +if [ -f "/etc/sysconfig/elasticsearch" ]; then + CONF_DIR=/etc/elasticsearch + CONF_FILE=$CONF_DIR/elasticsearch.yml + + . "/etc/sysconfig/elasticsearch" +elif [ -f "/etc/default/elasticsearch" ]; then + CONF_DIR=/etc/elasticsearch + CONF_FILE=$CONF_DIR/elasticsearch.yml + + . "/etc/default/elasticsearch" +fi + +# Parse any long getopt options and put them into properties before calling getopt below +# Be dash compatible to make sure running under ubuntu works +ARGCOUNT=$# +COUNT=0 +while [ $COUNT -lt $ARGCOUNT ] +do + case $1 in + --*=*) properties="$properties -Des.${1#--}" + shift 1; COUNT=$(($COUNT+1)) + ;; + --*) properties="$properties -Des.${1#--}=$2" + shift ; shift; COUNT=$(($COUNT+2)) + ;; + *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1)) + esac +done + +# check if properties already has a config file or config dir +if [ -e "$CONF_DIR" ]; then + case "$properties" in + *-Des.default.path.conf=*) ;; + *) + if [ ! -d "$CONF_DIR/xpack" ]; then + echo "ERROR: The configuration directory [$CONF_DIR/xpack] does not exist. The extension tool expects security configuration files in that location." + echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/xpack] exists, please copy the 'xpack' directory to [$CONF_DIR]" + exit 1 + fi + properties="$properties -Des.default.path.conf=$CONF_DIR" + ;; + esac +fi + +export HOSTNAME=`hostname -s` + +# include x-pack jars in classpath +ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/xpack/*" + +cd "$ES_HOME" > /dev/null +"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" $properties org.elasticsearch.xpack.extensions.XPackExtensionCli "$@" +status=$? +cd - > /dev/null +exit $status \ No newline at end of file diff --git a/elasticsearch/x-pack/bin/xpack/extension.bat b/elasticsearch/x-pack/bin/xpack/extension.bat new file mode 100644 index 00000000000..d1385dc6b59 --- /dev/null +++ b/elasticsearch/x-pack/bin/xpack/extension.bat @@ -0,0 +1,9 @@ +@echo off + +rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +rem or more contributor license agreements. Licensed under the Elastic License; +rem you may not use this file except in compliance with the Elastic License. + +PUSHD "%~dp0" +CALL "%~dp0.in.bat" org.elasticsearch.xpack.extensions.XPackExtensionCli %* +POPD \ No newline at end of file diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java index 9719a880d99..bdad3b9ec06 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java @@ -7,12 +7,10 @@ package org.elasticsearch.xpack; import org.elasticsearch.client.Client; import org.elasticsearch.shield.authc.support.SecuredString; -import org.elasticsearch.shield.authc.support.UsernamePasswordToken; import org.elasticsearch.shield.client.SecurityClient; import org.elasticsearch.watcher.client.WatcherClient; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER; diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java index 4b4f3a87737..adcf8768a48 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java @@ -22,15 +22,19 @@ import org.elasticsearch.marvel.Marvel; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.shield.Shield; +import org.elasticsearch.shield.authc.AuthenticationModule; import org.elasticsearch.watcher.Watcher; import org.elasticsearch.xpack.common.init.LazyInitializationModule; import org.elasticsearch.xpack.common.init.LazyInitializationService; +import org.elasticsearch.xpack.extensions.XPackExtension; +import org.elasticsearch.xpack.extensions.XPackExtensionsService; import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; public class XPackPlugin extends Plugin { @@ -67,6 +71,7 @@ public class XPackPlugin extends Plugin { } protected final Settings settings; + protected final XPackExtensionsService extensionsService; protected Licensing licensing; protected Shield shield; @@ -81,6 +86,14 @@ public class XPackPlugin extends Plugin { this.marvel = new Marvel(settings); this.watcher = new Watcher(settings); this.graph = new Graph(settings); + // Check if the node is a transport client. + if (transportClientMode(settings) == false) { + Environment env = new Environment(settings); + this.extensionsService = + new XPackExtensionsService(settings, resolveXPackExtensionsFile(env), getExtensions()); + } else { + this.extensionsService = null; + } } @Override public String name() { @@ -91,6 +104,11 @@ public class XPackPlugin extends Plugin { return "Elastic X-Pack"; } + // For tests only + public Collection> getExtensions() { + return Collections.emptyList(); + } + @Override public Collection nodeModules() { ArrayList modules = new ArrayList<>(); @@ -157,6 +175,12 @@ public class XPackPlugin extends Plugin { graph.onModule(module); } + public void onModule(AuthenticationModule module) { + if (extensionsService != null) { + extensionsService.onModule(module); + } + } + public void onIndexModule(IndexModule module) { shield.onIndexModule(module); graph.onIndexModule(module); @@ -221,4 +245,8 @@ public class XPackPlugin extends Plugin { settingsModule.registerSetting(Setting.boolSetting(legacyFeatureEnabledSetting(featureName), defaultValue, Setting.Property.NodeScope)); } + + public static Path resolveXPackExtensionsFile(Environment env) { + return env.pluginsFile().resolve("xpack").resolve("extensions"); + } } diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java new file mode 100644 index 00000000000..749cc745760 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; +import org.apache.lucene.util.IOUtils; + +import org.elasticsearch.bootstrap.JarHell; +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.env.Environment; + +import java.io.InputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Arrays; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile; +import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; + +/** + * A command for the extension cli to install an extension into x-pack. + * + * The install command takes a URL to an extension zip. + * + * Extensions are packaged as zip files. Each packaged extension must contain an + * extension properties file. See {@link XPackExtensionInfo}. + *

+ * The installation process first extracts the extensions files into a temporary + * directory in order to verify the extension satisfies the following requirements: + *

    + *
  • The property file exists and contains valid metadata. See {@link XPackExtensionInfo#readFromProperties(Path)}
    • + *
  • Jar hell does not exist, either between the extension's own jars or with the parent classloader (elasticsearch + xpack)
    • + *
+ */ +class InstallXPackExtensionCommand extends Command { + + private final Environment env; + private final OptionSpec batchOption; + private final OptionSpec arguments; + + InstallXPackExtensionCommand(Environment env) { + super("Install a plugin"); + this.env = env; + this.batchOption = parser.acceptsAll(Arrays.asList("b", "batch"), + "Enable batch mode explicitly, automatic confirmation of security permission"); + this.arguments = parser.nonOptions("plugin id"); + } + + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args + List args = arguments.values(options); + if (args.size() != 1) { + throw new UserError(ExitCodes.USAGE, "Must supply a single extension id argument"); + } + String extensionURL = args.get(0); + boolean isBatch = options.has(batchOption) || System.console() == null; + execute(terminal, extensionURL, isBatch); + } + + + // pkg private for testing + void execute(Terminal terminal, String extensionId, boolean isBatch) throws Exception { + if (Files.exists(resolveXPackExtensionsFile(env)) == false) { + terminal.println("xpack extensions directory [" + resolveXPackExtensionsFile(env) + "] does not exist. Creating..."); + Files.createDirectories(resolveXPackExtensionsFile(env)); + } + + Path extensionZip = download(terminal, extensionId, env.tmpFile()); + Path extractedZip = unzip(extensionZip, resolveXPackExtensionsFile(env)); + install(terminal, extractedZip, env); + } + + /** Downloads the extension and returns the file it was downloaded to. */ + private Path download(Terminal terminal, String extensionURL, Path tmpDir) throws Exception { + terminal.println("-> Downloading " + URLDecoder.decode(extensionURL, "UTF-8")); + URL url = new URL(extensionURL); + Path zip = Files.createTempFile(tmpDir, null, ".zip"); + try (InputStream in = url.openStream()) { + // must overwrite since creating the temp file above actually created the file + Files.copy(in, zip, StandardCopyOption.REPLACE_EXISTING); + } + return zip; + } + + private Path unzip(Path zip, Path extensionDir) throws IOException, UserError { + // unzip extension to a staging temp dir + Path target = Files.createTempDirectory(extensionDir, ".installing-"); + Files.createDirectories(target); + + // TODO: we should wrap this in a try/catch and try deleting the target dir on failure? + try (ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) { + ZipEntry entry; + byte[] buffer = new byte[8192]; + while ((entry = zipInput.getNextEntry()) != null) { + Path targetFile = target.resolve(entry.getName()); + // TODO: handle name being an absolute path + + // be on the safe side: do not rely on that directories are always extracted + // before their children (although this makes sense, but is it guaranteed?) + Files.createDirectories(targetFile.getParent()); + if (entry.isDirectory() == false) { + try (OutputStream out = Files.newOutputStream(targetFile)) { + int len; + while((len = zipInput.read(buffer)) >= 0) { + out.write(buffer, 0, len); + } + } + } + zipInput.closeEntry(); + } + } + Files.delete(zip); + return target; + } + + /** Load information about the extension, and verify it can be installed with no errors. */ + private XPackExtensionInfo verify(Terminal terminal, Path extensionRoot, Environment env) throws Exception { + // read and validate the extension descriptor + XPackExtensionInfo info = XPackExtensionInfo.readFromProperties(extensionRoot); + terminal.println(VERBOSE, info.toString()); + + // check for jar hell before any copying + jarHellCheck(extensionRoot); + return info; + } + + /** check a candidate extension for jar hell before installing it */ + private void jarHellCheck(Path candidate) throws Exception { + // create list of current jars in classpath + // including the x-pack jars (see $ES_CLASSPATH in bin/extension script) + final List jars = new ArrayList<>(); + jars.addAll(Arrays.asList(JarHell.parseClassPath())); + + // add extension jars to the list + Path extensionJars[] = FileSystemUtils.files(candidate, "*.jar"); + for (Path jar : extensionJars) { + jars.add(jar.toUri().toURL()); + } + // TODO: no jars should be an error + // TODO: verify the classname exists in one of the jars! + + // check combined (current classpath + new jars to-be-added) + JarHell.checkJarHell(jars.toArray(new URL[jars.size()])); + } + + /** + * Installs the extension from {@code tmpRoot} into the extensions dir. + */ + private void install(Terminal terminal, Path tmpRoot, Environment env) throws Exception { + List deleteOnFailure = new ArrayList<>(); + deleteOnFailure.add(tmpRoot); + try { + XPackExtensionInfo info = verify(terminal, tmpRoot, env); + final Path destination = resolveXPackExtensionsFile(env).resolve(info.getName()); + if (Files.exists(destination)) { + throw new UserError(ExitCodes.USAGE, + "extension directory " + destination.toAbsolutePath() + + " already exists. To update the extension, uninstall it first using 'remove " + + info.getName() + "' command"); + } + Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE); + terminal.println("-> Installed " + info.getName()); + } catch (Exception installProblem) { + try { + IOUtils.rm(deleteOnFailure.toArray(new Path[0])); + } catch (IOException exceptionWhileRemovingFiles) { + installProblem.addSuppressed(exceptionWhileRemovingFiles); + } + throw installProblem; + } + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java new file mode 100644 index 00000000000..ccf7768fb95 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import joptsimple.OptionSet; + +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.env.Environment; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile; +import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; + +/** + * A command for the extension cli to list extensions installed in x-pack. + */ +class ListXPackExtensionCommand extends Command { + private final Environment env; + + ListXPackExtensionCommand(Environment env) { + super("Lists installed x-pack extensions"); + this.env = env; + } + + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + if (Files.exists(resolveXPackExtensionsFile(env)) == false) { + throw new IOException("Extensions directory missing: " + resolveXPackExtensionsFile(env)); + } + + terminal.println(VERBOSE, "Extensions directory: " + resolveXPackExtensionsFile(env)); + try (DirectoryStream stream = Files.newDirectoryStream(resolveXPackExtensionsFile(env))) { + for (Path extension : stream) { + terminal.println(extension.getFileName().toString()); + } + } + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java new file mode 100644 index 00000000000..4b51adbaa82 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.Strings; +import org.elasticsearch.env.Environment; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile; +import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; + +/** + * A command for the extension cli to remove an extension from x-pack. + */ +class RemoveXPackExtensionCommand extends Command { + private final Environment env; + private final OptionSpec arguments; + + RemoveXPackExtensionCommand(Environment env) { + super("Removes an extension from x-pack"); + this.env = env; + this.arguments = parser.nonOptions("extension name"); + } + + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args + List args = arguments.values(options); + if (args.size() != 1) { + throw new UserError(ExitCodes.USAGE, "Must supply a single extension id argument"); + } + execute(terminal, args.get(0)); + } + + // pkg private for testing + void execute(Terminal terminal, String extensionName) throws Exception { + terminal.println("-> Removing " + Strings.coalesceToEmpty(extensionName) + "..."); + + Path extensionDir = resolveXPackExtensionsFile(env).resolve(extensionName); + if (Files.exists(extensionDir) == false) { + throw new UserError(ExitCodes.USAGE, + "Extension " + extensionName + " not found. Run 'bin/xpack/extension list' to get list of installed extensions."); + } + + List extensionPaths = new ArrayList<>(); + + terminal.println(VERBOSE, "Removing: " + extensionDir); + Path tmpExtensionDir = resolveXPackExtensionsFile(env).resolve(".removing-" + extensionName); + Files.move(extensionDir, tmpExtensionDir, StandardCopyOption.ATOMIC_MOVE); + extensionPaths.add(tmpExtensionDir); + + IOUtils.rm(extensionPaths.toArray(new Path[extensionPaths.size()])); + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtension.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtension.java new file mode 100644 index 00000000000..f1bf62a0b78 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtension.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.shield.authc.AuthenticationModule; + + +/** + * An extension point allowing to plug in custom functionality in x-pack authentication module. + */ +public abstract class XPackExtension { + /** + * The name of the plugin. + */ + public abstract String name(); + + /** + * The description of the plugin. + */ + public abstract String description(); + + /** + * Implement this function to register custom extensions in the authentication module. + */ + public void onModule(AuthenticationModule module) {} +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java new file mode 100644 index 00000000000..f18d3db3a02 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.varia.NullAppender; +import org.elasticsearch.cli.MultiCommand; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.internal.InternalSettingsPreparer; + +/** + * A cli tool for adding, removing and listing extensions for x-pack. + */ +public class XPackExtensionCli extends MultiCommand { + + public XPackExtensionCli(Environment env) { + super("A tool for managing installed x-pack extensions"); + subcommands.put("list", new ListXPackExtensionCommand(env)); + subcommands.put("install", new InstallXPackExtensionCommand(env)); + subcommands.put("remove", new RemoveXPackExtensionCommand(env)); + } + + public static void main(String[] args) throws Exception { + BasicConfigurator.configure(new NullAppender()); + Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, Terminal.DEFAULT); + exit(new XPackExtensionCli(env).main(args, Terminal.DEFAULT)); + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionInfo.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionInfo.java new file mode 100644 index 00000000000..08a3eb01626 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionInfo.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.Version; +import org.elasticsearch.bootstrap.JarHell; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +public class XPackExtensionInfo { + public static final String XPACK_EXTENSION_PROPERTIES = "xpack-extension-descriptor.properties"; + + private String name; + private String description; + private String version; + private String classname; + + public XPackExtensionInfo() { + } + + /** + * Information about extensions + * + * @param name Its name + * @param description Its description + * @param version Version number + */ + XPackExtensionInfo(String name, String description, String version, String classname) { + this.name = name; + this.description = description; + this.version = version; + this.classname = classname; + } + + /** reads (and validates) extension metadata descriptor file */ + public static XPackExtensionInfo readFromProperties(Path dir) throws IOException { + Path descriptor = dir.resolve(XPACK_EXTENSION_PROPERTIES); + Properties props = new Properties(); + try (InputStream stream = Files.newInputStream(descriptor)) { + props.load(stream); + } + String name = props.getProperty("name"); + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException("Property [name] is missing in [" + descriptor + "]"); + } + String description = props.getProperty("description"); + if (description == null) { + throw new IllegalArgumentException("Property [description] is missing for extension [" + name + "]"); + } + String version = props.getProperty("version"); + if (version == null) { + throw new IllegalArgumentException("Property [version] is missing for extension [" + name + "]"); + } + + String xpackVersionString = props.getProperty("xpack.version"); + if (xpackVersionString == null) { + throw new IllegalArgumentException("Property [xpack.version] is missing for extension [" + name + "]"); + } + Version xpackVersion = Version.fromString(xpackVersionString); + if (xpackVersion.equals(Version.CURRENT) == false) { + throw new IllegalArgumentException("extension [" + name + "] is incompatible with Elasticsearch [" + + Version.CURRENT.toString() + "]. Was designed for version [" + xpackVersionString + "]"); + } + String javaVersionString = props.getProperty("java.version"); + if (javaVersionString == null) { + throw new IllegalArgumentException("Property [java.version] is missing for extension [" + name + "]"); + } + JarHell.checkVersionFormat(javaVersionString); + JarHell.checkJavaVersion(name, javaVersionString); + String classname = props.getProperty("classname"); + if (classname == null) { + throw new IllegalArgumentException("Property [classname] is missing for extension [" + name + "]"); + } + + return new XPackExtensionInfo(name, description, version, classname); + } + + /** + * @return Extension's name + */ + public String getName() { + return name; + } + + /** + * @return Extension's description if any + */ + public String getDescription() { + return description; + } + + /** + * @return extension's classname + */ + public String getClassname() { + return classname; + } + + /** + * @return Version number for the extension + */ + public String getVersion() { + return version; + } + + @Override + public String toString() { + final StringBuilder information = new StringBuilder() + .append("- XPack Extension information:\n") + .append("Name: ").append(name).append("\n") + .append("Description: ").append(description).append("\n") + .append("Version: ").append(version).append("\n") + .append(" * Classname: ").append(classname); + + return information.toString(); + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionsService.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionsService.java new file mode 100644 index 00000000000..42ff95f39f3 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionsService.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.bootstrap.JarHell; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.shield.authc.AuthenticationModule; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Arrays; + +import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; + +/** + * + */ +public class XPackExtensionsService { + private final Settings settings; + + /** + * We keep around a list of extensions + */ + private final List > extensions; + + /** + * Constructs a new XPackExtensionsService + * @param settings The settings of the system + * @param extsDirectory The directory extensions exist in, or null if extensions should not be loaded from the filesystem + * @param classpathExtensions Extensions that exist in the classpath which should be loaded + */ + public XPackExtensionsService(Settings settings, Path extsDirectory, Collection> classpathExtensions) { + this.settings = settings; + List> extensionsLoaded = new ArrayList<>(); + + // first we load extensions that are on the classpath. this is for tests + for (Class extClass : classpathExtensions) { + XPackExtension ext = loadExtension(extClass, settings); + XPackExtensionInfo extInfo = new XPackExtensionInfo(ext.name(), ext.description(), "NA", extClass.getName()); + extensionsLoaded.add(new Tuple<>(extInfo, ext)); + } + + // now, find all the ones that are in plugins/xpack/extensions + if (extsDirectory != null) { + try { + List bundles = getExtensionBundles(extsDirectory); + List> loaded = loadBundles(bundles); + extensionsLoaded.addAll(loaded); + } catch (IOException ex) { + throw new IllegalStateException("Unable to initialize extensions", ex); + } + } + extensions = Collections.unmodifiableList(extensionsLoaded); + } + + public void onModule(AuthenticationModule module) { + for (Tuple tuple : extensions) { + tuple.v2().onModule(module); + } + } + + // a "bundle" is a an extension in a single classloader. + static class Bundle { + XPackExtensionInfo info; + List urls = new ArrayList<>(); + } + + static List getExtensionBundles(Path extsDirectory) throws IOException { + ESLogger logger = Loggers.getLogger(XPackExtensionsService.class); + + // TODO: remove this leniency, but tests bogusly rely on it + if (!isAccessibleDirectory(extsDirectory, logger)) { + return Collections.emptyList(); + } + + List bundles = new ArrayList<>(); + + try (DirectoryStream stream = Files.newDirectoryStream(extsDirectory)) { + for (Path extension : stream) { + if (FileSystemUtils.isHidden(extension)) { + logger.trace("--- skip hidden extension file[{}]", extension.toAbsolutePath()); + continue; + } + logger.trace("--- adding extension [{}]", extension.toAbsolutePath()); + final XPackExtensionInfo info; + try { + info = XPackExtensionInfo.readFromProperties(extension); + } catch (IOException e) { + throw new IllegalStateException("Could not load extension descriptor for existing extension [" + + extension.getFileName() + "]. Was the extension built before 2.0?", e); + } + + List urls = new ArrayList<>(); + try (DirectoryStream jarStream = Files.newDirectoryStream(extension, "*.jar")) { + for (Path jar : jarStream) { + // normalize with toRealPath to get symlinks out of our hair + urls.add(jar.toRealPath().toUri().toURL()); + } + } + final Bundle bundle = new Bundle(); + bundles.add(bundle); + bundle.info = info; + bundle.urls.addAll(urls); + } + } + + return bundles; + } + + private List > loadBundles(List bundles) { + List> exts = new ArrayList<>(); + + for (Bundle bundle : bundles) { + // jar-hell check the bundle against the parent classloader and the x-pack classloader + // pluginmanager does it, but we do it again, in case lusers mess with jar files manually + try { + final List jars = new ArrayList<>(); + // add the parent jars to the list + jars.addAll(Arrays.asList(JarHell.parseClassPath())); + + // add the x-pack jars to the list + ClassLoader xpackLoader = getClass().getClassLoader(); + // this class is loaded from the isolated x-pack plugin's classloader + if (xpackLoader instanceof URLClassLoader) { + jars.addAll(Arrays.asList(((URLClassLoader) xpackLoader).getURLs())); + } + + jars.addAll(bundle.urls); + + JarHell.checkJarHell(jars.toArray(new URL[0])); + } catch (Exception e) { + throw new IllegalStateException("failed to load bundle " + bundle.urls + " due to jar hell", e); + } + + // create a child to load the extension in this bundle + ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), getClass().getClassLoader()); + final Class extClass = loadExtensionClass(bundle.info.getClassname(), loader); + final XPackExtension ext = loadExtension(extClass, settings); + exts.add(new Tuple<>(bundle.info, ext)); + } + + return Collections.unmodifiableList(exts); + } + + private Class loadExtensionClass(String className, ClassLoader loader) { + try { + return loader.loadClass(className).asSubclass(XPackExtension.class); + } catch (ClassNotFoundException e) { + throw new ElasticsearchException("Could not find extension class [" + className + "]", e); + } + } + + private XPackExtension loadExtension(Class extClass, Settings settings) { + try { + try { + return extClass.getConstructor(Settings.class).newInstance(settings); + } catch (NoSuchMethodException e) { + try { + return extClass.getConstructor().newInstance(); + } catch (NoSuchMethodException e1) { + throw new ElasticsearchException("No constructor for [" + extClass + "]. An extension class must " + + "have either an empty default constructor or a single argument constructor accepting a " + + "Settings instance"); + } + } + } catch (Throwable e) { + throw new ElasticsearchException("Failed to load extension class [" + extClass.getName() + "]", e); + } + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java new file mode 100644 index 00000000000..e86e7112545 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.Version; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.DirectoryStream; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.FileVisitResult; +import java.nio.file.NoSuchFileException; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@LuceneTestCase.SuppressFileSystems("*") +public class InstallXPackExtensionCommandTests extends ESTestCase { + /** + * Creates a test environment with plugins and xpack extensions directories. + */ + static Environment createEnv() throws IOException { + Path home = createTempDir(); + Files.createDirectories(home.resolve("org/elasticsearch/xpack/extensions").resolve("xpack").resolve("extensions")); + Settings settings = Settings.builder() + .put("path.home", home) + .build(); + return new Environment(settings); + } + + /** + * creates a fake jar file with empty class files + */ + static void writeJar(Path jar, String... classes) throws IOException { + try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) { + for (String clazz : classes) { + stream.putNextEntry(new ZipEntry(clazz + ".class")); // no package names, just support simple classes + } + } + } + + static String writeZip(Path structure) throws IOException { + Path zip = createTempDir().resolve(structure.getFileName() + ".zip"); + try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) { + Files.walkFileTree(structure, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + String target = structure.relativize(file).toString(); + stream.putNextEntry(new ZipEntry(target)); + Files.copy(file, stream); + return FileVisitResult.CONTINUE; + } + }); + } + return zip.toUri().toURL().toString(); + } + + /** + * creates an extension .zip and returns the url for testing + */ + static String createExtension(String name, Path structure) throws IOException { + XPackExtensionTestUtil.writeProperties(structure, + "description", "fake desc", + "name", name, + "version", "1.0", + "xpack.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "FakeExtension"); + writeJar(structure.resolve("extension.jar"), "FakeExtension"); + return writeZip(structure); + } + + static MockTerminal installExtension(String extensionUrl, Environment env) throws Exception { + MockTerminal terminal = new MockTerminal(); + new InstallXPackExtensionCommand(env).execute(terminal, extensionUrl, true); + return terminal; + } + + void assertExtension(String name, Path original, Environment env) throws IOException { + Path got = env.pluginsFile().resolve("xpack").resolve("extensions").resolve(name); + assertTrue("dir " + name + " exists", Files.exists(got)); + assertTrue("jar was copied", Files.exists(got.resolve("extension.jar"))); + assertInstallCleaned(env); + } + + void assertInstallCleaned(Environment env) throws IOException { + try (DirectoryStream stream = Files.newDirectoryStream(env.pluginsFile().resolve("xpack").resolve("extensions"))) { + for (Path file : stream) { + if (file.getFileName().toString().startsWith(".installing")) { + fail("Installation dir still exists, " + file); + } + } + } + } + + public void testSomethingWorks() throws Exception { + Environment env = createEnv(); + Path extDir = createTempDir(); + String extZip = createExtension("fake", extDir); + installExtension(extZip, env); + assertExtension("fake", extDir, env); + } + + public void testSpaceInUrl() throws Exception { + Environment env = createEnv(); + Path extDir = createTempDir(); + String extZip = createExtension("fake", extDir); + Path extZipWithSpaces = createTempFile("foo bar", ".zip"); + try (InputStream in = new URL(extZip).openStream()) { + Files.copy(in, extZipWithSpaces, StandardCopyOption.REPLACE_EXISTING); + } + installExtension(extZipWithSpaces.toUri().toURL().toString(), env); + assertExtension("fake", extDir, env); + } + + public void testMalformedUrlNotMaven() throws Exception { + // has two colons, so it appears similar to maven coordinates + MalformedURLException e = expectThrows(MalformedURLException.class, () -> { + installExtension("://host:1234", createEnv()); + }); + assertTrue(e.getMessage(), e.getMessage().contains("no protocol")); + } + + public void testJarHell() throws Exception { + Environment env = createEnv(); + Path extDir = createTempDir(); + writeJar(extDir.resolve("other.jar"), "FakeExtension"); + String extZip = createExtension("fake", extDir); // adds extension.jar with FakeExtension + IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + installExtension(extZip, env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); + assertInstallCleaned(env); + } + + public void testIsolatedExtension() throws Exception { + Environment env = createEnv(); + // these both share the same FakeExtension class + Path extDir1 = createTempDir(); + String extZip1 = createExtension("fake1", extDir1); + installExtension(extZip1, env); + Path extDir2 = createTempDir(); + String extZip2 = createExtension("fake2", extDir2); + installExtension(extZip2, env); + assertExtension("fake1", extDir1, env); + assertExtension("fake2", extDir2, env); + } + + public void testExistingExtension() throws Exception { + Environment env = createEnv(); + String extZip = createExtension("fake", createTempDir()); + installExtension(extZip, env); + UserError e = expectThrows(UserError.class, () -> { + installExtension(extZip, env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("already exists")); + assertInstallCleaned(env); + } + + public void testMissingDescriptor() throws Exception { + Environment env = createEnv(); + Path extDir = createTempDir(); + Files.createFile(extDir.resolve("fake.yml")); + String extZip = writeZip(extDir); + NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> { + installExtension(extZip, env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("xpack-extension-descriptor.properties")); + assertInstallCleaned(env); + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java new file mode 100644 index 00000000000..2f61e219687 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +@LuceneTestCase.SuppressFileSystems("*") +public class ListXPackExtensionCommandTests extends ESTestCase { + + Environment createEnv() throws IOException { + Path home = createTempDir(); + Settings settings = Settings.builder() + .put("path.home", home) + .build(); + return new Environment(settings); + } + + Path createExtensionDir(Environment env) throws IOException { + Path path = env.pluginsFile().resolve("xpack").resolve("extensions"); + return Files.createDirectories(path); + } + + static MockTerminal listExtensions(Environment env) throws Exception { + MockTerminal terminal = new MockTerminal(); + String[] args = {}; + int status = new ListXPackExtensionCommand(env).main(args, terminal); + assertEquals(ExitCodes.OK, status); + return terminal; + } + + public void testExtensionsDirMissing() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + Files.delete(extDir); + IOException e = expectThrows(IOException.class, () -> { + listExtensions(env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("Extensions directory missing")); + } + + public void testNoExtensions() throws Exception { + Environment env = createEnv(); + createExtensionDir(env); + MockTerminal terminal = listExtensions(env); + assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty()); + } + + public void testOneExtension() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + Files.createDirectory(extDir.resolve("fake")); + MockTerminal terminal = listExtensions(env); + assertTrue(terminal.getOutput(), terminal.getOutput().contains("fake")); + } + + public void testTwoExtensions() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + Files.createDirectory(extDir.resolve("fake1")); + Files.createDirectory(extDir.resolve("fake2")); + MockTerminal terminal = listExtensions(env); + String output = terminal.getOutput(); + assertTrue(output, output.contains("fake1")); + assertTrue(output, output.contains("fake2")); + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java new file mode 100644 index 00000000000..aa5a27d1f76 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; + +@LuceneTestCase.SuppressFileSystems("*") +public class RemoveXPackExtensionCommandTests extends ESTestCase { + + /** Creates a test environment with bin, config and plugins directories. */ + static Environment createEnv() throws IOException { + Path home = createTempDir(); + Settings settings = Settings.builder() + .put("path.home", home) + .build(); + return new Environment(settings); + } + + Path createExtensionDir(Environment env) throws IOException { + Path path = env.pluginsFile().resolve("xpack").resolve("extensions"); + return Files.createDirectories(path); + } + + static MockTerminal removeExtension(String name, Environment env) throws Exception { + MockTerminal terminal = new MockTerminal(); + new RemoveXPackExtensionCommand(env).execute(terminal, name); + return terminal; + } + + static void assertRemoveCleaned(Path extDir) throws IOException { + try (DirectoryStream stream = Files.newDirectoryStream(extDir)) { + for (Path file : stream) { + if (file.getFileName().toString().startsWith(".removing")) { + fail("Removal dir still exists, " + file); + } + } + } + } + + public void testMissing() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + UserError e = expectThrows(UserError.class, () -> { + removeExtension("dne", env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("Extension dne not found")); + assertRemoveCleaned(extDir); + } + + public void testBasic() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + Files.createDirectory(extDir.resolve("fake")); + Files.createFile(extDir.resolve("fake").resolve("extension.jar")); + Files.createDirectory(extDir.resolve("fake").resolve("subdir")); + Files.createDirectory(extDir.resolve("other")); + removeExtension("fake", env); + assertFalse(Files.exists(extDir.resolve("fake"))); + assertTrue(Files.exists(extDir.resolve("other"))); + assertRemoveCleaned(extDir); + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionInfoTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionInfoTests.java new file mode 100644 index 00000000000..63c289d3592 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionInfoTests.java @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.Version; +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Path; + +public class XPackExtensionInfoTests extends ESTestCase { + + public void testReadFromProperties() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "version", "1.0", + "xpack.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "FakeExtension"); + XPackExtensionInfo info = XPackExtensionInfo.readFromProperties(extensionDir); + assertEquals("my_extension", info.getName()); + assertEquals("fake desc", info.getDescription()); + assertEquals("1.0", info.getVersion()); + assertEquals("FakeExtension", info.getClassname()); + } + + public void testReadFromPropertiesNameMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("Property [name] is missing in")); + XPackExtensionTestUtil.writeProperties(extensionDir, "name", ""); + IllegalArgumentException e1 = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e1.getMessage().contains("Property [name] is missing in")); + } + + public void testReadFromPropertiesDescriptionMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, "name", "fake-extension"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("[description] is missing")); + } + + public void testReadFromPropertiesVersionMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, "description", "fake desc", "name", "fake-extension"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("[version] is missing")); + } + + public void testReadFromPropertiesElasticsearchVersionMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "version", "1.0"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("[xpack.version] is missing")); + } + + public void testReadFromPropertiesJavaVersionMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "xpack.version", Version.CURRENT.toString(), + "version", "1.0"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("[java.version] is missing")); + } + + public void testReadFromPropertiesJavaVersionIncompatible() throws Exception { + String extensionName = "fake-extension"; + Path extensionDir = createTempDir().resolve(extensionName); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", extensionName, + "xpack.version", Version.CURRENT.toString(), + "java.version", "1000000.0", + "classname", "FakeExtension", + "version", "1.0"); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage(), e.getMessage().contains(extensionName + " requires Java")); + } + + public void testReadFromPropertiesBadJavaVersionFormat() throws Exception { + String extensionName = "fake-extension"; + Path extensionDir = createTempDir().resolve(extensionName); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", extensionName, + "xpack.version", Version.CURRENT.toString(), + "java.version", "1.7.0_80", + "classname", "FakeExtension", + "version", "1.0"); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage(), + e.getMessage().equals("version string must be a sequence of nonnegative decimal " + + "integers separated by \".\"'s and may have leading zeros but was 1.7.0_80")); + } + + public void testReadFromPropertiesBogusElasticsearchVersion() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "version", "1.0", + "name", "my_extension", + "xpack.version", "bogus"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("version needs to contain major, minor, and revision")); + } + + public void testReadFromPropertiesOldElasticsearchVersion() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "version", "1.0", + "xpack.version", Version.V_2_0_0.toString()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("Was designed for version [2.0.0]")); + } + + public void testReadFromPropertiesJvmMissingClassname() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "version", "1.0", + "xpack.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("Property [classname] is missing")); + } +} \ No newline at end of file diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionTestUtil.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionTestUtil.java new file mode 100644 index 00000000000..866b6e557e9 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionTestUtil.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +/** Utility methods for testing extensions */ +public class XPackExtensionTestUtil { + + /** convenience method to write a plugin properties file */ + public static void writeProperties(Path pluginDir, String... stringProps) throws IOException { + assert stringProps.length % 2 == 0; + Files.createDirectories(pluginDir); + Path propertiesFile = pluginDir.resolve(XPackExtensionInfo.XPACK_EXTENSION_PROPERTIES); + Properties properties = new Properties(); + for (int i = 0; i < stringProps.length; i += 2) { + properties.put(stringProps[i], stringProps[i + 1]); + } + try (OutputStream out = Files.newOutputStream(propertiesFile)) { + properties.store(out, ""); + } + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionsServiceTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionsServiceTests.java new file mode 100644 index 00000000000..7730ec7a4dd --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionsServiceTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Files; +import java.nio.file.Path; + +public class XPackExtensionsServiceTests extends ESTestCase { + public void testExistingPluginMissingDescriptor() throws Exception { + Path extensionsDir = createTempDir(); + Files.createDirectory(extensionsDir.resolve("extension-missing-descriptor")); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + XPackExtensionsService.getExtensionBundles(extensionsDir); + }); + assertTrue(e.getMessage(), + e.getMessage().contains("Could not load extension descriptor for existing extension")); + } +}