From 9c6aa6353eee1b42d97bc5567cb0ba45c95794a9 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Mon, 7 Mar 2016 18:00:30 +0100 Subject: [PATCH 1/8] Make xpack extensible: * Add XPackExtension: an api class (like Plugin in core) for what a x-pack extension can do. * Add XPackExtensionCli: a cli tool for adding, removing and listing extensions for x-pack. * Add XPackExtensionService: loading of jars from pluginsdir/x-pack/extensions, into child classloader. * Add bin/x-pack/extension script, similar to plugin cli, which installs an extension into pluginsdir/x-pack/extensions. * Add XPack extension integration test. Fixed elastic/elasticsearch#1515 Original commit: elastic/x-pack-elasticsearch@130ba03270b8c59faafc0b049a735005998a8392 --- .../qa/shield-example-realm/build.gradle | 42 +++- ...Plugin.java => ExampleRealmExtension.java} | 5 +- .../xpack-extension-descriptor.properties | 6 + .../example/realm/CustomRealmIT.java | 3 + elasticsearch/x-pack/bin/xpack/extension | 116 +++++++++++ elasticsearch/x-pack/bin/xpack/extension.bat | 9 + .../org/elasticsearch/xpack/XPackClient.java | 2 - .../org/elasticsearch/xpack/XPackPlugin.java | 28 +++ .../InstallXPackExtensionCommand.java | 190 ++++++++++++++++++ .../extensions/ListXPackExtensionCommand.java | 46 +++++ .../RemoveXPackExtensionCommand.java | 70 +++++++ .../xpack/extensions/XPackExtension.java | 29 +++ .../xpack/extensions/XPackExtensionCli.java | 33 +++ .../xpack/extensions/XPackExtensionInfo.java | 124 ++++++++++++ .../extensions/XPackExtensionsService.java | 186 +++++++++++++++++ .../InstallXPackExtensionCommandTests.java | 185 +++++++++++++++++ .../ListXPackExtensionCommandTests.java | 78 +++++++ .../RemoveXPackExtensionCommandTests.java | 75 +++++++ .../extensions/XPackExtensionInfoTests.java | 161 +++++++++++++++ .../extensions/XPackExtensionTestUtil.java | 31 +++ .../XPackExtensionsServiceTests.java | 23 +++ 21 files changed, 1427 insertions(+), 15 deletions(-) rename elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/{ExampleRealmPlugin.java => ExampleRealmExtension.java} (89%) create mode 100644 elasticsearch/qa/shield-example-realm/src/main/resources/xpack-extension-descriptor.properties create mode 100755 elasticsearch/x-pack/bin/xpack/extension create mode 100644 elasticsearch/x-pack/bin/xpack/extension.bat create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtension.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionInfo.java create mode 100644 elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionsService.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionInfoTests.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionTestUtil.java create mode 100644 elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionsServiceTests.java diff --git a/elasticsearch/qa/shield-example-realm/build.gradle b/elasticsearch/qa/shield-example-realm/build.gradle index b6e65102ac4..924133c3baf 100644 --- a/elasticsearch/qa/shield-example-realm/build.gradle +++ b/elasticsearch/qa/shield-example-realm/build.gradle @@ -1,19 +1,38 @@ -apply plugin: 'elasticsearch.esplugin' +import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.VersionProperties -esplugin { - description 'a very basic implementation of a custom realm to validate it works' - classname 'org.elasticsearch.example.ExampleRealmPlugin' - isolated false -} +apply plugin: 'elasticsearch.build' dependencies { + provided "org.elasticsearch:elasticsearch:${versions.elasticsearch}" + testCompile "org.elasticsearch.test:framework:${project.versions.elasticsearch}" provided project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime') } -compileJava.options.compilerArgs << "-Xlint:-rawtypes" -//compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" +Map generateSubstitutions() { + def stringSnap = { version -> + if (version.endsWith("-SNAPSHOT")) { + return version.substring(0, version.length() - 9) + } + return version + } + return [ + 'version': stringSnap(version), + 'xpack.version': stringSnap(VersionProperties.elasticsearch), + 'java.version': targetCompatibility as String + ] +} -integTest { +processResources { + MavenFilteringHack.filter(it, generateSubstitutions()) +} + +task buildZip(type:Zip, dependsOn: [jar]) { + from 'build/resources/main/xpack-extension-descriptor.properties' + from project.jar +} + +task integTest(type: org.elasticsearch.gradle.test.RestIntegTestTask, dependsOn: buildZip) { cluster { plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') // TODO: these should be settings? @@ -24,6 +43,8 @@ integTest { setupCommand 'setupDummyUser', 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin' + setupCommand 'installExtension', + 'bin/xpack/extension', 'install', 'file:' + buildZip.archivePath waitCondition = { node, ant -> File tmpFile = new File(node.cwd, 'wait.success') ant.get(src: "http://${node.httpUri()}", @@ -36,4 +57,5 @@ integTest { } } } - +check.dependsOn integTest +integTest.mustRunAfter precommit \ No newline at end of file diff --git a/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java b/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmExtension.java similarity index 89% rename from elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java rename to elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmExtension.java index da194a919cf..752e0a74b32 100644 --- a/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmPlugin.java +++ b/elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/ExampleRealmExtension.java @@ -8,11 +8,10 @@ package org.elasticsearch.example; import org.elasticsearch.example.realm.CustomAuthenticationFailureHandler; import org.elasticsearch.example.realm.CustomRealm; import org.elasticsearch.example.realm.CustomRealmFactory; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.shield.authc.AuthenticationModule; +import org.elasticsearch.xpack.extensions.XPackExtension; -public class ExampleRealmPlugin extends Plugin { - +public class ExampleRealmExtension extends XPackExtension { @Override public String name() { return "custom realm example"; diff --git a/elasticsearch/qa/shield-example-realm/src/main/resources/xpack-extension-descriptor.properties b/elasticsearch/qa/shield-example-realm/src/main/resources/xpack-extension-descriptor.properties new file mode 100644 index 00000000000..faa674a8b49 --- /dev/null +++ b/elasticsearch/qa/shield-example-realm/src/main/resources/xpack-extension-descriptor.properties @@ -0,0 +1,6 @@ +description=Custom Realm Extension +version=${version} +name=examplerealm +classname=org.elasticsearch.example.ExampleRealmExtension +java.version=${java.version} +xpack.version=${xpack.version} \ No newline at end of file diff --git a/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java b/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java index 2bd0c252abe..2c6b60b7b8f 100644 --- a/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java +++ b/elasticsearch/qa/shield-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.rest.client.http.HttpResponse; @@ -64,6 +65,7 @@ public class CustomRealmIT extends ESIntegTestCase { Settings settings = Settings.builder() .put("cluster.name", clusterName) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER) .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW) .build(); @@ -83,6 +85,7 @@ public class CustomRealmIT extends ESIntegTestCase { Settings settings = Settings.builder() .put("cluster.name", clusterName) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER + randomAsciiOfLength(1)) .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW) .build(); diff --git a/elasticsearch/x-pack/bin/xpack/extension b/elasticsearch/x-pack/bin/xpack/extension new file mode 100755 index 00000000000..35e061c7deb --- /dev/null +++ b/elasticsearch/x-pack/bin/xpack/extension @@ -0,0 +1,116 @@ +#!/bin/sh + +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +SCRIPT="$0" + +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# determine elasticsearch home +ES_HOME=`dirname "$SCRIPT"`/../.. + +# make ELASTICSEARCH_HOME absolute +ES_HOME=`cd "$ES_HOME"; pwd` + +# If an include wasn't specified in the environment, then search for one... +if [ "x$ES_INCLUDE" = "x" ]; then + # Locations (in order) to use when searching for an include file. + for include in /usr/share/elasticsearch/elasticsearch.in.sh \ + /usr/local/share/elasticsearch/elasticsearch.in.sh \ + /opt/elasticsearch/elasticsearch.in.sh \ + ~/.elasticsearch.in.sh \ + "`dirname "$0"`"/../elasticsearch.in.sh \ + "$ES_HOME/bin/elasticsearch.in.sh"; do + if [ -r "$include" ]; then + . "$include" + break + fi + done +# ...otherwise, source the specified include. +elif [ -r "$ES_INCLUDE" ]; then + . "$ES_INCLUDE" +fi + +if [ -x "$JAVA_HOME/bin/java" ]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA=`which java` +fi + +if [ ! -x "$JAVA" ]; then + echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME" + exit 1 +fi + +if [ -z "$ES_CLASSPATH" ]; then + echo "You must set the ES_CLASSPATH var" >&2 + exit 1 +fi + +# Try to read package config files +if [ -f "/etc/sysconfig/elasticsearch" ]; then + CONF_DIR=/etc/elasticsearch + CONF_FILE=$CONF_DIR/elasticsearch.yml + + . "/etc/sysconfig/elasticsearch" +elif [ -f "/etc/default/elasticsearch" ]; then + CONF_DIR=/etc/elasticsearch + CONF_FILE=$CONF_DIR/elasticsearch.yml + + . "/etc/default/elasticsearch" +fi + +# Parse any long getopt options and put them into properties before calling getopt below +# Be dash compatible to make sure running under ubuntu works +ARGCOUNT=$# +COUNT=0 +while [ $COUNT -lt $ARGCOUNT ] +do + case $1 in + --*=*) properties="$properties -Des.${1#--}" + shift 1; COUNT=$(($COUNT+1)) + ;; + --*) properties="$properties -Des.${1#--}=$2" + shift ; shift; COUNT=$(($COUNT+2)) + ;; + *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1)) + esac +done + +# check if properties already has a config file or config dir +if [ -e "$CONF_DIR" ]; then + case "$properties" in + *-Des.default.path.conf=*) ;; + *) + if [ ! -d "$CONF_DIR/xpack" ]; then + echo "ERROR: The configuration directory [$CONF_DIR/xpack] does not exist. The extension tool expects security configuration files in that location." + echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/xpack] exists, please copy the 'xpack' directory to [$CONF_DIR]" + exit 1 + fi + properties="$properties -Des.default.path.conf=$CONF_DIR" + ;; + esac +fi + +export HOSTNAME=`hostname -s` + +# include x-pack jars in classpath +ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/xpack/*" + +cd "$ES_HOME" > /dev/null +"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" $properties org.elasticsearch.xpack.extensions.XPackExtensionCli "$@" +status=$? +cd - > /dev/null +exit $status \ No newline at end of file diff --git a/elasticsearch/x-pack/bin/xpack/extension.bat b/elasticsearch/x-pack/bin/xpack/extension.bat new file mode 100644 index 00000000000..d1385dc6b59 --- /dev/null +++ b/elasticsearch/x-pack/bin/xpack/extension.bat @@ -0,0 +1,9 @@ +@echo off + +rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +rem or more contributor license agreements. Licensed under the Elastic License; +rem you may not use this file except in compliance with the Elastic License. + +PUSHD "%~dp0" +CALL "%~dp0.in.bat" org.elasticsearch.xpack.extensions.XPackExtensionCli %* +POPD \ No newline at end of file diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java index 9719a880d99..bdad3b9ec06 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java @@ -7,12 +7,10 @@ package org.elasticsearch.xpack; import org.elasticsearch.client.Client; import org.elasticsearch.shield.authc.support.SecuredString; -import org.elasticsearch.shield.authc.support.UsernamePasswordToken; import org.elasticsearch.shield.client.SecurityClient; import org.elasticsearch.watcher.client.WatcherClient; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER; diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java index 4b4f3a87737..adcf8768a48 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java @@ -22,15 +22,19 @@ import org.elasticsearch.marvel.Marvel; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.shield.Shield; +import org.elasticsearch.shield.authc.AuthenticationModule; import org.elasticsearch.watcher.Watcher; import org.elasticsearch.xpack.common.init.LazyInitializationModule; import org.elasticsearch.xpack.common.init.LazyInitializationService; +import org.elasticsearch.xpack.extensions.XPackExtension; +import org.elasticsearch.xpack.extensions.XPackExtensionsService; import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; public class XPackPlugin extends Plugin { @@ -67,6 +71,7 @@ public class XPackPlugin extends Plugin { } protected final Settings settings; + protected final XPackExtensionsService extensionsService; protected Licensing licensing; protected Shield shield; @@ -81,6 +86,14 @@ public class XPackPlugin extends Plugin { this.marvel = new Marvel(settings); this.watcher = new Watcher(settings); this.graph = new Graph(settings); + // Check if the node is a transport client. + if (transportClientMode(settings) == false) { + Environment env = new Environment(settings); + this.extensionsService = + new XPackExtensionsService(settings, resolveXPackExtensionsFile(env), getExtensions()); + } else { + this.extensionsService = null; + } } @Override public String name() { @@ -91,6 +104,11 @@ public class XPackPlugin extends Plugin { return "Elastic X-Pack"; } + // For tests only + public Collection> getExtensions() { + return Collections.emptyList(); + } + @Override public Collection nodeModules() { ArrayList modules = new ArrayList<>(); @@ -157,6 +175,12 @@ public class XPackPlugin extends Plugin { graph.onModule(module); } + public void onModule(AuthenticationModule module) { + if (extensionsService != null) { + extensionsService.onModule(module); + } + } + public void onIndexModule(IndexModule module) { shield.onIndexModule(module); graph.onIndexModule(module); @@ -221,4 +245,8 @@ public class XPackPlugin extends Plugin { settingsModule.registerSetting(Setting.boolSetting(legacyFeatureEnabledSetting(featureName), defaultValue, Setting.Property.NodeScope)); } + + public static Path resolveXPackExtensionsFile(Environment env) { + return env.pluginsFile().resolve("xpack").resolve("extensions"); + } } diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java new file mode 100644 index 00000000000..749cc745760 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; +import org.apache.lucene.util.IOUtils; + +import org.elasticsearch.bootstrap.JarHell; +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.env.Environment; + +import java.io.InputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Arrays; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile; +import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; + +/** + * A command for the extension cli to install an extension into x-pack. + * + * The install command takes a URL to an extension zip. + * + * Extensions are packaged as zip files. Each packaged extension must contain an + * extension properties file. See {@link XPackExtensionInfo}. + *

+ * The installation process first extracts the extensions files into a temporary + * directory in order to verify the extension satisfies the following requirements: + *

    + *
  • The property file exists and contains valid metadata. See {@link XPackExtensionInfo#readFromProperties(Path)}
    • + *
  • Jar hell does not exist, either between the extension's own jars or with the parent classloader (elasticsearch + xpack)
    • + *
+ */ +class InstallXPackExtensionCommand extends Command { + + private final Environment env; + private final OptionSpec batchOption; + private final OptionSpec arguments; + + InstallXPackExtensionCommand(Environment env) { + super("Install a plugin"); + this.env = env; + this.batchOption = parser.acceptsAll(Arrays.asList("b", "batch"), + "Enable batch mode explicitly, automatic confirmation of security permission"); + this.arguments = parser.nonOptions("plugin id"); + } + + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args + List args = arguments.values(options); + if (args.size() != 1) { + throw new UserError(ExitCodes.USAGE, "Must supply a single extension id argument"); + } + String extensionURL = args.get(0); + boolean isBatch = options.has(batchOption) || System.console() == null; + execute(terminal, extensionURL, isBatch); + } + + + // pkg private for testing + void execute(Terminal terminal, String extensionId, boolean isBatch) throws Exception { + if (Files.exists(resolveXPackExtensionsFile(env)) == false) { + terminal.println("xpack extensions directory [" + resolveXPackExtensionsFile(env) + "] does not exist. Creating..."); + Files.createDirectories(resolveXPackExtensionsFile(env)); + } + + Path extensionZip = download(terminal, extensionId, env.tmpFile()); + Path extractedZip = unzip(extensionZip, resolveXPackExtensionsFile(env)); + install(terminal, extractedZip, env); + } + + /** Downloads the extension and returns the file it was downloaded to. */ + private Path download(Terminal terminal, String extensionURL, Path tmpDir) throws Exception { + terminal.println("-> Downloading " + URLDecoder.decode(extensionURL, "UTF-8")); + URL url = new URL(extensionURL); + Path zip = Files.createTempFile(tmpDir, null, ".zip"); + try (InputStream in = url.openStream()) { + // must overwrite since creating the temp file above actually created the file + Files.copy(in, zip, StandardCopyOption.REPLACE_EXISTING); + } + return zip; + } + + private Path unzip(Path zip, Path extensionDir) throws IOException, UserError { + // unzip extension to a staging temp dir + Path target = Files.createTempDirectory(extensionDir, ".installing-"); + Files.createDirectories(target); + + // TODO: we should wrap this in a try/catch and try deleting the target dir on failure? + try (ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) { + ZipEntry entry; + byte[] buffer = new byte[8192]; + while ((entry = zipInput.getNextEntry()) != null) { + Path targetFile = target.resolve(entry.getName()); + // TODO: handle name being an absolute path + + // be on the safe side: do not rely on that directories are always extracted + // before their children (although this makes sense, but is it guaranteed?) + Files.createDirectories(targetFile.getParent()); + if (entry.isDirectory() == false) { + try (OutputStream out = Files.newOutputStream(targetFile)) { + int len; + while((len = zipInput.read(buffer)) >= 0) { + out.write(buffer, 0, len); + } + } + } + zipInput.closeEntry(); + } + } + Files.delete(zip); + return target; + } + + /** Load information about the extension, and verify it can be installed with no errors. */ + private XPackExtensionInfo verify(Terminal terminal, Path extensionRoot, Environment env) throws Exception { + // read and validate the extension descriptor + XPackExtensionInfo info = XPackExtensionInfo.readFromProperties(extensionRoot); + terminal.println(VERBOSE, info.toString()); + + // check for jar hell before any copying + jarHellCheck(extensionRoot); + return info; + } + + /** check a candidate extension for jar hell before installing it */ + private void jarHellCheck(Path candidate) throws Exception { + // create list of current jars in classpath + // including the x-pack jars (see $ES_CLASSPATH in bin/extension script) + final List jars = new ArrayList<>(); + jars.addAll(Arrays.asList(JarHell.parseClassPath())); + + // add extension jars to the list + Path extensionJars[] = FileSystemUtils.files(candidate, "*.jar"); + for (Path jar : extensionJars) { + jars.add(jar.toUri().toURL()); + } + // TODO: no jars should be an error + // TODO: verify the classname exists in one of the jars! + + // check combined (current classpath + new jars to-be-added) + JarHell.checkJarHell(jars.toArray(new URL[jars.size()])); + } + + /** + * Installs the extension from {@code tmpRoot} into the extensions dir. + */ + private void install(Terminal terminal, Path tmpRoot, Environment env) throws Exception { + List deleteOnFailure = new ArrayList<>(); + deleteOnFailure.add(tmpRoot); + try { + XPackExtensionInfo info = verify(terminal, tmpRoot, env); + final Path destination = resolveXPackExtensionsFile(env).resolve(info.getName()); + if (Files.exists(destination)) { + throw new UserError(ExitCodes.USAGE, + "extension directory " + destination.toAbsolutePath() + + " already exists. To update the extension, uninstall it first using 'remove " + + info.getName() + "' command"); + } + Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE); + terminal.println("-> Installed " + info.getName()); + } catch (Exception installProblem) { + try { + IOUtils.rm(deleteOnFailure.toArray(new Path[0])); + } catch (IOException exceptionWhileRemovingFiles) { + installProblem.addSuppressed(exceptionWhileRemovingFiles); + } + throw installProblem; + } + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java new file mode 100644 index 00000000000..ccf7768fb95 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import joptsimple.OptionSet; + +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.env.Environment; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile; +import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; + +/** + * A command for the extension cli to list extensions installed in x-pack. + */ +class ListXPackExtensionCommand extends Command { + private final Environment env; + + ListXPackExtensionCommand(Environment env) { + super("Lists installed x-pack extensions"); + this.env = env; + } + + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + if (Files.exists(resolveXPackExtensionsFile(env)) == false) { + throw new IOException("Extensions directory missing: " + resolveXPackExtensionsFile(env)); + } + + terminal.println(VERBOSE, "Extensions directory: " + resolveXPackExtensionsFile(env)); + try (DirectoryStream stream = Files.newDirectoryStream(resolveXPackExtensionsFile(env))) { + for (Path extension : stream) { + terminal.println(extension.getFileName().toString()); + } + } + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java new file mode 100644 index 00000000000..4b51adbaa82 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.Strings; +import org.elasticsearch.env.Environment; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile; +import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; + +/** + * A command for the extension cli to remove an extension from x-pack. + */ +class RemoveXPackExtensionCommand extends Command { + private final Environment env; + private final OptionSpec arguments; + + RemoveXPackExtensionCommand(Environment env) { + super("Removes an extension from x-pack"); + this.env = env; + this.arguments = parser.nonOptions("extension name"); + } + + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args + List args = arguments.values(options); + if (args.size() != 1) { + throw new UserError(ExitCodes.USAGE, "Must supply a single extension id argument"); + } + execute(terminal, args.get(0)); + } + + // pkg private for testing + void execute(Terminal terminal, String extensionName) throws Exception { + terminal.println("-> Removing " + Strings.coalesceToEmpty(extensionName) + "..."); + + Path extensionDir = resolveXPackExtensionsFile(env).resolve(extensionName); + if (Files.exists(extensionDir) == false) { + throw new UserError(ExitCodes.USAGE, + "Extension " + extensionName + " not found. Run 'bin/xpack/extension list' to get list of installed extensions."); + } + + List extensionPaths = new ArrayList<>(); + + terminal.println(VERBOSE, "Removing: " + extensionDir); + Path tmpExtensionDir = resolveXPackExtensionsFile(env).resolve(".removing-" + extensionName); + Files.move(extensionDir, tmpExtensionDir, StandardCopyOption.ATOMIC_MOVE); + extensionPaths.add(tmpExtensionDir); + + IOUtils.rm(extensionPaths.toArray(new Path[extensionPaths.size()])); + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtension.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtension.java new file mode 100644 index 00000000000..f1bf62a0b78 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtension.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.shield.authc.AuthenticationModule; + + +/** + * An extension point allowing to plug in custom functionality in x-pack authentication module. + */ +public abstract class XPackExtension { + /** + * The name of the plugin. + */ + public abstract String name(); + + /** + * The description of the plugin. + */ + public abstract String description(); + + /** + * Implement this function to register custom extensions in the authentication module. + */ + public void onModule(AuthenticationModule module) {} +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java new file mode 100644 index 00000000000..f18d3db3a02 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.varia.NullAppender; +import org.elasticsearch.cli.MultiCommand; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.internal.InternalSettingsPreparer; + +/** + * A cli tool for adding, removing and listing extensions for x-pack. + */ +public class XPackExtensionCli extends MultiCommand { + + public XPackExtensionCli(Environment env) { + super("A tool for managing installed x-pack extensions"); + subcommands.put("list", new ListXPackExtensionCommand(env)); + subcommands.put("install", new InstallXPackExtensionCommand(env)); + subcommands.put("remove", new RemoveXPackExtensionCommand(env)); + } + + public static void main(String[] args) throws Exception { + BasicConfigurator.configure(new NullAppender()); + Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, Terminal.DEFAULT); + exit(new XPackExtensionCli(env).main(args, Terminal.DEFAULT)); + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionInfo.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionInfo.java new file mode 100644 index 00000000000..08a3eb01626 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionInfo.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.Version; +import org.elasticsearch.bootstrap.JarHell; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +public class XPackExtensionInfo { + public static final String XPACK_EXTENSION_PROPERTIES = "xpack-extension-descriptor.properties"; + + private String name; + private String description; + private String version; + private String classname; + + public XPackExtensionInfo() { + } + + /** + * Information about extensions + * + * @param name Its name + * @param description Its description + * @param version Version number + */ + XPackExtensionInfo(String name, String description, String version, String classname) { + this.name = name; + this.description = description; + this.version = version; + this.classname = classname; + } + + /** reads (and validates) extension metadata descriptor file */ + public static XPackExtensionInfo readFromProperties(Path dir) throws IOException { + Path descriptor = dir.resolve(XPACK_EXTENSION_PROPERTIES); + Properties props = new Properties(); + try (InputStream stream = Files.newInputStream(descriptor)) { + props.load(stream); + } + String name = props.getProperty("name"); + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException("Property [name] is missing in [" + descriptor + "]"); + } + String description = props.getProperty("description"); + if (description == null) { + throw new IllegalArgumentException("Property [description] is missing for extension [" + name + "]"); + } + String version = props.getProperty("version"); + if (version == null) { + throw new IllegalArgumentException("Property [version] is missing for extension [" + name + "]"); + } + + String xpackVersionString = props.getProperty("xpack.version"); + if (xpackVersionString == null) { + throw new IllegalArgumentException("Property [xpack.version] is missing for extension [" + name + "]"); + } + Version xpackVersion = Version.fromString(xpackVersionString); + if (xpackVersion.equals(Version.CURRENT) == false) { + throw new IllegalArgumentException("extension [" + name + "] is incompatible with Elasticsearch [" + + Version.CURRENT.toString() + "]. Was designed for version [" + xpackVersionString + "]"); + } + String javaVersionString = props.getProperty("java.version"); + if (javaVersionString == null) { + throw new IllegalArgumentException("Property [java.version] is missing for extension [" + name + "]"); + } + JarHell.checkVersionFormat(javaVersionString); + JarHell.checkJavaVersion(name, javaVersionString); + String classname = props.getProperty("classname"); + if (classname == null) { + throw new IllegalArgumentException("Property [classname] is missing for extension [" + name + "]"); + } + + return new XPackExtensionInfo(name, description, version, classname); + } + + /** + * @return Extension's name + */ + public String getName() { + return name; + } + + /** + * @return Extension's description if any + */ + public String getDescription() { + return description; + } + + /** + * @return extension's classname + */ + public String getClassname() { + return classname; + } + + /** + * @return Version number for the extension + */ + public String getVersion() { + return version; + } + + @Override + public String toString() { + final StringBuilder information = new StringBuilder() + .append("- XPack Extension information:\n") + .append("Name: ").append(name).append("\n") + .append("Description: ").append(description).append("\n") + .append("Version: ").append(version).append("\n") + .append(" * Classname: ").append(classname); + + return information.toString(); + } +} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionsService.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionsService.java new file mode 100644 index 00000000000..42ff95f39f3 --- /dev/null +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionsService.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.bootstrap.JarHell; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.shield.authc.AuthenticationModule; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Arrays; + +import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; + +/** + * + */ +public class XPackExtensionsService { + private final Settings settings; + + /** + * We keep around a list of extensions + */ + private final List > extensions; + + /** + * Constructs a new XPackExtensionsService + * @param settings The settings of the system + * @param extsDirectory The directory extensions exist in, or null if extensions should not be loaded from the filesystem + * @param classpathExtensions Extensions that exist in the classpath which should be loaded + */ + public XPackExtensionsService(Settings settings, Path extsDirectory, Collection> classpathExtensions) { + this.settings = settings; + List> extensionsLoaded = new ArrayList<>(); + + // first we load extensions that are on the classpath. this is for tests + for (Class extClass : classpathExtensions) { + XPackExtension ext = loadExtension(extClass, settings); + XPackExtensionInfo extInfo = new XPackExtensionInfo(ext.name(), ext.description(), "NA", extClass.getName()); + extensionsLoaded.add(new Tuple<>(extInfo, ext)); + } + + // now, find all the ones that are in plugins/xpack/extensions + if (extsDirectory != null) { + try { + List bundles = getExtensionBundles(extsDirectory); + List> loaded = loadBundles(bundles); + extensionsLoaded.addAll(loaded); + } catch (IOException ex) { + throw new IllegalStateException("Unable to initialize extensions", ex); + } + } + extensions = Collections.unmodifiableList(extensionsLoaded); + } + + public void onModule(AuthenticationModule module) { + for (Tuple tuple : extensions) { + tuple.v2().onModule(module); + } + } + + // a "bundle" is a an extension in a single classloader. + static class Bundle { + XPackExtensionInfo info; + List urls = new ArrayList<>(); + } + + static List getExtensionBundles(Path extsDirectory) throws IOException { + ESLogger logger = Loggers.getLogger(XPackExtensionsService.class); + + // TODO: remove this leniency, but tests bogusly rely on it + if (!isAccessibleDirectory(extsDirectory, logger)) { + return Collections.emptyList(); + } + + List bundles = new ArrayList<>(); + + try (DirectoryStream stream = Files.newDirectoryStream(extsDirectory)) { + for (Path extension : stream) { + if (FileSystemUtils.isHidden(extension)) { + logger.trace("--- skip hidden extension file[{}]", extension.toAbsolutePath()); + continue; + } + logger.trace("--- adding extension [{}]", extension.toAbsolutePath()); + final XPackExtensionInfo info; + try { + info = XPackExtensionInfo.readFromProperties(extension); + } catch (IOException e) { + throw new IllegalStateException("Could not load extension descriptor for existing extension [" + + extension.getFileName() + "]. Was the extension built before 2.0?", e); + } + + List urls = new ArrayList<>(); + try (DirectoryStream jarStream = Files.newDirectoryStream(extension, "*.jar")) { + for (Path jar : jarStream) { + // normalize with toRealPath to get symlinks out of our hair + urls.add(jar.toRealPath().toUri().toURL()); + } + } + final Bundle bundle = new Bundle(); + bundles.add(bundle); + bundle.info = info; + bundle.urls.addAll(urls); + } + } + + return bundles; + } + + private List > loadBundles(List bundles) { + List> exts = new ArrayList<>(); + + for (Bundle bundle : bundles) { + // jar-hell check the bundle against the parent classloader and the x-pack classloader + // pluginmanager does it, but we do it again, in case lusers mess with jar files manually + try { + final List jars = new ArrayList<>(); + // add the parent jars to the list + jars.addAll(Arrays.asList(JarHell.parseClassPath())); + + // add the x-pack jars to the list + ClassLoader xpackLoader = getClass().getClassLoader(); + // this class is loaded from the isolated x-pack plugin's classloader + if (xpackLoader instanceof URLClassLoader) { + jars.addAll(Arrays.asList(((URLClassLoader) xpackLoader).getURLs())); + } + + jars.addAll(bundle.urls); + + JarHell.checkJarHell(jars.toArray(new URL[0])); + } catch (Exception e) { + throw new IllegalStateException("failed to load bundle " + bundle.urls + " due to jar hell", e); + } + + // create a child to load the extension in this bundle + ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), getClass().getClassLoader()); + final Class extClass = loadExtensionClass(bundle.info.getClassname(), loader); + final XPackExtension ext = loadExtension(extClass, settings); + exts.add(new Tuple<>(bundle.info, ext)); + } + + return Collections.unmodifiableList(exts); + } + + private Class loadExtensionClass(String className, ClassLoader loader) { + try { + return loader.loadClass(className).asSubclass(XPackExtension.class); + } catch (ClassNotFoundException e) { + throw new ElasticsearchException("Could not find extension class [" + className + "]", e); + } + } + + private XPackExtension loadExtension(Class extClass, Settings settings) { + try { + try { + return extClass.getConstructor(Settings.class).newInstance(settings); + } catch (NoSuchMethodException e) { + try { + return extClass.getConstructor().newInstance(); + } catch (NoSuchMethodException e1) { + throw new ElasticsearchException("No constructor for [" + extClass + "]. An extension class must " + + "have either an empty default constructor or a single argument constructor accepting a " + + "Settings instance"); + } + } + } catch (Throwable e) { + throw new ElasticsearchException("Failed to load extension class [" + extClass.getName() + "]", e); + } + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java new file mode 100644 index 00000000000..e86e7112545 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.Version; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.DirectoryStream; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.FileVisitResult; +import java.nio.file.NoSuchFileException; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@LuceneTestCase.SuppressFileSystems("*") +public class InstallXPackExtensionCommandTests extends ESTestCase { + /** + * Creates a test environment with plugins and xpack extensions directories. + */ + static Environment createEnv() throws IOException { + Path home = createTempDir(); + Files.createDirectories(home.resolve("org/elasticsearch/xpack/extensions").resolve("xpack").resolve("extensions")); + Settings settings = Settings.builder() + .put("path.home", home) + .build(); + return new Environment(settings); + } + + /** + * creates a fake jar file with empty class files + */ + static void writeJar(Path jar, String... classes) throws IOException { + try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) { + for (String clazz : classes) { + stream.putNextEntry(new ZipEntry(clazz + ".class")); // no package names, just support simple classes + } + } + } + + static String writeZip(Path structure) throws IOException { + Path zip = createTempDir().resolve(structure.getFileName() + ".zip"); + try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) { + Files.walkFileTree(structure, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + String target = structure.relativize(file).toString(); + stream.putNextEntry(new ZipEntry(target)); + Files.copy(file, stream); + return FileVisitResult.CONTINUE; + } + }); + } + return zip.toUri().toURL().toString(); + } + + /** + * creates an extension .zip and returns the url for testing + */ + static String createExtension(String name, Path structure) throws IOException { + XPackExtensionTestUtil.writeProperties(structure, + "description", "fake desc", + "name", name, + "version", "1.0", + "xpack.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "FakeExtension"); + writeJar(structure.resolve("extension.jar"), "FakeExtension"); + return writeZip(structure); + } + + static MockTerminal installExtension(String extensionUrl, Environment env) throws Exception { + MockTerminal terminal = new MockTerminal(); + new InstallXPackExtensionCommand(env).execute(terminal, extensionUrl, true); + return terminal; + } + + void assertExtension(String name, Path original, Environment env) throws IOException { + Path got = env.pluginsFile().resolve("xpack").resolve("extensions").resolve(name); + assertTrue("dir " + name + " exists", Files.exists(got)); + assertTrue("jar was copied", Files.exists(got.resolve("extension.jar"))); + assertInstallCleaned(env); + } + + void assertInstallCleaned(Environment env) throws IOException { + try (DirectoryStream stream = Files.newDirectoryStream(env.pluginsFile().resolve("xpack").resolve("extensions"))) { + for (Path file : stream) { + if (file.getFileName().toString().startsWith(".installing")) { + fail("Installation dir still exists, " + file); + } + } + } + } + + public void testSomethingWorks() throws Exception { + Environment env = createEnv(); + Path extDir = createTempDir(); + String extZip = createExtension("fake", extDir); + installExtension(extZip, env); + assertExtension("fake", extDir, env); + } + + public void testSpaceInUrl() throws Exception { + Environment env = createEnv(); + Path extDir = createTempDir(); + String extZip = createExtension("fake", extDir); + Path extZipWithSpaces = createTempFile("foo bar", ".zip"); + try (InputStream in = new URL(extZip).openStream()) { + Files.copy(in, extZipWithSpaces, StandardCopyOption.REPLACE_EXISTING); + } + installExtension(extZipWithSpaces.toUri().toURL().toString(), env); + assertExtension("fake", extDir, env); + } + + public void testMalformedUrlNotMaven() throws Exception { + // has two colons, so it appears similar to maven coordinates + MalformedURLException e = expectThrows(MalformedURLException.class, () -> { + installExtension("://host:1234", createEnv()); + }); + assertTrue(e.getMessage(), e.getMessage().contains("no protocol")); + } + + public void testJarHell() throws Exception { + Environment env = createEnv(); + Path extDir = createTempDir(); + writeJar(extDir.resolve("other.jar"), "FakeExtension"); + String extZip = createExtension("fake", extDir); // adds extension.jar with FakeExtension + IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + installExtension(extZip, env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); + assertInstallCleaned(env); + } + + public void testIsolatedExtension() throws Exception { + Environment env = createEnv(); + // these both share the same FakeExtension class + Path extDir1 = createTempDir(); + String extZip1 = createExtension("fake1", extDir1); + installExtension(extZip1, env); + Path extDir2 = createTempDir(); + String extZip2 = createExtension("fake2", extDir2); + installExtension(extZip2, env); + assertExtension("fake1", extDir1, env); + assertExtension("fake2", extDir2, env); + } + + public void testExistingExtension() throws Exception { + Environment env = createEnv(); + String extZip = createExtension("fake", createTempDir()); + installExtension(extZip, env); + UserError e = expectThrows(UserError.class, () -> { + installExtension(extZip, env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("already exists")); + assertInstallCleaned(env); + } + + public void testMissingDescriptor() throws Exception { + Environment env = createEnv(); + Path extDir = createTempDir(); + Files.createFile(extDir.resolve("fake.yml")); + String extZip = writeZip(extDir); + NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> { + installExtension(extZip, env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("xpack-extension-descriptor.properties")); + assertInstallCleaned(env); + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java new file mode 100644 index 00000000000..2f61e219687 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +@LuceneTestCase.SuppressFileSystems("*") +public class ListXPackExtensionCommandTests extends ESTestCase { + + Environment createEnv() throws IOException { + Path home = createTempDir(); + Settings settings = Settings.builder() + .put("path.home", home) + .build(); + return new Environment(settings); + } + + Path createExtensionDir(Environment env) throws IOException { + Path path = env.pluginsFile().resolve("xpack").resolve("extensions"); + return Files.createDirectories(path); + } + + static MockTerminal listExtensions(Environment env) throws Exception { + MockTerminal terminal = new MockTerminal(); + String[] args = {}; + int status = new ListXPackExtensionCommand(env).main(args, terminal); + assertEquals(ExitCodes.OK, status); + return terminal; + } + + public void testExtensionsDirMissing() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + Files.delete(extDir); + IOException e = expectThrows(IOException.class, () -> { + listExtensions(env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("Extensions directory missing")); + } + + public void testNoExtensions() throws Exception { + Environment env = createEnv(); + createExtensionDir(env); + MockTerminal terminal = listExtensions(env); + assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty()); + } + + public void testOneExtension() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + Files.createDirectory(extDir.resolve("fake")); + MockTerminal terminal = listExtensions(env); + assertTrue(terminal.getOutput(), terminal.getOutput().contains("fake")); + } + + public void testTwoExtensions() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + Files.createDirectory(extDir.resolve("fake1")); + Files.createDirectory(extDir.resolve("fake2")); + MockTerminal terminal = listExtensions(env); + String output = terminal.getOutput(); + assertTrue(output, output.contains("fake1")); + assertTrue(output, output.contains("fake2")); + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java new file mode 100644 index 00000000000..aa5a27d1f76 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; + +@LuceneTestCase.SuppressFileSystems("*") +public class RemoveXPackExtensionCommandTests extends ESTestCase { + + /** Creates a test environment with bin, config and plugins directories. */ + static Environment createEnv() throws IOException { + Path home = createTempDir(); + Settings settings = Settings.builder() + .put("path.home", home) + .build(); + return new Environment(settings); + } + + Path createExtensionDir(Environment env) throws IOException { + Path path = env.pluginsFile().resolve("xpack").resolve("extensions"); + return Files.createDirectories(path); + } + + static MockTerminal removeExtension(String name, Environment env) throws Exception { + MockTerminal terminal = new MockTerminal(); + new RemoveXPackExtensionCommand(env).execute(terminal, name); + return terminal; + } + + static void assertRemoveCleaned(Path extDir) throws IOException { + try (DirectoryStream stream = Files.newDirectoryStream(extDir)) { + for (Path file : stream) { + if (file.getFileName().toString().startsWith(".removing")) { + fail("Removal dir still exists, " + file); + } + } + } + } + + public void testMissing() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + UserError e = expectThrows(UserError.class, () -> { + removeExtension("dne", env); + }); + assertTrue(e.getMessage(), e.getMessage().contains("Extension dne not found")); + assertRemoveCleaned(extDir); + } + + public void testBasic() throws Exception { + Environment env = createEnv(); + Path extDir = createExtensionDir(env); + Files.createDirectory(extDir.resolve("fake")); + Files.createFile(extDir.resolve("fake").resolve("extension.jar")); + Files.createDirectory(extDir.resolve("fake").resolve("subdir")); + Files.createDirectory(extDir.resolve("other")); + removeExtension("fake", env); + assertFalse(Files.exists(extDir.resolve("fake"))); + assertTrue(Files.exists(extDir.resolve("other"))); + assertRemoveCleaned(extDir); + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionInfoTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionInfoTests.java new file mode 100644 index 00000000000..63c289d3592 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionInfoTests.java @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.Version; +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Path; + +public class XPackExtensionInfoTests extends ESTestCase { + + public void testReadFromProperties() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "version", "1.0", + "xpack.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "FakeExtension"); + XPackExtensionInfo info = XPackExtensionInfo.readFromProperties(extensionDir); + assertEquals("my_extension", info.getName()); + assertEquals("fake desc", info.getDescription()); + assertEquals("1.0", info.getVersion()); + assertEquals("FakeExtension", info.getClassname()); + } + + public void testReadFromPropertiesNameMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("Property [name] is missing in")); + XPackExtensionTestUtil.writeProperties(extensionDir, "name", ""); + IllegalArgumentException e1 = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e1.getMessage().contains("Property [name] is missing in")); + } + + public void testReadFromPropertiesDescriptionMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, "name", "fake-extension"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("[description] is missing")); + } + + public void testReadFromPropertiesVersionMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, "description", "fake desc", "name", "fake-extension"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("[version] is missing")); + } + + public void testReadFromPropertiesElasticsearchVersionMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "version", "1.0"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("[xpack.version] is missing")); + } + + public void testReadFromPropertiesJavaVersionMissing() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "xpack.version", Version.CURRENT.toString(), + "version", "1.0"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("[java.version] is missing")); + } + + public void testReadFromPropertiesJavaVersionIncompatible() throws Exception { + String extensionName = "fake-extension"; + Path extensionDir = createTempDir().resolve(extensionName); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", extensionName, + "xpack.version", Version.CURRENT.toString(), + "java.version", "1000000.0", + "classname", "FakeExtension", + "version", "1.0"); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage(), e.getMessage().contains(extensionName + " requires Java")); + } + + public void testReadFromPropertiesBadJavaVersionFormat() throws Exception { + String extensionName = "fake-extension"; + Path extensionDir = createTempDir().resolve(extensionName); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", extensionName, + "xpack.version", Version.CURRENT.toString(), + "java.version", "1.7.0_80", + "classname", "FakeExtension", + "version", "1.0"); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage(), + e.getMessage().equals("version string must be a sequence of nonnegative decimal " + + "integers separated by \".\"'s and may have leading zeros but was 1.7.0_80")); + } + + public void testReadFromPropertiesBogusElasticsearchVersion() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "version", "1.0", + "name", "my_extension", + "xpack.version", "bogus"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("version needs to contain major, minor, and revision")); + } + + public void testReadFromPropertiesOldElasticsearchVersion() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "version", "1.0", + "xpack.version", Version.V_2_0_0.toString()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("Was designed for version [2.0.0]")); + } + + public void testReadFromPropertiesJvmMissingClassname() throws Exception { + Path extensionDir = createTempDir().resolve("fake-extension"); + XPackExtensionTestUtil.writeProperties(extensionDir, + "description", "fake desc", + "name", "my_extension", + "version", "1.0", + "xpack.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + XPackExtensionInfo.readFromProperties(extensionDir); + }); + assertTrue(e.getMessage().contains("Property [classname] is missing")); + } +} \ No newline at end of file diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionTestUtil.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionTestUtil.java new file mode 100644 index 00000000000..866b6e557e9 --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionTestUtil.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +/** Utility methods for testing extensions */ +public class XPackExtensionTestUtil { + + /** convenience method to write a plugin properties file */ + public static void writeProperties(Path pluginDir, String... stringProps) throws IOException { + assert stringProps.length % 2 == 0; + Files.createDirectories(pluginDir); + Path propertiesFile = pluginDir.resolve(XPackExtensionInfo.XPACK_EXTENSION_PROPERTIES); + Properties properties = new Properties(); + for (int i = 0; i < stringProps.length; i += 2) { + properties.put(stringProps[i], stringProps[i + 1]); + } + try (OutputStream out = Files.newOutputStream(propertiesFile)) { + properties.store(out, ""); + } + } +} diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionsServiceTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionsServiceTests.java new file mode 100644 index 00000000000..7730ec7a4dd --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/XPackExtensionsServiceTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.extensions; + +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Files; +import java.nio.file.Path; + +public class XPackExtensionsServiceTests extends ESTestCase { + public void testExistingPluginMissingDescriptor() throws Exception { + Path extensionsDir = createTempDir(); + Files.createDirectory(extensionsDir.resolve("extension-missing-descriptor")); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + XPackExtensionsService.getExtensionBundles(extensionsDir); + }); + assertTrue(e.getMessage(), + e.getMessage().contains("Could not load extension descriptor for existing extension")); + } +} From 0f8f70a404024be2a9a77668d804b473887044a2 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 22 Mar 2016 14:13:33 +0100 Subject: [PATCH 2/8] Fix windows build Avoid empty elements in x-pack bat script classpath to make JarHell happy Original commit: elastic/x-pack-elasticsearch@06dd95b8ca020449d5fb9b552711e75b98b275d2 --- elasticsearch/x-pack/bin/xpack/.in.bat | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/elasticsearch/x-pack/bin/xpack/.in.bat b/elasticsearch/x-pack/bin/xpack/.in.bat index 638065040a6..465109449b0 100644 --- a/elasticsearch/x-pack/bin/xpack/.in.bat +++ b/elasticsearch/x-pack/bin/xpack/.in.bat @@ -76,7 +76,12 @@ REM JAVA_OPTS=%JAVA_OPTS% -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof REM Disables explicit GC set JAVA_OPTS=%JAVA_OPTS% -XX:+DisableExplicitGC -set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/xpack/* +REM Avoid empty elements in classpath to make JarHell happy +if "%ES_CLASSPATH%" == "" ( + set ES_CLASSPATH=%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/xpack/* +) else ( + set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/xpack/* +) set ES_PARAMS=-Des.path.home="%ES_HOME%" SET HOSTNAME=%COMPUTERNAME% From cc152a867a02f7f03eec73922769175bd6a93330 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 22 Mar 2016 15:07:03 +0100 Subject: [PATCH 3/8] Remove sigar from the x-pack windows script classpath Original commit: elastic/x-pack-elasticsearch@247e945ff5968ce9b85d30966ad3a3978a27956b --- elasticsearch/x-pack/bin/xpack/.in.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch/x-pack/bin/xpack/.in.bat b/elasticsearch/x-pack/bin/xpack/.in.bat index 465109449b0..9d09123147f 100644 --- a/elasticsearch/x-pack/bin/xpack/.in.bat +++ b/elasticsearch/x-pack/bin/xpack/.in.bat @@ -78,9 +78,9 @@ REM Disables explicit GC set JAVA_OPTS=%JAVA_OPTS% -XX:+DisableExplicitGC REM Avoid empty elements in classpath to make JarHell happy if "%ES_CLASSPATH%" == "" ( - set ES_CLASSPATH=%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/xpack/* + set ES_CLASSPATH=%ES_HOME%/lib/*;%ES_HOME%/plugins/xpack/* ) else ( - set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/xpack/* + set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/*;%ES_HOME%/plugins/xpack/* ) set ES_PARAMS=-Des.path.home="%ES_HOME%" From 1fa22c921aaa2210c598d337c2dca25c21b5a342 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 22 Mar 2016 15:27:30 +0100 Subject: [PATCH 4/8] xpack plugin can be isolated now that we have xpack extension support Original commit: elastic/x-pack-elasticsearch@9f742c754f084976c11d646749828009a705a003 --- elasticsearch/x-pack/build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/elasticsearch/x-pack/build.gradle b/elasticsearch/x-pack/build.gradle index d23c14eed0b..8e68915e7b0 100644 --- a/elasticsearch/x-pack/build.gradle +++ b/elasticsearch/x-pack/build.gradle @@ -6,8 +6,6 @@ esplugin { name 'xpack' description 'Elasticsearch Expanded Pack Plugin' classname 'org.elasticsearch.xpack.XPackPlugin' - // FIXME we still can't be isolated due to shield custom realms - isolated false } ext.versions = [ From 40dc7479681afe6d810b65e85b4cb057edfd555f Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 8 Mar 2016 19:54:41 +0100 Subject: [PATCH 5/8] Monitoring: Add MonitoringClientProxy Similar to WatcherClientProxy, the elasticsearch client used in exporters must be proxied to avoid circular dependencies at Guice's injection time. This commit add a MonitoringClientProxy as well as a MonitoringClient to be used later in monitoring's transport actions. (cherry picked from commit b70c095) Original commit: elastic/x-pack-elasticsearch@17327cffe559cec9a4b0314de8e09c35eb52c331 --- .../java/org/elasticsearch/marvel/Marvel.java | 10 +++++++ .../agent/exporter/local/LocalBulk.java | 6 ++--- .../agent/exporter/local/LocalExporter.java | 12 ++++----- .../marvel/client/MonitoringClient.java | 21 +++++++++++++++ .../marvel/client/MonitoringClientModule.java | 16 +++++++++++ .../init/proxy/MonitoringClientProxy.java | 27 +++++++++++++++++++ .../marvel/MarvelPluginClientTests.java | 2 +- .../marvel/agent/exporter/ExportersTests.java | 4 +-- .../marvel/test/MarvelIntegTestCase.java | 7 +++++ .../org/elasticsearch/xpack/XPackClient.java | 8 ++++++ .../org/elasticsearch/xpack/XPackPlugin.java | 1 + .../xpack/common/init/proxy/ClientProxy.java | 5 ++++ 12 files changed, 107 insertions(+), 12 deletions(-) create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClient.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClientModule.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/support/init/proxy/MonitoringClientProxy.java diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Marvel.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Marvel.java index 76fd263ebd7..5ba5e8f7848 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Marvel.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Marvel.java @@ -16,9 +16,12 @@ import org.elasticsearch.marvel.agent.AgentService; import org.elasticsearch.marvel.agent.collector.CollectorModule; import org.elasticsearch.marvel.agent.exporter.ExporterModule; import org.elasticsearch.marvel.cleaner.CleanerService; +import org.elasticsearch.marvel.client.MonitoringClientModule; import org.elasticsearch.marvel.license.LicenseModule; import org.elasticsearch.marvel.license.MarvelLicensee; +import org.elasticsearch.marvel.support.init.proxy.MonitoringClientProxy; import org.elasticsearch.xpack.XPackPlugin; +import org.elasticsearch.xpack.common.init.LazyInitializationModule; import java.util.ArrayList; import java.util.Arrays; @@ -52,6 +55,7 @@ public class Marvel { modules.add(new LicenseModule()); modules.add(new CollectorModule()); modules.add(new ExporterModule(settings)); + modules.add(new MonitoringClientModule()); } return Collections.unmodifiableList(modules); } @@ -76,4 +80,10 @@ public class Marvel { public void onModule(SettingsModule module) { MarvelSettings.register(module); } + + public void onModule(LazyInitializationModule module) { + if (enabled) { + module.registerLazyInitializable(MonitoringClientProxy.class); + } + } } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java index a09d244fd58..0f41d48d21c 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java @@ -10,13 +10,13 @@ import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.client.Client; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.marvel.agent.exporter.ExportBulk; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; import org.elasticsearch.marvel.agent.resolver.ResolversRegistry; +import org.elasticsearch.marvel.support.init.proxy.MonitoringClientProxy; import java.io.IOException; import java.util.Collection; @@ -28,13 +28,13 @@ import java.util.concurrent.atomic.AtomicReference; public class LocalBulk extends ExportBulk { private final ESLogger logger; - private final Client client; + private final MonitoringClientProxy client; private final ResolversRegistry resolvers; BulkRequestBuilder requestBuilder; AtomicReference state = new AtomicReference<>(); - public LocalBulk(String name, ESLogger logger, Client client, ResolversRegistry resolvers) { + public LocalBulk(String name, ESLogger logger, MonitoringClientProxy client, ResolversRegistry resolvers) { super(name); this.logger = logger; this.client = client; diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java index 6a351648059..3929a52211f 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -34,7 +33,7 @@ import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; import org.elasticsearch.marvel.agent.resolver.ResolversRegistry; import org.elasticsearch.marvel.cleaner.CleanerService; -import org.elasticsearch.shield.InternalClient; +import org.elasticsearch.marvel.support.init.proxy.MonitoringClientProxy; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -52,7 +51,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle public static final String TYPE = "local"; - private final Client client; + private final MonitoringClientProxy client; private final ClusterService clusterService; private final ResolversRegistry resolvers; private final CleanerService cleanerService; @@ -63,7 +62,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle /** Version number of built-in templates **/ private final Integer templateVersion; - public LocalExporter(Exporter.Config config, Client client, ClusterService clusterService, CleanerService cleanerService) { + public LocalExporter(Exporter.Config config, MonitoringClientProxy client, + ClusterService clusterService, CleanerService cleanerService) { super(TYPE, config); this.client = client; this.clusterService = clusterService; @@ -344,12 +344,12 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle public static class Factory extends Exporter.Factory { - private final InternalClient client; + private final MonitoringClientProxy client; private final ClusterService clusterService; private final CleanerService cleanerService; @Inject - public Factory(InternalClient client, ClusterService clusterService, CleanerService cleanerService) { + public Factory(MonitoringClientProxy client, ClusterService clusterService, CleanerService cleanerService) { super(TYPE, true); this.client = client; this.clusterService = clusterService; diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClient.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClient.java new file mode 100644 index 00000000000..596096ebb6f --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClient.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.client; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; + +public class MonitoringClient { + + private final Client client; + + @Inject + public MonitoringClient(Client client) { + this.client = client; + } + + // to be implemented: specific API for monitoring +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClientModule.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClientModule.java new file mode 100644 index 00000000000..0a3f6075c9a --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClientModule.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.client; + +import org.elasticsearch.common.inject.AbstractModule; + +public class MonitoringClientModule extends AbstractModule { + + @Override + protected void configure() { + bind(MonitoringClient.class).asEagerSingleton(); + } +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/support/init/proxy/MonitoringClientProxy.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/support/init/proxy/MonitoringClientProxy.java new file mode 100644 index 00000000000..f09877b3590 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/support/init/proxy/MonitoringClientProxy.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.support.init.proxy; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.shield.InternalClient; +import org.elasticsearch.xpack.common.init.proxy.ClientProxy; + +public class MonitoringClientProxy extends ClientProxy { + + @Inject + public MonitoringClientProxy() { + } + + /** + * Creates a proxy to the given internal client (can be used for testing) + */ + public static MonitoringClientProxy of(Client client) { + MonitoringClientProxy proxy = new MonitoringClientProxy(); + proxy.client = client instanceof InternalClient ? (InternalClient) client : new InternalClient.Insecure(client); + return proxy; + } +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java index eba47a89afc..a85d8456bd1 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java @@ -35,6 +35,6 @@ public class MarvelPluginClientTests extends ESTestCase { Marvel plugin = new Marvel(settings); assertThat(plugin.isEnabled(), is(true)); Collection modules = plugin.nodeModules(); - assertThat(modules.size(), is(4)); + assertThat(modules.size(), is(5)); } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java index 3968ab43e82..e44f8c06a9c 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java @@ -17,7 +17,7 @@ import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.MonitoredSystem; import org.elasticsearch.marvel.agent.exporter.local.LocalExporter; import org.elasticsearch.marvel.cleaner.CleanerService; -import org.elasticsearch.shield.InternalClient; +import org.elasticsearch.marvel.support.init.proxy.MonitoringClientProxy; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -67,7 +67,7 @@ public class ExportersTests extends ESTestCase { clusterService = mock(ClusterService.class); // we always need to have the local exporter as it serves as the default one - factories.put(LocalExporter.TYPE, new LocalExporter.Factory(new InternalClient.Insecure(client), clusterService, + factories.put(LocalExporter.TYPE, new LocalExporter.Factory(MonitoringClientProxy.of(client), clusterService, mock(CleanerService.class))); clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(MarvelSettings.COLLECTORS, MarvelSettings.INTERVAL, MarvelSettings.EXPORTERS_SETTINGS))); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java index 1f404a59ec5..9e4c8dbbd4e 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java @@ -23,6 +23,7 @@ import org.elasticsearch.marvel.agent.AgentService; import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; +import org.elasticsearch.marvel.client.MonitoringClient; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.shield.authc.esusers.ESUsersRealm; import org.elasticsearch.shield.authc.support.Hasher; @@ -34,6 +35,7 @@ import org.elasticsearch.test.store.MockFSIndexStore; import org.elasticsearch.test.transport.AssertingLocalTransport; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.watcher.Watcher; +import org.elasticsearch.xpack.XPackClient; import org.elasticsearch.xpack.XPackPlugin; import org.hamcrest.Matcher; import org.jboss.netty.util.internal.SystemPropertyUtil; @@ -135,6 +137,11 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { return client -> (client instanceof NodeClient) ? client.filterWithHeader(headers) : client; } + protected MonitoringClient monitoringClient() { + Client client = shieldEnabled ? internalCluster().transportClient() : client(); + return randomBoolean() ? new XPackClient(client).monitoring() : new MonitoringClient(client); + } + @Override protected Set excludeTemplates() { Set templates = new HashSet<>(); diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java index bdad3b9ec06..bf281eda1d8 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack; import org.elasticsearch.client.Client; +import org.elasticsearch.marvel.client.MonitoringClient; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.shield.client.SecurityClient; import org.elasticsearch.watcher.client.WatcherClient; @@ -22,15 +23,22 @@ import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basic public class XPackClient { private final Client client; + + private final MonitoringClient monitoringClient; private final SecurityClient securityClient; private final WatcherClient watcherClient; public XPackClient(Client client) { this.client = client; + this.monitoringClient = new MonitoringClient(client); this.securityClient = new SecurityClient(client); this.watcherClient = new WatcherClient(client); } + public MonitoringClient monitoring() { + return monitoringClient; + } + public SecurityClient security() { return securityClient; } diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java index adcf8768a48..17e835d3d97 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java @@ -187,6 +187,7 @@ public class XPackPlugin extends Plugin { } public void onModule(LazyInitializationModule module) { + marvel.onModule(module); watcher.onModule(module); } diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/init/proxy/ClientProxy.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/init/proxy/ClientProxy.java index 8e990009182..8d722a552c9 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/init/proxy/ClientProxy.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/init/proxy/ClientProxy.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.common.init.proxy; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; @@ -36,6 +37,10 @@ public class ClientProxy implements LazyInitializable { client.bulk(preProcess(request), listener); } + public BulkRequestBuilder prepareBulk() { + return client.prepareBulk(); + } + protected M preProcess(M message) { return message; } From fe97d2ba51df65ef31e7cbbd7a517dc48164ad32 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 15 Mar 2016 21:20:12 +0100 Subject: [PATCH 6/8] Monitoring: Add REST endpoint to allow external systems to index monitoring data Original commit: elastic/x-pack-elasticsearch@04aa96a228ae25498c1d4b0bccc007f745efb181 --- .../java/org/elasticsearch/marvel/Marvel.java | 66 ++-- .../marvel/action/MonitoringBulkAction.java | 30 ++ .../marvel/action/MonitoringBulkDoc.java | 80 +++++ .../marvel/action/MonitoringBulkRequest.java | 128 ++++++++ .../action/MonitoringBulkRequestBuilder.java | 29 ++ .../marvel/action/MonitoringBulkResponse.java | 138 +++++++++ .../action/TransportMonitoringBulkAction.java | 127 ++++++++ .../marvel/agent/AgentService.java | 3 + .../marvel/agent/exporter/ExportBulk.java | 43 +-- .../agent/exporter/ExportException.java | 76 +++++ .../marvel/agent/exporter/Exporters.java | 7 +- .../marvel/agent/exporter/MonitoringDoc.java | 69 ++--- .../agent/exporter/http/HttpExporter.java | 93 +++--- .../agent/exporter/local/LocalBulk.java | 82 +++-- .../agent/exporter/local/LocalExporter.java | 2 +- .../agent/resolver/ResolversRegistry.java | 36 ++- .../resolver/bulk/MonitoringBulkResolver.java | 36 +++ .../marvel/client/MonitoringClient.java | 41 ++- .../marvel/rest/MonitoringRestHandler.java | 32 ++ .../rest/action/RestMonitoringBulkAction.java | 86 +++++ .../init/proxy/MonitoringClientProxy.java | 5 - .../src/main/resources/monitoring-data.json | 6 +- .../src/main/resources/monitoring-es.json | 1 - .../marvel/MarvelPluginClientTests.java | 4 +- .../marvel/action/MonitoringBulkDocTests.java | 104 +++++++ .../action/MonitoringBulkRequestTests.java | 193 ++++++++++++ .../action/MonitoringBulkResponseTests.java | 73 +++++ .../marvel/action/MonitoringBulkTests.java | 147 +++++++++ .../TransportMonitoringBulkActionTests.java | 293 ++++++++++++++++++ .../marvel/agent/exporter/ExportersTests.java | 5 +- .../agent/exporter/MonitoringDocTests.java | 4 +- .../exporter/local/LocalExporterTests.java | 11 +- .../MonitoringIndexNameResolverTestCase.java | 2 +- .../bulk/MonitoringBulkResolverTests.java | 69 +++++ .../marvel/test/MarvelIntegTestCase.java | 6 +- .../org/elasticsearch/xpack/XPackPlugin.java | 2 + 36 files changed, 1929 insertions(+), 200 deletions(-) create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkAction.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkDoc.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequest.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequestBuilder.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkResponse.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/TransportMonitoringBulkAction.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportException.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/MonitoringRestHandler.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/action/RestMonitoringBulkAction.java create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkDocTests.java create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkRequestTests.java create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkResponseTests.java create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkTests.java create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/TransportMonitoringBulkActionTests.java create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Marvel.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Marvel.java index 5ba5e8f7848..6440795572d 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Marvel.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Marvel.java @@ -5,13 +5,14 @@ */ package org.elasticsearch.marvel; -import org.elasticsearch.client.Client; +import org.elasticsearch.action.ActionModule; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.marvel.action.MonitoringBulkAction; +import org.elasticsearch.marvel.action.TransportMonitoringBulkAction; import org.elasticsearch.marvel.agent.AgentService; import org.elasticsearch.marvel.agent.collector.CollectorModule; import org.elasticsearch.marvel.agent.exporter.ExporterModule; @@ -19,49 +20,58 @@ import org.elasticsearch.marvel.cleaner.CleanerService; import org.elasticsearch.marvel.client.MonitoringClientModule; import org.elasticsearch.marvel.license.LicenseModule; import org.elasticsearch.marvel.license.MarvelLicensee; +import org.elasticsearch.marvel.rest.action.RestMonitoringBulkAction; import org.elasticsearch.marvel.support.init.proxy.MonitoringClientProxy; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.common.init.LazyInitializationModule; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.List; +/** + * This class activates/deactivates the monitoring modules depending if we're running a node client, transport client or tribe client: + * - node clients: all modules are binded + * - transport clients: only action/transport actions are binded + * - tribe clients: everything is disables by default but can be enabled per tribe cluster + */ public class Marvel { - private static final ESLogger logger = Loggers.getLogger(XPackPlugin.class); - public static final String NAME = "monitoring"; private final Settings settings; private final boolean enabled; + private final boolean transportClientMode; public Marvel(Settings settings) { this.settings = settings; - this.enabled = enabled(settings); + this.enabled = MarvelSettings.ENABLED.get(settings); + this.transportClientMode = XPackPlugin.transportClientMode(settings); } boolean isEnabled() { return enabled; } - public Collection nodeModules() { - List modules = new ArrayList<>(); + boolean isTransportClient() { + return transportClientMode; + } - if (enabled) { - modules.add(new MarvelModule()); - modules.add(new LicenseModule()); - modules.add(new CollectorModule()); - modules.add(new ExporterModule(settings)); - modules.add(new MonitoringClientModule()); + public Collection nodeModules() { + if (enabled == false || transportClientMode) { + return Collections.emptyList(); } - return Collections.unmodifiableList(modules); + + return Arrays.asList( + new MarvelModule(), + new LicenseModule(), + new CollectorModule(), + new ExporterModule(settings), + new MonitoringClientModule()); } public Collection> nodeServices() { - if (enabled == false) { + if (enabled == false || transportClientMode) { return Collections.emptyList(); } return Arrays.>asList(MarvelLicensee.class, @@ -69,18 +79,22 @@ public class Marvel { CleanerService.class); } - public static boolean enabled(Settings settings) { - if ("node".equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey())) == false) { - logger.trace("monitoring cannot be started on a transport client"); - return false; - } - return MarvelSettings.ENABLED.get(settings); - } - public void onModule(SettingsModule module) { MarvelSettings.register(module); } + public void onModule(ActionModule module) { + if (enabled) { + module.registerAction(MonitoringBulkAction.INSTANCE, TransportMonitoringBulkAction.class); + } + } + + public void onModule(NetworkModule module) { + if (enabled && transportClientMode == false) { + module.registerRestHandler(RestMonitoringBulkAction.class); + } + } + public void onModule(LazyInitializationModule module) { if (enabled) { module.registerLazyInitializable(MonitoringClientProxy.class); diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkAction.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkAction.java new file mode 100644 index 00000000000..c0a8f005122 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkAction.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class MonitoringBulkAction extends Action { + + public static final MonitoringBulkAction INSTANCE = new MonitoringBulkAction(); + public static final String NAME = "cluster:admin/xpack/monitoring/bulk"; + + private MonitoringBulkAction() { + super(NAME); + } + + @Override + public MonitoringBulkRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new MonitoringBulkRequestBuilder(client); + } + + @Override + public MonitoringBulkResponse newResponse() { + return new MonitoringBulkResponse(); + } +} + diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkDoc.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkDoc.java new file mode 100644 index 00000000000..96fcce29ab2 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkDoc.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; + +import java.io.IOException; + +public class MonitoringBulkDoc extends MonitoringDoc { + + private String index; + private String type; + private String id; + + private BytesReference source; + + public MonitoringBulkDoc(String monitoringId, String monitoringVersion) { + super(monitoringId, monitoringVersion); + } + + public MonitoringBulkDoc(StreamInput in) throws IOException { + super(in); + index = in.readOptionalString(); + type = in.readOptionalString(); + id = in.readOptionalString(); + source = in.readBytesReference(); + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public BytesReference getSource() { + return source; + } + + public void setSource(BytesReference source) { + this.source = source; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(index); + out.writeOptionalString(type); + out.writeOptionalString(id); + out.writeBytesReference(source); + } + + @Override + public MonitoringBulkDoc readFrom(StreamInput in) throws IOException { + return new MonitoringBulkDoc(in); + } +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequest.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequest.java new file mode 100644 index 00000000000..27a0626fd6e --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequest.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Requests; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * A monitoring bulk request holds one or more {@link MonitoringBulkDoc}s. + *

+ * Every monitoring document added to the request is associated to a monitoring system id and version. If this {id, version} pair is + * supported by the monitoring plugin, the monitoring documents will be indexed in a single batch using a normal bulk request. + *

+ * The monitoring {id, version} pair is used by {org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver} to resolve the index, + * type and id of the final document to be indexed. A {@link MonitoringBulkDoc} can also hold its own index/type/id values but there's no + * guarantee that these information will be effectively used. + */ +public class MonitoringBulkRequest extends ActionRequest { + + final List docs = new ArrayList<>(); + + /** + * @return the list of monitoring documents to be indexed + */ + public Collection getDocs() { + return Collections.unmodifiableCollection(new ArrayList<>(this.docs)); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (docs.isEmpty()) { + validationException = addValidationError("no monitoring documents added", validationException); + } + + for (int i = 0; i < docs.size(); i++) { + MonitoringBulkDoc doc = docs.get(i); + if (Strings.hasLength(doc.getMonitoringId()) == false) { + validationException = addValidationError("monitored system id is missing for monitoring document [" + i + "]", + validationException); + } + if (Strings.hasLength(doc.getMonitoringVersion()) == false) { + validationException = addValidationError("monitored system version is missing for monitoring document [" + i + "]", + validationException); + } + if (Strings.hasLength(doc.getType()) == false) { + validationException = addValidationError("type is missing for monitoring document [" + i + "]", + validationException); + } + if (doc.getSource() == null || doc.getSource().length() == 0) { + validationException = addValidationError("source is missing for monitoring document [" + i + "]", validationException); + } + } + + return validationException; + } + + /** + * Adds a monitoring document to the list of documents to be indexed. + */ + public MonitoringBulkRequest add(MonitoringBulkDoc doc) { + docs.add(doc); + return this; + } + + /** + * Parses a monitoring bulk request and builds the list of documents to be indexed. + */ + public MonitoringBulkRequest add(BytesReference content, String defaultMonitoringId, String defaultMonitoringVersion, + String defaultIndex, String defaultType) throws Exception { + // MonitoringBulkRequest accepts a body request that has the same format as the BulkRequest: + // instead of duplicating the parsing logic here we use a new BulkRequest instance to parse the content. + BulkRequest bulkRequest = Requests.bulkRequest().add(content, defaultIndex, defaultType); + + for (ActionRequest request : bulkRequest.requests()) { + if (request instanceof IndexRequest) { + IndexRequest indexRequest = (IndexRequest) request; + + // builds a new monitoring document based on the index request + MonitoringBulkDoc doc = new MonitoringBulkDoc(defaultMonitoringId, defaultMonitoringVersion); + doc.setIndex(indexRequest.index()); + doc.setType(indexRequest.type()); + doc.setId(indexRequest.id()); + doc.setSource(indexRequest.source()); + add(doc); + } else { + throw new IllegalArgumentException("monitoring bulk requests should only contain index requests"); + } + } + return this; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + int size = in.readVInt(); + for (int i = 0; i < size; i++) { + add(new MonitoringBulkDoc(in)); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(docs.size()); + for (MonitoringBulkDoc doc : docs) { + doc.writeTo(out); + } + } +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequestBuilder.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequestBuilder.java new file mode 100644 index 00000000000..112c6bf4551 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequestBuilder.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.bytes.BytesReference; + +public class MonitoringBulkRequestBuilder + extends ActionRequestBuilder { + + public MonitoringBulkRequestBuilder(ElasticsearchClient client) { + super(client, MonitoringBulkAction.INSTANCE, new MonitoringBulkRequest()); + } + + public MonitoringBulkRequestBuilder add(MonitoringBulkDoc doc) { + request.add(doc); + return this; + } + + public MonitoringBulkRequestBuilder add(BytesReference content, String defaultId, String defaultVersion, String defaultIndex, + String defaultType) throws Exception { + request.add(content, defaultId, defaultVersion, defaultIndex, defaultType); + return this; + } +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkResponse.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkResponse.java new file mode 100644 index 00000000000..070f10f5e2a --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkResponse.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.util.Objects; + +public class MonitoringBulkResponse extends ActionResponse { + + private long tookInMillis; + private Error error; + + MonitoringBulkResponse() { + } + + public MonitoringBulkResponse(long tookInMillis) { + this(tookInMillis, null); + } + + public MonitoringBulkResponse(long tookInMillis, Error error) { + this.tookInMillis = tookInMillis; + this.error = error; + } + + public TimeValue getTook() { + return new TimeValue(tookInMillis); + } + + public long getTookInMillis() { + return tookInMillis; + } + + /** + * Returns HTTP status + *

    + *
  • {@link RestStatus#OK} if monitoring bulk request was successful
    • + *
  • {@link RestStatus#INTERNAL_SERVER_ERROR} if monitoring bulk request was partially successful or failed completely
    • + *
+ */ + public RestStatus status() { + return error == null ? RestStatus.OK : RestStatus.INTERNAL_SERVER_ERROR; + } + + public Error getError() { + return error; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + tookInMillis = in.readVLong(); + error = in.readOptionalWritable(Error::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVLong(tookInMillis); + out.writeOptionalWriteable(error); + } + + public static class Error implements Writeable, ToXContent { + + private final Throwable cause; + private final RestStatus status; + + public Error(Throwable t) { + cause = Objects.requireNonNull(t); + status = ExceptionsHelper.status(t); + } + + Error(StreamInput in) throws IOException { + this(in.readThrowable()); + } + + /** + * The failure message. + */ + public String getMessage() { + return this.cause.toString(); + } + + /** + * The rest status. + */ + public RestStatus getStatus() { + return this.status; + } + + /** + * The actual cause of the failure. + */ + public Throwable getCause() { + return cause; + } + + @Override + public Error readFrom(StreamInput in) throws IOException { + return new Error(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeThrowable(getCause()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + ElasticsearchException.toXContent(builder, params, cause); + builder.endObject(); + return builder; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Error ["); + sb.append("cause=").append(cause); + sb.append(", status=").append(status); + sb.append(']'); + return sb.toString(); + } + } +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/TransportMonitoringBulkAction.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/TransportMonitoringBulkAction.java new file mode 100644 index 00000000000..56e95d2c574 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/TransportMonitoringBulkAction.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.marvel.agent.exporter.Exporters; +import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class TransportMonitoringBulkAction extends HandledTransportAction { + + private final ClusterService clusterService; + private final Exporters exportService; + + @Inject + public TransportMonitoringBulkAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, + TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, Exporters exportService) { + super(settings, MonitoringBulkAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + MonitoringBulkRequest::new); + this.clusterService = clusterService; + this.exportService = exportService; + } + + @Override + protected void doExecute(MonitoringBulkRequest request, ActionListener listener) { + clusterService.state().blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE); + new AsyncAction(request, listener, exportService, clusterService).start(); + } + + class AsyncAction { + + private final MonitoringBulkRequest request; + private final ActionListener listener; + private final Exporters exportService; + private final ClusterService clusterService; + + public AsyncAction(MonitoringBulkRequest request, ActionListener listener, + Exporters exportService, ClusterService clusterService) { + this.request = request; + this.listener = listener; + this.exportService = exportService; + this.clusterService = clusterService; + } + + void start() { + executeExport(prepareForExport(request.getDocs()), System.nanoTime(), listener); + } + + /** + * Iterate over the documents and set the values of common fields if needed: + * - cluster UUID + * - timestamp + * - source node + */ + Collection prepareForExport(Collection docs) { + final String clusterUUID = clusterService.state().metaData().clusterUUID(); + Function updateClusterUUID = doc -> { + if (doc.getClusterUUID() == null) { + doc.setClusterUUID(clusterUUID); + } + return doc; + }; + + final long timestamp = System.currentTimeMillis(); + Function updateTimestamp = doc -> { + if (doc.getTimestamp() == 0) { + doc.setTimestamp(timestamp); + } + return doc; + }; + + final DiscoveryNode sourceNode = clusterService.localNode(); + Function updateSourceNode = doc -> { + if (doc.getSourceNode() == null) { + doc.setSourceNode(sourceNode); + } + return doc; + }; + + return docs.stream() + .map(updateClusterUUID.andThen(updateTimestamp.andThen(updateSourceNode))) + .collect(Collectors.toList()); + } + + /** + * Exports the documents + */ + void executeExport(final Collection docs, final long startTimeNanos, + final ActionListener listener) { + threadPool.generic().execute(new AbstractRunnable() { + @Override + protected void doRun() throws Exception { + exportService.export(docs); + listener.onResponse(new MonitoringBulkResponse(buildTookInMillis(startTimeNanos))); + } + + @Override + public void onFailure(Throwable t) { + listener.onResponse(new MonitoringBulkResponse(buildTookInMillis(startTimeNanos), new MonitoringBulkResponse.Error(t))); + } + }); + } + } + + private long buildTookInMillis(long startTimeNanos) { + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); + } +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/AgentService.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/AgentService.java index 3f4199866dc..f8099becc1e 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/AgentService.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/AgentService.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.agent.collector.Collector; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector; +import org.elasticsearch.marvel.agent.exporter.ExportException; import org.elasticsearch.marvel.agent.exporter.Exporter; import org.elasticsearch.marvel.agent.exporter.Exporters; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; @@ -199,6 +200,8 @@ public class AgentService extends AbstractLifecycleComponent { exporters.export(docs); } + } catch (ExportException e) { + logger.error("exception when exporting documents", e); } catch (InterruptedException e) { logger.trace("interrupted"); Thread.currentThread().interrupt(); diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java index 963df57f04c..f02a11ac95e 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportBulk.java @@ -5,8 +5,6 @@ */ package org.elasticsearch.marvel.agent.exporter; -import org.elasticsearch.ElasticsearchException; - import java.util.Collection; /** @@ -25,18 +23,14 @@ public abstract class ExportBulk { return name; } - public abstract ExportBulk add(Collection docs) throws Exception; + public abstract ExportBulk add(Collection docs) throws ExportException; - public abstract void flush() throws Exception; + public abstract void flush() throws ExportException; - public final void close(boolean flush) throws Exception { - Exception exception = null; + public final void close(boolean flush) throws ExportException { + ExportException exception = null; if (flush) { - try { - flush(); - } catch (Exception e) { - exception = e; - } + flush(); } // now closing @@ -46,7 +40,7 @@ public abstract class ExportBulk { if (exception != null) { exception.addSuppressed(e); } else { - exception = e; + exception = new ExportException("Exception when closing export bulk", e); } } @@ -69,24 +63,35 @@ public abstract class ExportBulk { } @Override - public ExportBulk add(Collection docs) throws Exception { + public ExportBulk add(Collection docs) throws ExportException { + ExportException exception = null; for (ExportBulk bulk : bulks) { - bulk.add(docs); + try { + bulk.add(docs); + } catch (ExportException e) { + if (exception == null) { + exception = new ExportException("failed to add documents to export bulks"); + } + exception.addExportException(e); + } + } + if (exception != null) { + throw exception; } return this; } @Override - public void flush() throws Exception { - Exception exception = null; + public void flush() throws ExportException { + ExportException exception = null; for (ExportBulk bulk : bulks) { try { bulk.flush(); - } catch (Exception e) { + } catch (ExportException e) { if (exception == null) { - exception = new ElasticsearchException("failed to flush exporter bulks"); + exception = new ExportException("failed to flush export bulks"); } - exception.addSuppressed(new ElasticsearchException("failed to flush [{}] exporter bulk", e, bulk.name)); + exception.addExportException(e); } } if (exception != null) { diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportException.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportException.java new file mode 100644 index 00000000000..02cddd86ece --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExportException.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.exporter; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class ExportException extends ElasticsearchException implements Iterable { + + private final List exceptions = new ArrayList<>(); + + public ExportException(Throwable throwable) { + super(throwable); + } + + public ExportException(String msg, Object... args) { + super(msg, args); + } + + public ExportException(String msg, Throwable throwable, Object... args) { + super(msg, throwable, args); + } + + public ExportException(StreamInput in) throws IOException { + super(in); + for (int i = in.readVInt(); i > 0; i--) { + exceptions.add(new ExportException(in)); + } + } + + public boolean addExportException(ExportException e) { + return exceptions.add(e); + } + + public boolean hasExportExceptions() { + return exceptions.size() > 0; + } + + @Override + public Iterator iterator() { + return exceptions.iterator(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(exceptions.size()); + for (ExportException e : exceptions) { + e.writeTo(out); + } + } + + @Override + protected void innerToXContent(XContentBuilder builder, Params params) throws IOException { + super.innerToXContent(builder, params); + if (hasExportExceptions()) { + builder.startArray("exceptions"); + for (ExportException exception : exceptions) { + builder.startObject(); + exception.toXContent(builder, params); + builder.endObject(); + } + builder.endArray(); + } + } +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporters.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporters.java index a01946922cc..aeadf2fff66 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporters.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporters.java @@ -122,7 +122,7 @@ public class Exporters extends AbstractLifecycleComponent implements bulks.add(bulk); } } catch (Exception e) { - logger.error("exporter [{}] failed to export monitoring data", e, exporter.name()); + logger.error("exporter [{}] failed to open exporting bulk", e, exporter.name()); } } return bulks.isEmpty() ? null : new ExportBulk.Compound(bulks); @@ -179,9 +179,9 @@ public class Exporters extends AbstractLifecycleComponent implements /** * Exports a collection of monitoring documents using the configured exporters */ - public synchronized void export(Collection docs) throws Exception { + public synchronized void export(Collection docs) throws ExportException { if (this.lifecycleState() != Lifecycle.State.STARTED) { - throw new IllegalStateException("Export service is not started"); + throw new ExportException("Export service is not started"); } if (docs != null && docs.size() > 0) { ExportBulk bulk = openBulk(); @@ -191,7 +191,6 @@ public class Exporters extends AbstractLifecycleComponent implements } try { - logger.debug("exporting [{}] monitoring documents", docs.size()); bulk.add(docs); } finally { bulk.close(lifecycleState() == Lifecycle.State.STARTED); diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MonitoringDoc.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MonitoringDoc.java index 6e1bee7bf54..b2199b7f417 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MonitoringDoc.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MonitoringDoc.java @@ -22,8 +22,6 @@ import java.io.IOException; */ public class MonitoringDoc implements Writeable { - private static final MonitoringDoc PROTO = new MonitoringDoc(); - private final String monitoringId; private final String monitoringVersion; @@ -31,16 +29,18 @@ public class MonitoringDoc implements Writeable { private long timestamp; private Node sourceNode; - // Used by {@link #PROTO} instance and tests - MonitoringDoc() { - this(null, null); - } - public MonitoringDoc(String monitoringId, String monitoringVersion) { this.monitoringId = monitoringId; this.monitoringVersion = monitoringVersion; } + public MonitoringDoc(StreamInput in) throws IOException { + this(in.readOptionalString(), in.readOptionalString()); + clusterUUID = in.readOptionalString(); + timestamp = in.readVLong(); + sourceNode = in.readOptionalWritable(Node::new); + } + public String getClusterUUID() { return clusterUUID; } @@ -80,7 +80,7 @@ public class MonitoringDoc implements Writeable { @Override public String toString() { - return "marvel document [class=" + getClass().getName() + + return "monitoring document [class=" + getClass().getSimpleName() + ", monitoring id=" + getMonitoringId() + ", monitoring version=" + getMonitoringVersion() + "]"; @@ -92,33 +92,16 @@ public class MonitoringDoc implements Writeable { out.writeOptionalString(getMonitoringVersion()); out.writeOptionalString(getClusterUUID()); out.writeVLong(getTimestamp()); - if (getSourceNode() != null) { - out.writeBoolean(true); - getSourceNode().writeTo(out); - } else { - out.writeBoolean(false); - } + out.writeOptionalWriteable(getSourceNode()); } @Override public MonitoringDoc readFrom(StreamInput in) throws IOException { - MonitoringDoc doc = new MonitoringDoc(in.readOptionalString(), in.readOptionalString()); - doc.setClusterUUID(in.readOptionalString()); - doc.setTimestamp(in.readVLong()); - if (in.readBoolean()) { - doc.setSourceNode(Node.PROTO.readFrom(in)); - } - return doc; - } - - public static MonitoringDoc readMonitoringDoc(StreamInput in) throws IOException { - return PROTO.readFrom(in); + return new MonitoringDoc(in); } public static class Node implements Writeable, ToXContent { - public static final Node PROTO = new Node(); - private String uuid; private String host; private String transportAddress; @@ -126,10 +109,6 @@ public class MonitoringDoc implements Writeable { private String name; private ImmutableOpenMap attributes; - // Used by the {@link #PROTO} instance - Node() { - } - public Node(String uuid, String host, String transportAddress, String ip, String name, ImmutableOpenMap attributes) { this.uuid = uuid; @@ -147,6 +126,20 @@ public class MonitoringDoc implements Writeable { this.attributes = builder.build(); } + public Node(StreamInput in) throws IOException { + uuid = in.readOptionalString(); + host = in.readOptionalString(); + transportAddress = in.readOptionalString(); + ip = in.readOptionalString(); + name = in.readOptionalString(); + int size = in.readVInt(); + ImmutableOpenMap.Builder attributes = ImmutableOpenMap.builder(size); + for (int i = 0; i < size; i++) { + attributes.put(in.readOptionalString(), in.readOptionalString()); + } + this.attributes = attributes.build(); + } + public String getUUID() { return uuid; } @@ -208,19 +201,7 @@ public class MonitoringDoc implements Writeable { @Override public Node readFrom(StreamInput in) throws IOException { - Node node = new Node(); - node.uuid = in.readOptionalString(); - node.host = in.readOptionalString(); - node.transportAddress = in.readOptionalString(); - node.ip = in.readOptionalString(); - node.name = in.readOptionalString(); - int size = in.readVInt(); - ImmutableOpenMap.Builder attributes = ImmutableOpenMap.builder(size); - for (int i = 0; i < size; i++) { - attributes.put(in.readOptionalString(), in.readOptionalString()); - } - node.attributes = attributes.build(); - return node; + return new Node(in); } @Override diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java index 21a9ad75c22..5f92489d0bd 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.marvel.agent.exporter.ExportBulk; +import org.elasticsearch.marvel.agent.exporter.ExportException; import org.elasticsearch.marvel.agent.exporter.Exporter; import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; @@ -79,7 +80,9 @@ public class HttpExporter extends Exporter { public static final String SSL_TRUSTSTORE_ALGORITHM_SETTING = "truststore.algorithm"; public static final String SSL_HOSTNAME_VERIFICATION_SETTING = SSL_SETTING + ".hostname_verification"; - /** Minimum supported version of the remote monitoring cluster **/ + /** + * Minimum supported version of the remote monitoring cluster + **/ public static final Version MIN_SUPPORTED_CLUSTER_VERSION = Version.V_2_0_0_beta2; private static final XContentType CONTENT_TYPE = XContentType.JSON; @@ -89,19 +92,24 @@ public class HttpExporter extends Exporter { final TimeValue connectionReadTimeout; final BasicAuth auth; - /** https support * */ + /** + * https support * + */ final SSLSocketFactory sslSocketFactory; final boolean hostnameVerification; final Environment env; final ResolversRegistry resolvers; - final @Nullable TimeValue templateCheckTimeout; + @Nullable + final TimeValue templateCheckTimeout; volatile boolean checkedAndUploadedIndexTemplate = false; volatile boolean supportedClusterVersion = false; - /** Version number of built-in templates **/ + /** + * Version number of built-in templates + **/ private final Integer templateVersion; boolean keepAlive; @@ -218,9 +226,9 @@ public class HttpExporter extends Exporter { logger.trace("http exporter [{}] - added index request [index={}, type={}, id={}]", name(), index, type, id); } - } else { - logger.warn("http exporter [{}] - unable to render monitoring document of type [{}]: no renderer found in registry", - name(), doc); + } else if (logger.isTraceEnabled()) { + logger.trace("http exporter [{}] - no resolver found for monitoring document [class={}, id={}, version={}]", + name(), doc.getClass().getName(), doc.getMonitoringId(), doc.getMonitoringVersion()); } } catch (Exception e) { logger.warn("http exporter [{}] - failed to render document [{}], skipping it", e, name(), doc); @@ -318,7 +326,9 @@ public class HttpExporter extends Exporter { return null; } - /** open a connection to the given hosts, returning null when not successful * */ + /** + * open a connection to the given hosts, returning null when not successful * + */ private HttpURLConnection openConnection(String host, String method, String path, @Nullable String contentType) { try { final URL url = HttpExporterUtils.parseHostWithPath(host, path); @@ -450,7 +460,7 @@ public class HttpExporter extends Exporter { // 200 means that the template has been found, 404 otherwise if (connection.getResponseCode() == 200) { - logger.debug("monitoring template [{}] found",templateName); + logger.debug("monitoring template [{}] found", templateName); return true; } } catch (Exception e) { @@ -543,7 +553,9 @@ public class HttpExporter extends Exporter { } } - /** SSL Initialization * */ + /** + * SSL Initialization * + */ public SSLSocketFactory createSSLSocketFactory(Settings settings) { if (settings.names().isEmpty()) { logger.trace("no ssl context configured"); @@ -693,47 +705,54 @@ public class HttpExporter extends Exporter { } @Override - public Bulk add(Collection docs) throws Exception { - if (connection == null) { - connection = openExportingConnection(); - } - if ((docs != null) && (!docs.isEmpty())) { - if (out == null) { - out = connection.getOutputStream(); - } + public Bulk add(Collection docs) throws ExportException { + try { + if ((docs != null) && (!docs.isEmpty())) { + if (connection == null) { + connection = openExportingConnection(); + if (connection == null) { + throw new IllegalStateException("No connection available to export documents"); + } + } + if (out == null) { + out = connection.getOutputStream(); + } - // We need to use a buffer to render each monitoring document - // because the renderer might close the outputstream (ex: XContentBuilder) - try (BytesStreamOutput buffer = new BytesStreamOutput()) { - for (MonitoringDoc monitoringDoc : docs) { - try { - render(monitoringDoc, buffer); - // write the result to the connection - out.write(buffer.bytes().toBytes()); - } finally { - buffer.reset(); + // We need to use a buffer to render each monitoring document + // because the renderer might close the outputstream (ex: XContentBuilder) + try (BytesStreamOutput buffer = new BytesStreamOutput()) { + for (MonitoringDoc monitoringDoc : docs) { + try { + render(monitoringDoc, buffer); + // write the result to the connection + out.write(buffer.bytes().toBytes()); + } finally { + buffer.reset(); + } } } } + } catch (Exception e) { + throw new ExportException("failed to add documents to export bulk [{}]", name); } return this; } @Override - public void flush() throws IOException { + public void flush() throws ExportException { if (connection != null) { - flush(connection); - connection = null; + try { + flush(connection); + } catch (Exception e) { + throw new ExportException("failed to flush export bulk [{}]", e, name); + } finally { + connection = null; + } } } private void flush(HttpURLConnection connection) throws IOException { - try { - sendCloseExportingConnection(connection); - } catch (IOException e) { - logger.error("failed sending data to [{}]: {}", connection.getURL(), ExceptionsHelper.detailedMessage(e)); - throw e; - } + sendCloseExportingConnection(connection); } } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java index 0f41d48d21c..5dbe433b082 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.marvel.agent.exporter.local; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -13,12 +12,13 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.marvel.agent.exporter.ExportBulk; +import org.elasticsearch.marvel.agent.exporter.ExportException; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; import org.elasticsearch.marvel.agent.resolver.ResolversRegistry; import org.elasticsearch.marvel.support.init.proxy.MonitoringClientProxy; -import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.concurrent.atomic.AtomicReference; @@ -43,7 +43,9 @@ public class LocalBulk extends ExportBulk { } @Override - public synchronized ExportBulk add(Collection docs) throws Exception { + public synchronized ExportBulk add(Collection docs) throws ExportException { + ExportException exception = null; + for (MonitoringDoc doc : docs) { if (state.get() != State.ACTIVE) { return this; @@ -54,42 +56,61 @@ public class LocalBulk extends ExportBulk { try { MonitoringIndexNameResolver resolver = resolvers.getResolver(doc); - if (resolver != null) { - IndexRequest request = new IndexRequest(resolver.index(doc), resolver.type(doc), resolver.id(doc)); - request.source(resolver.source(doc, XContentType.SMILE)); - requestBuilder.add(request); + IndexRequest request = new IndexRequest(resolver.index(doc), resolver.type(doc), resolver.id(doc)); + request.source(resolver.source(doc, XContentType.SMILE)); + requestBuilder.add(request); - if (logger.isTraceEnabled()) { - logger.trace("local exporter [{}] - added index request [index={}, type={}, id={}]", - name, request.index(), request.type(), request.id()); - } - } else { - logger.warn("local exporter [{}] - unable to render monitoring document of type [{}]: no renderer found in registry", - name, doc); + if (logger.isTraceEnabled()) { + logger.trace("local exporter [{}] - added index request [index={}, type={}, id={}]", + name, request.index(), request.type(), request.id()); } } catch (Exception e) { - logger.warn("local exporter [{}] - failed to add document [{}], skipping it", e, name, doc); + if (exception == null) { + exception = new ExportException("failed to add documents to export bulk [{}]", name); + } + exception.addExportException(new ExportException("failed to add document [{}]", e, doc, name)); } } + + if (exception != null) { + throw exception; + } + return this; } @Override - public void flush() throws IOException { - if (state.get() != State.ACTIVE || requestBuilder == null) { + public void flush() throws ExportException { + if (state.get() != State.ACTIVE || requestBuilder == null || requestBuilder.numberOfActions() == 0) { return; } try { logger.trace("exporter [{}] - exporting {} documents", name, requestBuilder.numberOfActions()); BulkResponse bulkResponse = requestBuilder.get(); + if (bulkResponse.hasFailures()) { - throw new ElasticsearchException(buildFailureMessage(bulkResponse)); + throwExportException(bulkResponse.getItems()); } + } catch (Exception e) { + throw new ExportException("failed to flush export bulk [{}]", e, name); } finally { requestBuilder = null; } } + void throwExportException(BulkItemResponse[] bulkItemResponses) { + ExportException exception = new ExportException("bulk [{}] reports failures when exporting documents", name); + + Arrays.stream(bulkItemResponses) + .filter(BulkItemResponse::isFailed) + .map(item -> new ExportException(item.getFailure().getCause())) + .forEach(exception::addExportException); + + if (exception.hasExportExceptions()) { + throw exception; + } + } + void terminate() { state.set(State.TERMINATING); synchronized (this) { @@ -98,31 +119,6 @@ public class LocalBulk extends ExportBulk { } } - /** - * In case of something goes wrong and there's a lot of shards/indices, - * we limit the number of failures displayed in log. - */ - private String buildFailureMessage(BulkResponse bulkResponse) { - BulkItemResponse[] items = bulkResponse.getItems(); - - if (logger.isDebugEnabled() || (items.length < 100)) { - return bulkResponse.buildFailureMessage(); - } - - StringBuilder sb = new StringBuilder(); - sb.append("failure in bulk execution, only the first 100 failures are printed:"); - for (int i = 0; i < items.length && i < 100; i++) { - BulkItemResponse item = items[i]; - if (item.isFailed()) { - sb.append("\n[").append(i) - .append("]: index [").append(item.getIndex()).append("], type [").append(item.getType()) - .append("], id [").append(item.getId()).append("], message [").append(item.getFailureMessage()) - .append("]"); - } - } - return sb.toString(); - } - enum State { ACTIVE, TERMINATING, diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java index 3929a52211f..c547114b5df 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java @@ -282,7 +282,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle .distinct() .toArray(String[]::new); - MonitoringDoc monitoringDoc = new MonitoringDoc(MonitoredSystem.ES.getSystem(), Version.CURRENT.toString()); + MonitoringDoc monitoringDoc = new MonitoringDoc(null, null); monitoringDoc.setTimestamp(System.currentTimeMillis()); // Get the names of the current monitoring indices diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java index 5703c2f415d..cff51999817 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java @@ -5,8 +5,10 @@ */ package org.elasticsearch.marvel.agent.resolver; +import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.MonitoredSystem; +import org.elasticsearch.marvel.action.MonitoringBulkDoc; import org.elasticsearch.marvel.agent.collector.cluster.ClusterInfoMonitoringDoc; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateMonitoringDoc; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateNodeMonitoringDoc; @@ -19,6 +21,7 @@ import org.elasticsearch.marvel.agent.collector.node.NodeStatsMonitoringDoc; import org.elasticsearch.marvel.agent.collector.shards.ShardMonitoringDoc; import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; +import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver; import org.elasticsearch.marvel.agent.resolver.cluster.ClusterInfoResolver; import org.elasticsearch.marvel.agent.resolver.cluster.ClusterStateNodeResolver; import org.elasticsearch.marvel.agent.resolver.cluster.ClusterStateResolver; @@ -46,8 +49,8 @@ public class ResolversRegistry implements Iterable // register built-in defaults resolvers registerBuiltIn(ES, MarvelTemplateUtils.TEMPLATE_VERSION, settings); - // register resolvers for external applications, something like: - //registrations.add(resolveByIdVersion(MonitoringIds.KIBANA, "4.4.1", new KibanaDocResolver(KIBANA, 0, settings))); + // register resolvers for external applications + registerKibana(settings); } /** @@ -66,6 +69,14 @@ public class ResolversRegistry implements Iterable registrations.add(resolveByClass(ShardMonitoringDoc.class, new ShardsResolver(id, version, settings))); } + /** + * Registers resolvers for Kibana + */ + private void registerKibana(Settings settings) { + final MonitoringBulkResolver kibana = new MonitoringBulkResolver(MonitoredSystem.KIBANA, 0, settings); + registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, MonitoredSystem.KIBANA, Version.CURRENT, kibana)); + } + /** * @return a Resolver that is able to resolver the given monitoring document */ @@ -75,8 +86,7 @@ public class ResolversRegistry implements Iterable return registration.resolver(); } } - throw new IllegalArgumentException("No resolver found for monitoring document [class=" + document.getClass().getName() - + ", id=" + document.getMonitoringId() + ", version=" + document.getMonitoringVersion() + "]"); + throw new IllegalArgumentException("No resolver found for monitoring document"); } @Override @@ -88,6 +98,23 @@ public class ResolversRegistry implements Iterable return new Registration(resolver, type::isInstance); } + static Registration resolveByClassSystemVersion(Class type, MonitoredSystem system, Version version, + MonitoringIndexNameResolver resolver) { + return new Registration(resolver, doc -> { + try { + if (type.isInstance(doc) == false) { + return false; + } + if (system != MonitoredSystem.fromSystem(doc.getMonitoringId())) { + return false; + } + return version == Version.fromString(doc.getMonitoringVersion()); + } catch (Exception e) { + return false; + } + }); + } + static class Registration { private final MonitoringIndexNameResolver resolver; @@ -106,5 +133,4 @@ public class ResolversRegistry implements Iterable return resolver; } } - } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java new file mode 100644 index 00000000000..e92b3e34452 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.resolver.bulk; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.marvel.MonitoredSystem; +import org.elasticsearch.marvel.action.MonitoringBulkDoc; +import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; + +import java.io.IOException; + +public class MonitoringBulkResolver extends MonitoringIndexNameResolver.Timestamped { + + public MonitoringBulkResolver(MonitoredSystem id, int version, Settings settings) { + super(id, version, settings); + } + + @Override + public String type(MonitoringBulkDoc document) { + return document.getType(); + } + + @Override + protected void buildXContent(MonitoringBulkDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException { + BytesReference source = document.getSource(); + if (source != null && source.length() > 0) { + builder.rawField(type(document), source); + } + } +} \ No newline at end of file diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClient.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClient.java index 596096ebb6f..36072fc73ad 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClient.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/client/MonitoringClient.java @@ -5,8 +5,16 @@ */ package org.elasticsearch.marvel.client; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.marvel.action.MonitoringBulkAction; +import org.elasticsearch.marvel.action.MonitoringBulkRequest; +import org.elasticsearch.marvel.action.MonitoringBulkRequestBuilder; +import org.elasticsearch.marvel.action.MonitoringBulkResponse; + +import java.util.Map; public class MonitoringClient { @@ -17,5 +25,36 @@ public class MonitoringClient { this.client = client; } - // to be implemented: specific API for monitoring + + /** + * Creates a request builder that bulk index monitoring documents. + * + * @return The request builder + */ + public MonitoringBulkRequestBuilder prepareMonitoringBulk() { + return new MonitoringBulkRequestBuilder(client); + } + + /** + * Executes a bulk of index operations that concern monitoring documents. + * + * @param request The monitoring bulk request + * @param listener A listener to be notified with a result + */ + public void bulk(MonitoringBulkRequest request, ActionListener listener) { + client.execute(MonitoringBulkAction.INSTANCE, request, listener); + } + + /** + * Executes a bulk of index operations that concern monitoring documents. + * + * @param request The monitoring bulk request + */ + public ActionFuture bulk(MonitoringBulkRequest request) { + return client.execute(MonitoringBulkAction.INSTANCE, request); + } + + public MonitoringClient filterWithHeader(Map headers) { + return new MonitoringClient(client.filterWithHeader(headers)); + } } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/MonitoringRestHandler.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/MonitoringRestHandler.java new file mode 100644 index 00000000000..ab67bb39016 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/MonitoringRestHandler.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.rest; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.marvel.client.MonitoringClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.xpack.XPackPlugin; + +import java.util.Locale; + +public abstract class MonitoringRestHandler extends BaseRestHandler { + + protected static String URI_BASE = String.format(Locale.ROOT, "/_%s/monitoring", XPackPlugin.NAME); + + public MonitoringRestHandler(Settings settings, Client client) { + super(settings, client); + } + + @Override + protected final void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception { + handleRequest(request, channel, new MonitoringClient(client)); + } + + protected abstract void handleRequest(RestRequest request, RestChannel channel, MonitoringClient client) throws Exception; +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/action/RestMonitoringBulkAction.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/action/RestMonitoringBulkAction.java new file mode 100644 index 00000000000..16cfaa81cd2 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/action/RestMonitoringBulkAction.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.rest.action; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.marvel.action.MonitoringBulkRequestBuilder; +import org.elasticsearch.marvel.action.MonitoringBulkResponse; +import org.elasticsearch.marvel.client.MonitoringClient; +import org.elasticsearch.marvel.rest.MonitoringRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.action.support.RestActions; +import org.elasticsearch.rest.action.support.RestBuilderListener; + +public class RestMonitoringBulkAction extends MonitoringRestHandler { + + public static final String MONITORING_ID = "system_id"; + public static final String MONITORING_VERSION = "system_version"; + + @Inject + public RestMonitoringBulkAction(Settings settings, RestController controller, Client client) { + super(settings, client); + controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/_bulk", this); + controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/_bulk", this); + controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/{index}/_bulk", this); + controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/{index}/_bulk", this); + controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/{index}/{type}/_bulk", this); + controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/{index}/{type}/_bulk", this); + } + + @Override + protected void handleRequest(RestRequest request, RestChannel channel, MonitoringClient client) throws Exception { + String defaultIndex = request.param("index"); + String defaultType = request.param("type"); + + String id = request.param(MONITORING_ID); + if (Strings.hasLength(id) == false) { + throw new IllegalArgumentException("no monitoring id for monitoring bulk request"); + } + String version = request.param(MONITORING_VERSION); + if (Strings.hasLength(version) == false) { + throw new IllegalArgumentException("no monitoring version for monitoring bulk request"); + } + + if (!RestActions.hasBodyContent(request)) { + throw new ElasticsearchParseException("no body content for monitoring bulk request"); + } + + MonitoringBulkRequestBuilder requestBuilder = client.prepareMonitoringBulk(); + requestBuilder.add(request.content(), id, version, defaultIndex, defaultType); + requestBuilder.execute(new RestBuilderListener(channel) { + @Override + public RestResponse buildResponse(MonitoringBulkResponse response, XContentBuilder builder) throws Exception { + builder.startObject(); + builder.field(Fields.TOOK, response.getTookInMillis()); + + MonitoringBulkResponse.Error error = response.getError(); + builder.field(Fields.ERRORS, error != null); + + if (error != null) { + builder.field(Fields.ERROR, response.getError()); + } + builder.endObject(); + return new BytesRestResponse(response.status(), builder); + } + }); + } + + static final class Fields { + static final XContentBuilderString TOOK = new XContentBuilderString("took"); + static final XContentBuilderString ERRORS = new XContentBuilderString("errors"); + static final XContentBuilderString ERROR = new XContentBuilderString("error"); + } +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/support/init/proxy/MonitoringClientProxy.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/support/init/proxy/MonitoringClientProxy.java index f09877b3590..4c7c6bc53ea 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/support/init/proxy/MonitoringClientProxy.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/support/init/proxy/MonitoringClientProxy.java @@ -6,16 +6,11 @@ package org.elasticsearch.marvel.support.init.proxy; import org.elasticsearch.client.Client; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.shield.InternalClient; import org.elasticsearch.xpack.common.init.proxy.ClientProxy; public class MonitoringClientProxy extends ClientProxy { - @Inject - public MonitoringClientProxy() { - } - /** * Creates a proxy to the given internal client (can be used for testing) */ diff --git a/elasticsearch/x-pack/marvel/src/main/resources/monitoring-data.json b/elasticsearch/x-pack/marvel/src/main/resources/monitoring-data.json index b01dac11022..6b7f62fd116 100644 --- a/elasticsearch/x-pack/marvel/src/main/resources/monitoring-data.json +++ b/elasticsearch/x-pack/marvel/src/main/resources/monitoring-data.json @@ -1,7 +1,6 @@ { "template": ".monitoring-data-${monitoring.template.version}", "settings": { - "index.xpack.version": "${project.version}", "index.number_of_shards": 1, "index.number_of_replicas": 1, "index.codec": "best_compression", @@ -9,7 +8,10 @@ }, "mappings": { "cluster_info": { - "enabled": false + "enabled": false, + "_meta": { + "xpack.version": "${project.version}" + } }, "node": { "enabled": false diff --git a/elasticsearch/x-pack/marvel/src/main/resources/monitoring-es.json b/elasticsearch/x-pack/marvel/src/main/resources/monitoring-es.json index 6ed747e3c17..5919d19c1d8 100644 --- a/elasticsearch/x-pack/marvel/src/main/resources/monitoring-es.json +++ b/elasticsearch/x-pack/marvel/src/main/resources/monitoring-es.json @@ -1,7 +1,6 @@ { "template": ".monitoring-es-${monitoring.template.version}-*", "settings": { - "index.xpack.version": "${project.version}", "index.number_of_shards": 1, "index.number_of_replicas": 1, "index.codec": "best_compression", diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java index a85d8456bd1..6eb4d8bbb0f 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java @@ -22,7 +22,8 @@ public class MarvelPluginClientTests extends ESTestCase { .build(); Marvel plugin = new Marvel(settings); - assertThat(plugin.isEnabled(), is(false)); + assertThat(plugin.isEnabled(), is(true)); + assertThat(plugin.isTransportClient(), is(true)); Collection modules = plugin.nodeModules(); assertThat(modules.size(), is(0)); } @@ -34,6 +35,7 @@ public class MarvelPluginClientTests extends ESTestCase { .build(); Marvel plugin = new Marvel(settings); assertThat(plugin.isEnabled(), is(true)); + assertThat(plugin.isTransportClient(), is(false)); Collection modules = plugin.nodeModules(); assertThat(modules.size(), is(5)); } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkDocTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkDocTests.java new file mode 100644 index 00000000000..7dc931dfe57 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkDocTests.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.elasticsearch.test.VersionUtils.randomVersion; +import static org.hamcrest.Matchers.equalTo; + +public class MonitoringBulkDocTests extends ESTestCase { + + public void testSerialization() throws IOException { + int iterations = randomIntBetween(5, 50); + for (int i = 0; i < iterations; i++) { + MonitoringBulkDoc doc = newRandomMonitoringBulkDoc(); + + boolean hasSourceNode = randomBoolean(); + if (hasSourceNode) { + doc.setSourceNode(newRandomSourceNode()); + } + + BytesStreamOutput output = new BytesStreamOutput(); + Version outputVersion = randomVersion(random()); + output.setVersion(outputVersion); + doc.writeTo(output); + + StreamInput streamInput = StreamInput.wrap(output.bytes()); + streamInput.setVersion(randomVersion(random())); + MonitoringBulkDoc doc2 = new MonitoringBulkDoc(streamInput); + + assertThat(doc2.getMonitoringId(), equalTo(doc.getMonitoringId())); + assertThat(doc2.getMonitoringVersion(), equalTo(doc.getMonitoringVersion())); + assertThat(doc2.getClusterUUID(), equalTo(doc.getClusterUUID())); + assertThat(doc2.getTimestamp(), equalTo(doc.getTimestamp())); + assertThat(doc2.getSourceNode(), equalTo(doc.getSourceNode())); + assertThat(doc2.getIndex(), equalTo(doc.getIndex())); + assertThat(doc2.getType(), equalTo(doc.getType())); + assertThat(doc2.getId(), equalTo(doc.getId())); + if (doc.getSource() == null) { + assertThat(doc2.getSource(), equalTo(BytesArray.EMPTY)); + } else { + assertThat(doc2.getSource(), equalTo(doc.getSource())); + } + } + } + + private MonitoringBulkDoc newRandomMonitoringBulkDoc() { + MonitoringBulkDoc doc = new MonitoringBulkDoc(randomAsciiOfLength(2), randomAsciiOfLength(2)); + if (frequently()) { + doc.setClusterUUID(randomAsciiOfLength(5)); + doc.setType(randomAsciiOfLength(5)); + } + if (randomBoolean()) { + doc.setTimestamp(System.currentTimeMillis()); + doc.setSource(new BytesArray("{\"key\" : \"value\"}")); + } + if (rarely()) { + doc.setIndex(randomAsciiOfLength(5)); + doc.setId(randomAsciiOfLength(2)); + } + return doc; + } + + private MonitoringDoc.Node newRandomSourceNode() { + String uuid = null; + String name = null; + String ip = null; + String transportAddress = null; + String host = null; + ImmutableOpenMap attributes = null; + + if (frequently()) { + uuid = randomAsciiOfLength(5); + name = randomAsciiOfLength(5); + } + if (randomBoolean()) { + ip = randomAsciiOfLength(5); + transportAddress = randomAsciiOfLength(5); + host = randomAsciiOfLength(3); + } + if (rarely()) { + int nbAttributes = randomIntBetween(0, 5); + + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + for (int i = 0; i < nbAttributes; i++) { + builder.put("key#" + i, String.valueOf(i)); + } + attributes = builder.build(); + } + return new MonitoringDoc.Node(uuid, host, transportAddress, ip, name, attributes); + } + +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkRequestTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkRequestTests.java new file mode 100644 index 00000000000..4e9f2ef9e79 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkRequestTests.java @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.CoreMatchers; +import org.hamcrest.Matcher; + +import java.io.IOException; + +import static org.elasticsearch.test.VersionUtils.randomVersion; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.nullValue; + +public class MonitoringBulkRequestTests extends ESTestCase { + + private static final BytesArray SOURCE = new BytesArray("{\"key\" : \"value\"}"); + + public void testValidateRequestNoDocs() { + assertValidationErrors(new MonitoringBulkRequest(), hasItems("no monitoring documents added")); + } + + public void testValidateRequestSingleDoc() { + MonitoringBulkDoc doc = new MonitoringBulkDoc(null, null); + + assertValidationErrors(new MonitoringBulkRequest().add(doc), hasItems("monitored system id is missing for monitoring document [0]", + "monitored system version is missing for monitoring document [0]", + "type is missing for monitoring document [0]", + "source is missing for monitoring document [0]")); + + doc = new MonitoringBulkDoc("id", null); + assertValidationErrors(new MonitoringBulkRequest().add(doc), + hasItems("monitored system version is missing for monitoring document [0]", + "type is missing for monitoring document [0]", + "source is missing for monitoring document [0]")); + + doc = new MonitoringBulkDoc("id", "version"); + assertValidationErrors(new MonitoringBulkRequest().add(doc), hasItems("type is missing for monitoring document [0]", + "source is missing for monitoring document [0]")); + + doc.setType("type"); + assertValidationErrors(new MonitoringBulkRequest().add(doc), hasItems("source is missing for monitoring document [0]")); + + doc.setSource(SOURCE); + assertValidationErrors(new MonitoringBulkRequest().add(doc), nullValue()); + } + + + public void testValidateRequestMultiDocs() { + MonitoringBulkRequest request = new MonitoringBulkRequest(); + + // Doc0 is complete + MonitoringBulkDoc doc0 = new MonitoringBulkDoc(randomAsciiOfLength(2), randomAsciiOfLength(2)); + doc0.setType(randomAsciiOfLength(5)); + doc0.setSource(SOURCE); + request.add(doc0); + + // Doc1 has no type + MonitoringBulkDoc doc1 = new MonitoringBulkDoc(randomAsciiOfLength(2), randomAsciiOfLength(2)); + doc1.setSource(SOURCE); + request.add(doc1); + + // Doc2 has no source + MonitoringBulkDoc doc2 = new MonitoringBulkDoc(randomAsciiOfLength(2), randomAsciiOfLength(2)); + doc2.setType(randomAsciiOfLength(5)); + doc2.setSource(BytesArray.EMPTY); + request.add(doc2); + + // Doc3 has no version + MonitoringBulkDoc doc3 = new MonitoringBulkDoc(randomAsciiOfLength(2), null); + doc3.setType(randomAsciiOfLength(5)); + doc3.setSource(SOURCE); + request.add(doc3); + + // Doc4 has no id + MonitoringBulkDoc doc4 = new MonitoringBulkDoc(null, randomAsciiOfLength(2)); + doc4.setType(randomAsciiOfLength(5)); + doc4.setSource(SOURCE); + request.add(doc4); + + assertValidationErrors(request, hasItems("type is missing for monitoring document [1]", + "source is missing for monitoring document [2]", + "monitored system version is missing for monitoring document [3]", + "monitored system id is missing for monitoring document [4]")); + + } + + public void testAddSingleDoc() { + MonitoringBulkRequest request = new MonitoringBulkRequest(); + final int nbDocs = randomIntBetween(1, 20); + for (int i = 0; i < nbDocs; i++) { + request.add(new MonitoringBulkDoc(String.valueOf(i), String.valueOf(i))); + } + assertThat(request.getDocs(), hasSize(nbDocs)); + } + + public void testAddMultipleDocs() throws Exception { + final int nbDocs = randomIntBetween(3, 20); + final XContentType xContentType = XContentType.JSON; + + try (BytesStreamOutput content = new BytesStreamOutput()) { + try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, content)) { + for (int i = 0; i < nbDocs; i++) { + builder.startObject().startObject("index").endObject().endObject().flush(); + content.write(xContentType.xContent().streamSeparator()); + builder.startObject().field("foo").value(i).endObject().flush(); + content.write(xContentType.xContent().streamSeparator()); + } + } + + String defaultMonitoringId = randomBoolean() ? randomAsciiOfLength(2) : null; + String defaultMonitoringVersion = randomBoolean() ? randomAsciiOfLength(3) : null; + String defaultIndex = randomBoolean() ? randomAsciiOfLength(5) : null; + String defaultType = randomBoolean() ? randomAsciiOfLength(4) : null; + + MonitoringBulkRequest request = new MonitoringBulkRequest(); + request.add(content.bytes(), defaultMonitoringId, defaultMonitoringVersion, defaultIndex, defaultType); + assertThat(request.getDocs(), hasSize(nbDocs)); + + for (MonitoringBulkDoc doc : request.getDocs()) { + assertThat(doc.getMonitoringId(), equalTo(defaultMonitoringId)); + assertThat(doc.getMonitoringVersion(), equalTo(defaultMonitoringVersion)); + assertThat(doc.getIndex(), equalTo(defaultIndex)); + assertThat(doc.getType(), equalTo(defaultType)); + } + } + } + + public void testSerialization() throws IOException { + MonitoringBulkRequest request = new MonitoringBulkRequest(); + + int numDocs = iterations(10, 30); + for (int i = 0; i < numDocs; i++) { + MonitoringBulkDoc doc = new MonitoringBulkDoc(randomAsciiOfLength(2), randomVersion(random()).toString()); + doc.setType(randomFrom("type1", "type2", "type3")); + doc.setSource(SOURCE); + if (randomBoolean()) { + doc.setIndex("index"); + } + if (randomBoolean()) { + doc.setId(randomAsciiOfLength(3)); + } + if (rarely()) { + doc.setClusterUUID(randomAsciiOfLength(5)); + } + request.add(doc); + } + + BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(randomVersion(random())); + request.writeTo(out); + + StreamInput in = StreamInput.wrap(out.bytes()); + in.setVersion(out.getVersion()); + MonitoringBulkRequest request2 = new MonitoringBulkRequest(); + request2.readFrom(in); + + assertThat(request2.docs.size(), CoreMatchers.equalTo(request.docs.size())); + for (int i = 0; i < request2.docs.size(); i++) { + MonitoringBulkDoc doc = request.docs.get(i); + MonitoringBulkDoc doc2 = request2.docs.get(i); + assertThat(doc2.getMonitoringId(), equalTo(doc.getMonitoringId())); + assertThat(doc2.getMonitoringVersion(), equalTo(doc.getMonitoringVersion())); + assertThat(doc2.getClusterUUID(), equalTo(doc.getClusterUUID())); + assertThat(doc2.getIndex(), equalTo(doc.getIndex())); + assertThat(doc2.getType(), equalTo(doc.getType())); + assertThat(doc2.getId(), equalTo(doc.getId())); + assertThat(doc2.getSource(), equalTo(doc.getSource())); + } + } + + @SuppressWarnings("unchecked") + private static void assertValidationErrors(MonitoringBulkRequest request, Matcher matcher) { + ActionRequestValidationException validation = request.validate(); + if (validation != null) { + assertThat((T) validation.validationErrors(), matcher); + } else { + assertThat(null, matcher); + } + } +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkResponseTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkResponseTests.java new file mode 100644 index 00000000000..8c414555277 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkResponseTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.marvel.agent.exporter.ExportException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.elasticsearch.test.VersionUtils.randomVersion; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class MonitoringBulkResponseTests extends ESTestCase { + + public void testResponseStatus() { + final long took = Math.abs(randomLong()); + MonitoringBulkResponse response = new MonitoringBulkResponse(took); + + assertThat(response.getTookInMillis(), equalTo(took)); + assertThat(response.getError(), is(nullValue())); + assertThat(response.status(), equalTo(RestStatus.OK)); + + ExportException exception = new ExportException(randomAsciiOfLength(10)); + response = new MonitoringBulkResponse(took, new MonitoringBulkResponse.Error(exception)); + + assertThat(response.getTookInMillis(), equalTo(took)); + assertThat(response.getError(), is(notNullValue())); + assertThat(response.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); + } + + public void testSerialization() throws IOException { + int iterations = randomIntBetween(5, 50); + for (int i = 0; i < iterations; i++) { + MonitoringBulkResponse response; + if (randomBoolean()) { + response = new MonitoringBulkResponse(Math.abs(randomLong())); + } else { + Exception exception = randomFrom( + new ExportException(randomAsciiOfLength(5), new IllegalStateException(randomAsciiOfLength(5))), + new IllegalStateException(randomAsciiOfLength(5)), + new IllegalArgumentException(randomAsciiOfLength(5))); + response = new MonitoringBulkResponse(Math.abs(randomLong()), new MonitoringBulkResponse.Error(exception)); + } + + BytesStreamOutput output = new BytesStreamOutput(); + Version outputVersion = randomVersion(random()); + output.setVersion(outputVersion); + response.writeTo(output); + + StreamInput streamInput = StreamInput.wrap(output.bytes()); + streamInput.setVersion(randomVersion(random())); + MonitoringBulkResponse response2 = new MonitoringBulkResponse(); + response2.readFrom(streamInput); + + assertThat(response2.getTookInMillis(), equalTo(response.getTookInMillis())); + if (response.getError() == null) { + assertThat(response2.getError(), is(nullValue())); + } else { + assertThat(response2.getError(), is(notNullValue())); + } + } + } +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkTests.java new file mode 100644 index 00000000000..d5644708f6a --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkTests.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.marvel.MonitoredSystem; +import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver; +import org.elasticsearch.marvel.test.MarvelIntegTestCase; +import org.elasticsearch.search.SearchHit; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class MonitoringBulkTests extends MarvelIntegTestCase { + + @Override + protected Settings transportClientSettings() { + return super.transportClientSettings(); + } + + public void testMonitoringBulkIndexing() throws Exception { + MonitoringBulkRequestBuilder requestBuilder = monitoringClient().prepareMonitoringBulk(); + String[] types = {"type1", "type2", "type3"}; + + int numDocs = scaledRandomIntBetween(100, 5000); + for (int i = 0; i < numDocs; i++) { + MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(), Version.CURRENT.toString()); + doc.setType(randomFrom(types)); + doc.setSource(jsonBuilder().startObject().field("num", numDocs).endObject().bytes()); + requestBuilder.add(doc); + } + + MonitoringBulkResponse response = requestBuilder.get(); + assertThat(response.getError(), is(nullValue())); + refresh(); + + SearchResponse searchResponse = client().prepareSearch().setTypes(types).setSize(numDocs).get(); + assertHitCount(searchResponse, numDocs); + + for (SearchHit searchHit : searchResponse.getHits()) { + Map source = searchHit.sourceAsMap(); + assertNotNull(source.get(MonitoringBulkResolver.Fields.CLUSTER_UUID.underscore().toString())); + assertNotNull(source.get(MonitoringBulkResolver.Fields.TIMESTAMP.underscore().toString())); + assertNotNull(source.get(MonitoringBulkResolver.Fields.SOURCE_NODE.underscore().toString())); + } + } + + /** + * This test creates N threads that execute a random number of monitoring bulk requests. + */ + public void testConcurrentRequests() throws Exception { + final Thread[] threads = new Thread[3 + randomInt(7)]; + final List exceptions = new CopyOnWriteArrayList<>(); + + AtomicInteger total = new AtomicInteger(0); + + logger.info("--> using {} concurrent clients to execute requests", threads.length); + for (int i = 0; i < threads.length; i++) { + final int nbRequests = randomIntBetween(3, 10); + + threads[i] = new Thread(new AbstractRunnable() { + @Override + public void onFailure(Throwable t) { + logger.error("unexpected error in exporting thread", t); + exceptions.add(t); + } + + @Override + protected void doRun() throws Exception { + for (int j = 0; j < nbRequests; j++) { + MonitoringBulkRequestBuilder requestBuilder = monitoringClient().prepareMonitoringBulk(); + + int numDocs = scaledRandomIntBetween(10, 1000); + for (int k = 0; k < numDocs; k++) { + MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(), Version.CURRENT.toString()); + doc.setType("concurrent"); + doc.setSource(jsonBuilder().startObject().field("num", k).endObject().bytes()); + requestBuilder.add(doc); + } + + total.addAndGet(numDocs); + MonitoringBulkResponse response = requestBuilder.get(); + assertThat(response.getError(), is(nullValue())); + } + } + }, "export_thread_" + i); + threads[i].start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + assertThat(exceptions, empty()); + refresh(); + + SearchResponse countResponse = client().prepareSearch().setTypes("concurrent").setSize(0).get(); + assertHitCount(countResponse, total.get()); + } + + public void testUnsupportedSystem() throws Exception { + MonitoringBulkRequestBuilder requestBuilder = monitoringClient().prepareMonitoringBulk(); + String[] types = {"type1", "type2", "type3"}; + + int totalDocs = randomIntBetween(10, 1000); + int unsupportedDocs = 0; + + for (int i = 0; i < totalDocs; i++) { + MonitoringBulkDoc doc; + if (randomBoolean()) { + doc = new MonitoringBulkDoc("unknown", Version.CURRENT.toString()); + unsupportedDocs++; + } else { + doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(), Version.CURRENT.toString()); + } + doc.setType(randomFrom(types)); + doc.setSource(jsonBuilder().startObject().field("num", i).endObject().bytes()); + requestBuilder.add(doc); + } + + MonitoringBulkResponse response = requestBuilder.get(); + if (unsupportedDocs == 0) { + assertThat(response.getError(), is(nullValue())); + } else { + assertThat(response.getError(), is(notNullValue())); + } + refresh(); + + SearchResponse countResponse = client().prepareSearch().setTypes(types).setSize(0).get(); + assertHitCount(countResponse, totalDocs - unsupportedDocs); + } +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/TransportMonitoringBulkActionTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/TransportMonitoringBulkActionTests.java new file mode 100644 index 00000000000..f99b2af4bf3 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/TransportMonitoringBulkActionTests.java @@ -0,0 +1,293 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.NodeConnectionsService; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.discovery.DiscoverySettings; +import org.elasticsearch.marvel.MarvelSettings; +import org.elasticsearch.marvel.MonitoredSystem; +import org.elasticsearch.marvel.agent.exporter.ExportException; +import org.elasticsearch.marvel.agent.exporter.Exporters; +import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.transport.CapturingTransport; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.VersionUtils.randomVersion; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.IsEqual.equalTo; + +public class TransportMonitoringBulkActionTests extends ESTestCase { + + private static ThreadPool threadPool; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private ClusterService clusterService; + private TransportService transportService; + private CapturingExporters exportService; + private TransportMonitoringBulkAction action; + + @BeforeClass + public static void beforeClass() { + threadPool = new ThreadPool(TransportMonitoringBulkActionTests.class.getSimpleName()); + } + + @AfterClass + public static void afterClass() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + + @Before + public void setUp() throws Exception { + super.setUp(); + CapturingTransport transport = new CapturingTransport(); + clusterService = new ClusterService(Settings.EMPTY, null, new ClusterSettings(Settings.EMPTY, + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool, + new ClusterName(TransportMonitoringBulkActionTests.class.getName())); + clusterService.setLocalNode(new DiscoveryNode("node", DummyTransportAddress.INSTANCE, Version.CURRENT)); + clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) { + @Override + public void connectToAddedNodes(ClusterChangedEvent event) { + // skip + } + + @Override + public void disconnectFromRemovedNodes(ClusterChangedEvent event) { + // skip + } + }); + clusterService.setClusterStatePublisher((event, ackListener) -> {}); + clusterService.start(); + + transportService = new TransportService(transport, threadPool); + transportService.start(); + transportService.acceptIncomingRequests(); + exportService = new CapturingExporters(); + action = new TransportMonitoringBulkAction( + Settings.EMPTY, + threadPool, + clusterService, + transportService, + new ActionFilters(Collections.emptySet()), + new IndexNameExpressionResolver(Settings.EMPTY), + exportService + ); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + clusterService.close(); + transportService.close(); + } + + public void testGlobalBlock() throws Exception { + expectedException.expect(ExecutionException.class); + expectedException.expect(hasToString(containsString("ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master]"))); + + final ClusterBlocks.Builder block = ClusterBlocks.builder().addGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ALL); + final CountDownLatch latch = new CountDownLatch(1); + + clusterService.submitStateUpdateTask("add blocks to cluster state", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + // make sure we increment versions as listener may depend on it for change + return ClusterState.builder(currentState).blocks(block).version(currentState.version() + 1).build(); + } + + @Override + public boolean runOnlyOnMaster() { + return false; + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + latch.countDown(); + } + + @Override + public void onFailure(String source, Throwable t) { + fail("unexpected exception: " + t); + } + }); + + try { + latch.await(); + } catch (InterruptedException e) { + throw new ElasticsearchException("unexpected interruption", e); + } + + MonitoringBulkRequest request = randomRequest(); + action.execute(request).get(); + } + + public void testEmptyRequest() throws Exception { + expectedException.expect(ExecutionException.class); + expectedException.expect(hasToString(containsString("no monitoring documents added"))); + + MonitoringBulkRequest request = randomRequest(0); + action.execute(request).get(); + + assertThat(exportService.getExported(), hasSize(0)); + } + + public void testBasicRequest() throws Exception { + MonitoringBulkRequest request = randomRequest(); + action.execute(request).get(); + + assertThat(exportService.getExported(), hasSize(request.getDocs().size())); + } + + public void testAsyncActionPrepareDocs() throws Exception { + final PlainActionFuture listener = new PlainActionFuture<>(); + final MonitoringBulkRequest request = randomRequest(); + + Collection results = action.new AsyncAction(request, listener, exportService, clusterService) + .prepareForExport(request.getDocs()); + + assertThat(results, hasSize(request.getDocs().size())); + for (MonitoringDoc exported : results) { + assertThat(exported.getClusterUUID(), equalTo(clusterService.state().metaData().clusterUUID())); + assertThat(exported.getTimestamp(), greaterThan(0L)); + assertThat(exported.getSourceNode(), notNullValue()); + assertThat(exported.getSourceNode().getUUID(), equalTo(clusterService.localNode().getId())); + assertThat(exported.getSourceNode().getName(), equalTo(clusterService.localNode().getName())); + } + } + + public void testAsyncActionExecuteExport() throws Exception { + final PlainActionFuture listener = new PlainActionFuture<>(); + final MonitoringBulkRequest request = randomRequest(); + final Collection docs = Collections.unmodifiableCollection(request.getDocs()); + + action.new AsyncAction(request, listener, exportService, clusterService).executeExport(docs, 0L, listener); + assertThat(listener.get().getError(), nullValue()); + + Collection exported = exportService.getExported(); + assertThat(exported, hasSize(request.getDocs().size())); + } + + public void testAsyncActionExportThrowsException() throws Exception { + final PlainActionFuture listener = new PlainActionFuture<>(); + final MonitoringBulkRequest request = randomRequest(); + + final Exporters exporters = new ConsumingExporters(docs -> { + throw new IllegalStateException(); + }); + + action.new AsyncAction(request, listener, exporters, clusterService).start(); + assertThat(listener.get().getError(), notNullValue()); + assertThat(listener.get().getError().getCause(), instanceOf(IllegalStateException.class)); + } + + /** + * @return a new MonitoringBulkRequest instance with random number of documents + */ + private static MonitoringBulkRequest randomRequest() throws IOException { + return randomRequest(scaledRandomIntBetween(1, 100)); + } + + /** + * @return a new MonitoringBulkRequest instance with given number of documents + */ + private static MonitoringBulkRequest randomRequest(final int numDocs) throws IOException { + MonitoringBulkRequest request = new MonitoringBulkRequest(); + for (int i = 0; i < numDocs; i++) { + MonitoringBulkDoc doc = new MonitoringBulkDoc(randomFrom(MonitoredSystem.values()).getSystem(), + randomVersion(random()).toString()); + doc.setType(randomFrom("type1", "type2")); + doc.setSource(jsonBuilder().startObject().field("num", i).endObject().bytes()); + request.add(doc); + } + return request; + } + + /** + * A Exporters implementation that captures the documents to export + */ + class CapturingExporters extends Exporters { + + private final Collection exported = ConcurrentCollections.newConcurrentSet(); + + public CapturingExporters() { + super(Settings.EMPTY, Collections.emptyMap(), clusterService, + new ClusterSettings(Settings.EMPTY, Collections.singleton(MarvelSettings.EXPORTERS_SETTINGS))); + } + + @Override + public synchronized void export(Collection docs) throws ExportException { + exported.addAll(docs); + } + + public Collection getExported() { + return exported; + } + } + + /** + * A Exporters implementation that applies a Consumer when exporting documents + */ + class ConsumingExporters extends Exporters { + + private final Consumer> consumer; + + public ConsumingExporters(Consumer> consumer) { + super(Settings.EMPTY, Collections.emptyMap(), clusterService, + new ClusterSettings(Settings.EMPTY, Collections.singleton(MarvelSettings.EXPORTERS_SETTINGS))); + this.consumer = consumer; + } + + @Override + public synchronized void export(Collection docs) throws ExportException { + consumer.accept(docs); + } + } + + public static void setState(ClusterService clusterService, ClusterState clusterState) { + + } +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java index e44f8c06a9c..66ae9e3595a 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/ExportersTests.java @@ -327,7 +327,6 @@ public class ExportersTests extends ESTestCase { } } - static class TestFactory extends Exporter.Factory { public TestFactory(String type, boolean singleton) { super(type, singleton); @@ -424,13 +423,13 @@ public class ExportersTests extends ESTestCase { } @Override - public ExportBulk add(Collection docs) throws Exception { + public ExportBulk add(Collection docs) throws ExportException { count.addAndGet(docs.size()); return this; } @Override - public void flush() throws Exception { + public void flush() throws ExportException { } AtomicInteger getCount() { diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MonitoringDocTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MonitoringDocTests.java index 6172dfc9ba4..298021e148d 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MonitoringDocTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MonitoringDocTests.java @@ -43,7 +43,7 @@ public class MonitoringDocTests extends ESTestCase { StreamInput streamInput = StreamInput.wrap(output.bytes()); streamInput.setVersion(randomVersion(random())); - MonitoringDoc monitoringDoc2 = MonitoringDoc.readMonitoringDoc(streamInput); + MonitoringDoc monitoringDoc2 = new MonitoringDoc(streamInput); assertThat(monitoringDoc2.getMonitoringId(), equalTo(monitoringDoc.getMonitoringId())); assertThat(monitoringDoc2.getMonitoringVersion(), equalTo(monitoringDoc.getMonitoringVersion())); @@ -64,7 +64,7 @@ public class MonitoringDocTests extends ESTestCase { public void testSetSourceNode() { int iterations = randomIntBetween(5, 50); for (int i = 0; i < iterations; i++) { - MonitoringDoc monitoringDoc = new MonitoringDoc(); + MonitoringDoc monitoringDoc = new MonitoringDoc(null, null); if (randomBoolean()) { DiscoveryNode discoveryNode = newRandomDiscoveryNode(); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java index 5a39a514882..eb97432e3ba 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.marvel.MarvelSettings; import org.elasticsearch.marvel.MonitoredSystem; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateMonitoringDoc; import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryMonitoringDoc; +import org.elasticsearch.marvel.agent.exporter.ExportException; import org.elasticsearch.marvel.agent.exporter.Exporter; import org.elasticsearch.marvel.agent.exporter.Exporters; import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; @@ -39,7 +40,6 @@ import java.util.concurrent.atomic.AtomicLong; import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName; import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -167,7 +167,14 @@ public class LocalExporterTests extends MarvelIntegTestCase { logger.debug("--> exporting a second monitoring doc"); exporter.export(Collections.singletonList(newRandomMarvelDoc())); } catch (ElasticsearchException e) { - assertThat(e.getMessage(), allOf(containsString("failure in bulk execution"), containsString("IndexClosedException[closed]"))); + assertThat(e.getMessage(), containsString("failed to flush export bulk [_local]")); + assertThat(e.getCause(), instanceOf(ExportException.class)); + + ExportException cause = (ExportException) e.getCause(); + assertTrue(cause.hasExportExceptions()); + for (ExportException c : cause) { + assertThat(c.getMessage(), containsString("IndexClosedException[closed]")); + } assertNull(exporter.getBulk().requestBuilder); } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java index e80c4226862..3c2fdb0e9a1 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java @@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.startsWith; public abstract class MonitoringIndexNameResolverTestCase> extends ESTestCase { - private final ResolversRegistry resolversRegistry = new ResolversRegistry(Settings.EMPTY); + protected final ResolversRegistry resolversRegistry = new ResolversRegistry(Settings.EMPTY); /** * @return the {@link MonitoringIndexNameResolver} to test diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java new file mode 100644 index 00000000000..3ffeb72e4b2 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.resolver.bulk; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.marvel.MonitoredSystem; +import org.elasticsearch.marvel.action.MonitoringBulkDoc; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; +import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolverTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class MonitoringBulkResolverTests extends MonitoringIndexNameResolverTestCase { + + @Override + protected MonitoringBulkDoc newMarvelDoc() { + MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(), Version.CURRENT.toString()); + doc.setClusterUUID(randomAsciiOfLength(5)); + doc.setTimestamp(Math.abs(randomLong())); + doc.setSourceNode(new DiscoveryNode("id", DummyTransportAddress.INSTANCE, Version.CURRENT)); + doc.setType("kibana_stats"); + doc.setSource(new BytesArray("{\"field1\" : \"value1\"}")); + return doc; + } + + @Override + protected boolean checkResolvedId() { + return false; + } + + @Override + protected boolean checkFilters() { + return false; + } + + public void testMonitoringBulkResolver() throws Exception { + MonitoringBulkDoc doc = newMarvelDoc(); + doc.setTimestamp(1437580442979L); + if (randomBoolean()) { + doc.setIndex(randomAsciiOfLength(5)); + } + if (randomBoolean()) { + doc.setId(randomAsciiOfLength(35)); + } + if (randomBoolean()) { + doc.setClusterUUID(randomAsciiOfLength(5)); + } + + MonitoringBulkResolver resolver = newResolver(); + assertThat(resolver.index(doc), equalTo(".monitoring-kibana-0-2015.07.22")); + assertThat(resolver.type(doc), equalTo(doc.getType())); + assertThat(resolver.id(doc), nullValue()); + + assertSource(resolver.source(doc, XContentType.JSON), + "cluster_uuid", + "timestamp", + "source_node", + "kibana_stats", + "kibana_stats.field1"); + } +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java index 9e4c8dbbd4e..b7af75d0320 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java @@ -442,9 +442,9 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { public static final String ROLES = "test:\n" + // a user for the test infra. " cluster: [ 'cluster:monitor/nodes/info', 'cluster:monitor/state', 'cluster:monitor/health', 'cluster:monitor/stats'," + - " 'cluster:admin/settings/update', 'cluster:admin/repository/delete', 'cluster:monitor/nodes/liveness'," + - " 'indices:admin/template/get', 'indices:admin/template/put', 'indices:admin/template/delete'," + - " 'cluster:monitor/task']\n" + + " 'cluster:admin/settings/update', 'cluster:admin/repository/delete', 'cluster:monitor/nodes/liveness'," + + " 'indices:admin/template/get', 'indices:admin/template/put', 'indices:admin/template/delete'," + + " 'cluster:monitor/task', 'cluster:admin/xpack/monitoring/bulk' ]\n" + " indices:\n" + " - names: '*'\n" + " privileges: [ all ]\n" + diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java index 17e835d3d97..df0ab3e000a 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java @@ -163,6 +163,7 @@ public class XPackPlugin extends Plugin { public void onModule(NetworkModule module) { licensing.onModule(module); + marvel.onModule(module); shield.onModule(module); watcher.onModule(module); graph.onModule(module); @@ -170,6 +171,7 @@ public class XPackPlugin extends Plugin { public void onModule(ActionModule module) { licensing.onModule(module); + marvel.onModule(module); shield.onModule(module); watcher.onModule(module); graph.onModule(module); From 52a91d7c6fde0b70565b32992a43b699826c4738 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 23 Mar 2016 11:10:32 +0100 Subject: [PATCH 7/8] Fix compilation. Original commit: elastic/x-pack-elasticsearch@83e6882b101705baa76f74ab6baedf3c4af61f5c --- .../FieldDataCacheWithFieldSubsetReaderTests.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java index 723c6edb73c..000b967979c 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java @@ -23,14 +23,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; -import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.index.mapper.core.TextFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -59,12 +57,13 @@ public class FieldDataCacheWithFieldSubsetReaderTests extends ESTestCase { IndexSettings indexSettings = createIndexSettings(); CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); String name = "_field"; - FieldDataType fieldDataType = new StringFieldMapper.StringFieldType().fieldDataType(); indexFieldDataCache = new DummyAccountingFieldDataCache(); sortedSetDVOrdinalsIndexFieldData = new SortedSetDVOrdinalsIndexFieldData(indexSettings,indexFieldDataCache, name, - circuitBreakerService, fieldDataType); - pagedBytesIndexFieldData = new PagedBytesIndexFieldData(indexSettings, name, fieldDataType, indexFieldDataCache, circuitBreakerService); + pagedBytesIndexFieldData = new PagedBytesIndexFieldData(indexSettings, name, indexFieldDataCache, + circuitBreakerService, TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, + TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, + TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE); dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(null); From 71542594e6c5a4ca6df14241f2f6181f7ba88a7f Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 17 Mar 2016 09:12:12 +0100 Subject: [PATCH 8/8] ShieldIndexSearcherWrapper should create the scorer only once. elastic/elasticsearch#1725 Currently it first creates a scorer, then checks if the role bits are sparse, and falls back to the bulk scorer if they are dense. The issue is that creating scorers and bulk scorers is very expensive on some queries such as ranges, prefix and terms queries. So it should rather check whether bits are sparse first in order to decide whether to use the scorer or bulk scorer. Original commit: elastic/x-pack-elasticsearch@067d630099a368bd7f10bc2008e6ae101f59afd7 --- .../accesscontrol/DocumentSubsetReader.java | 3 +- .../ShieldIndexSearcherWrapper.java | 101 ++++++----- .../DocumentLevelSecurityTests.java | 2 +- .../ShieldIndexSearcherWrapperUnitTests.java | 160 ++++++++++++++++++ 4 files changed, 220 insertions(+), 46 deletions(-) diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java index 6877054d43b..5a56a303dd6 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/DocumentSubsetReader.java @@ -26,7 +26,8 @@ import java.io.IOException; */ public final class DocumentSubsetReader extends FilterLeafReader { - public static DirectoryReader wrap(DirectoryReader in, BitsetFilterCache bitsetFilterCache, Query roleQuery) throws IOException { + public static DocumentSubsetDirectoryReader wrap(DirectoryReader in, BitsetFilterCache bitsetFilterCache, + Query roleQuery) throws IOException { return new DocumentSubsetDirectoryReader(in, bitsetFilterCache, roleQuery); } diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java index c2e6a2a7863..83229f46b44 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapper.java @@ -6,6 +6,7 @@ package org.elasticsearch.shield.authz.accesscontrol; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BulkScorer; @@ -156,50 +157,7 @@ public class ShieldIndexSearcherWrapper extends IndexSearcherWrapper { // The reasons why we return a custom searcher: // 1) in the case the role query is sparse then large part of the main query can be skipped // 2) If the role query doesn't match with any docs in a segment, that a segment can be skipped - IndexSearcher indexSearcher = new IndexSearcher(directoryReader) { - - @Override - protected void search(List leaves, Weight weight, Collector collector) throws IOException { - for (LeafReaderContext ctx : leaves) { // search each subreader - final LeafCollector leafCollector; - try { - leafCollector = collector.getLeafCollector(ctx); - } catch (CollectionTerminatedException e) { - // there is no doc of interest in this reader context - // continue with the following leaf - continue; - } - // The reader is always of type DocumentSubsetReader when we get here: - DocumentSubsetReader reader = (DocumentSubsetReader) ctx.reader(); - - BitSet roleQueryBits = reader.getRoleQueryBits(); - if (roleQueryBits == null) { - // nothing matches with the role query, so skip this segment: - continue; - } - - Scorer scorer = weight.scorer(ctx); - if (scorer != null) { - try { - // if the role query result set is sparse then we should use the SparseFixedBitSet for advancing: - if (roleQueryBits instanceof SparseFixedBitSet) { - SparseFixedBitSet sparseFixedBitSet = (SparseFixedBitSet) roleQueryBits; - Bits realLiveDocs = reader.getWrappedLiveDocs(); - intersectScorerAndRoleBits(scorer, sparseFixedBitSet, leafCollector, realLiveDocs); - } else { - BulkScorer bulkScorer = weight.bulkScorer(ctx); - Bits liveDocs = reader.getLiveDocs(); - bulkScorer.score(leafCollector, liveDocs); - } - } catch (CollectionTerminatedException e) { - // collection was terminated prematurely - // continue with the following leaf - } - } - - } - } - }; + IndexSearcher indexSearcher = new IndexSearcherWrapper((DocumentSubsetDirectoryReader) directoryReader); indexSearcher.setQueryCache(indexSearcher.getQueryCache()); indexSearcher.setQueryCachingPolicy(indexSearcher.getQueryCachingPolicy()); indexSearcher.setSimilarity(indexSearcher.getSimilarity(true)); @@ -208,6 +166,61 @@ public class ShieldIndexSearcherWrapper extends IndexSearcherWrapper { return searcher; } + static class IndexSearcherWrapper extends IndexSearcher { + + public IndexSearcherWrapper(DocumentSubsetDirectoryReader r) { + super(r); + } + + @Override + protected void search(List leaves, Weight weight, Collector collector) throws IOException { + for (LeafReaderContext ctx : leaves) { // search each subreader + final LeafCollector leafCollector; + try { + leafCollector = collector.getLeafCollector(ctx); + } catch (CollectionTerminatedException e) { + // there is no doc of interest in this reader context + // continue with the following leaf + continue; + } + // The reader is always of type DocumentSubsetReader when we get here: + DocumentSubsetReader reader = (DocumentSubsetReader) ctx.reader(); + + BitSet roleQueryBits = reader.getRoleQueryBits(); + if (roleQueryBits == null) { + // nothing matches with the role query, so skip this segment: + continue; + } + + // if the role query result set is sparse then we should use the SparseFixedBitSet for advancing: + if (roleQueryBits instanceof SparseFixedBitSet) { + Scorer scorer = weight.scorer(ctx); + if (scorer != null) { + SparseFixedBitSet sparseFixedBitSet = (SparseFixedBitSet) roleQueryBits; + Bits realLiveDocs = reader.getWrappedLiveDocs(); + try { + intersectScorerAndRoleBits(scorer, sparseFixedBitSet, leafCollector, realLiveDocs); + } catch (CollectionTerminatedException e) { + // collection was terminated prematurely + // continue with the following leaf + } + } + } else { + BulkScorer bulkScorer = weight.bulkScorer(ctx); + if (bulkScorer != null) { + Bits liveDocs = reader.getLiveDocs(); + try { + bulkScorer.score(leafCollector, liveDocs); + } catch (CollectionTerminatedException e) { + // collection was terminated prematurely + // continue with the following leaf + } + } + } + } + } + } + public Set getAllowedMetaFields() { return allowedMetaFields; } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 48678fc9e63..66d7b9a9dd2 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -549,7 +549,7 @@ public class DocumentLevelSecurityTests extends ShieldIntegTestCase { searchResponse = client().prepareSearch("test") .setQuery(hasParentQuery("parent", matchAllQuery())) - .addSort("_id", SortOrder.ASC) + .addSort("_uid", SortOrder.ASC) .get(); assertHitCount(searchResponse, 3L); assertThat(searchResponse.getHits().getAt(0).id(), equalTo("c1")); diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java index c31d90b4d04..a5d7a61fe57 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/accesscontrol/ShieldIndexSearcherWrapperUnitTests.java @@ -9,7 +9,9 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; @@ -17,15 +19,21 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.SparseFixedBitSet; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; @@ -43,6 +51,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.shield.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader; import org.elasticsearch.shield.license.ShieldLicenseState; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -51,6 +60,8 @@ import org.junit.Before; import java.io.IOException; import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; @@ -58,6 +69,7 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.shield.authz.accesscontrol.ShieldIndexSearcherWrapper.intersectScorerAndRoleBits; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @@ -370,4 +382,152 @@ public class ShieldIndexSearcherWrapperUnitTests extends ESTestCase { } } + public void testIndexSearcherWrapperSparseNoDeletions() throws IOException { + doTestIndexSearcherWrapper(true, false); + } + + public void testIndexSearcherWrapperDenseNoDeletions() throws IOException { + doTestIndexSearcherWrapper(false, false); + } + + public void testIndexSearcherWrapperSparseWithDeletions() throws IOException { + doTestIndexSearcherWrapper(true, true); + } + + public void testIndexSearcherWrapperDenseWithDeletions() throws IOException { + doTestIndexSearcherWrapper(false, true); + } + + static class CreateScorerOnceWeight extends Weight { + + private final Weight weight; + private final Set seenLeaves = Collections.newSetFromMap(new IdentityHashMap<>()); + + protected CreateScorerOnceWeight(Weight weight) { + super(weight.getQuery()); + this.weight = weight; + } + + @Override + public void extractTerms(Set terms) { + weight.extractTerms(terms); + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + return weight.explain(context, doc); + } + + @Override + public float getValueForNormalization() throws IOException { + return weight.getValueForNormalization(); + } + + @Override + public void normalize(float norm, float boost) { + weight.normalize(norm, boost); + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + assertTrue(seenLeaves.add(context.reader().getCoreCacheKey())); + return weight.scorer(context); + } + + @Override + public BulkScorer bulkScorer(LeafReaderContext context) + throws IOException { + assertTrue(seenLeaves.add(context.reader().getCoreCacheKey())); + return weight.bulkScorer(context); + } + } + + static class CreateScorerOnceQuery extends Query { + + private final Query query; + + CreateScorerOnceQuery(Query query) { + this.query = query; + } + + @Override + public String toString(String field) { + return query.toString(field); + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + Query queryRewritten = query.rewrite(reader); + if (query != queryRewritten) { + return new CreateScorerOnceQuery(queryRewritten); + } + return super.rewrite(reader); + } + + @Override + public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + return new CreateScorerOnceWeight(query.createWeight(searcher, needsScores)); + } + } + + public void doTestIndexSearcherWrapper(boolean sparse, boolean deletions) throws IOException { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(null)); + Document doc = new Document(); + StringField allowedField = new StringField("allowed", "yes", Store.NO); + doc.add(allowedField); + StringField fooField = new StringField("foo", "bar", Store.NO); + doc.add(fooField); + StringField deleteField = new StringField("delete", "no", Store.NO); + doc.add(deleteField); + w.addDocument(doc); + if (deletions) { + // add a document that matches foo:bar but will be deleted + deleteField.setStringValue("yes"); + w.addDocument(doc); + deleteField.setStringValue("no"); + } + allowedField.setStringValue("no"); + w.addDocument(doc); + if (sparse) { + for (int i = 0; i < 1000; ++i) { + w.addDocument(doc); + } + w.forceMerge(1); + } + w.deleteDocuments(new Term("delete", "yes")); + + DirectoryReader reader = DirectoryReader.open(w); + IndexSettings settings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY); + BitsetFilterCache.Listener listener = new BitsetFilterCache.Listener() { + @Override + public void onCache(ShardId shardId, Accountable accountable) { + + } + @Override + public void onRemoval(ShardId shardId, Accountable accountable) { + + } + }; + BitsetFilterCache cache = new BitsetFilterCache(settings, listener); + Query roleQuery = new TermQuery(new Term("allowed", "yes")); + BitSet bitSet = cache.getBitSetProducer(roleQuery).getBitSet(reader.leaves().get(0)); + if (sparse) { + assertThat(bitSet, instanceOf(SparseFixedBitSet.class)); + } else { + assertThat(bitSet, instanceOf(FixedBitSet.class)); + } + + DocumentSubsetDirectoryReader filteredReader = DocumentSubsetReader.wrap(reader, cache, roleQuery); + IndexSearcher searcher = new ShieldIndexSearcherWrapper.IndexSearcherWrapper(filteredReader); + + // Searching a non-existing term will trigger a null scorer + assertEquals(0, searcher.count(new TermQuery(new Term("non_existing_field", "non_existing_value")))); + + assertEquals(1, searcher.count(new TermQuery(new Term("foo", "bar")))); + + // make sure scorers are created only once, see #1725 + assertEquals(1, searcher.count(new CreateScorerOnceQuery(new MatchAllDocsQuery()))); + IOUtils.close(reader, w, dir); + } }