Shield Integration

- Adds internal `__marvel_user`
- Adds a `SecuredClient` that binds the internal marvel user to all requests it sends

Original commit: elastic/x-pack-elasticsearch@77c4716261

marvel/pom.xml

@@ -35,6 +35,13 @@
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.elasticsearch.plugin</groupId>
+            <artifactId>shield</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
     </dependencies>
 
     <build>

marvel/src/main/java/org/elasticsearch/marvel/MarvelPlugin.java

@@ -23,6 +23,7 @@ import org.elasticsearch.marvel.agent.settings.MarvelSetting;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseModule;
 import org.elasticsearch.marvel.license.LicenseService;
+import org.elasticsearch.marvel.shield.MarvelShieldModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tribe.TribeService;
 
@@ -37,9 +38,11 @@ public class MarvelPlugin extends Plugin {
     public static final String NAME = "marvel";
     public static final String ENABLED = NAME + ".enabled";
 
+    private final Settings settings;
     private final boolean enabled;
 
     public MarvelPlugin(Settings settings) {
+        this.settings = settings;
         this.enabled = marvelEnabled(settings);
     }
 
@@ -66,7 +69,8 @@ public class MarvelPlugin extends Plugin {
             new MarvelModule(),
             new LicenseModule(),
             new CollectorModule(),
-            new ExporterModule(),
+            new ExporterModule(settings),
+            new MarvelShieldModule(settings),
             new RendererModule());
     }
 

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java

@@ -17,6 +17,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseService;
+import org.elasticsearch.marvel.shield.SecuredClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -41,7 +42,7 @@ public class ClusterInfoCollector extends AbstractCollector<ClusterInfoMarvelDoc
 
     @Inject
     public ClusterInfoCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
-                                ClusterName clusterName, Client client) {
+                                ClusterName clusterName, SecuredClient client) {
         super(settings, NAME, clusterService, marvelSettings, licenseService);
         this.clusterName = clusterName;
         this.licenseService = licenseService;

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java

@@ -15,6 +15,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseService;
+import org.elasticsearch.marvel.shield.SecuredClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -36,7 +37,7 @@ public class ClusterStateCollector extends AbstractCollector<ClusterStateCollect
 
     @Inject
     public ClusterStateCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,  LicenseService licenseService,
-                                 Client client) {
+                                 SecuredClient client) {
         super(settings, NAME, clusterService, marvelSettings, licenseService);
         this.client = client;
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollector.java

@@ -14,6 +14,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseService;
+import org.elasticsearch.marvel.shield.SecuredClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -35,7 +36,7 @@ public class ClusterStatsCollector extends AbstractCollector<ClusterStatsCollect
 
     @Inject
     public ClusterStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,  LicenseService licenseService,
-                                 Client client) {
+                                 SecuredClient client) {
         super(settings, NAME, clusterService, marvelSettings, licenseService);
         this.client = client;
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java

@@ -15,6 +15,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseService;
+import org.elasticsearch.marvel.shield.SecuredClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -36,7 +37,7 @@ public class IndexRecoveryCollector extends AbstractCollector<IndexRecoveryColle
 
     @Inject
     public IndexRecoveryCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,  LicenseService licenseService,
-                                  Client client) {
+                                  SecuredClient client) {
         super(settings, NAME, clusterService, marvelSettings, licenseService);
         this.client = client;
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java

@@ -16,6 +16,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseService;
+import org.elasticsearch.marvel.shield.SecuredClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -37,7 +38,7 @@ public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector>
 
     @Inject
     public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,  LicenseService licenseService,
-                               Client client) {
+                               SecuredClient client) {
         super(settings, NAME, clusterService, marvelSettings, licenseService);
         this.client = client;
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java

@@ -14,6 +14,7 @@ import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseService;
+import org.elasticsearch.marvel.shield.SecuredClient;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -32,7 +33,7 @@ public class IndicesStatsCollector extends AbstractCollector<IndicesStatsCollect
 
     @Inject
     public IndicesStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
-                                 Client client) {
+                                 SecuredClient client) {
         super(settings, NAME, clusterService, marvelSettings, licenseService);
         this.client = client;
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExporterModule.java

@@ -7,6 +7,7 @@ package org.elasticsearch.marvel.agent.exporter;
 
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
+import org.elasticsearch.common.settings.Settings;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -15,7 +16,13 @@ public class ExporterModule extends AbstractModule {
 
     private final Set<Class<? extends Exporter>> exporters = new HashSet<>();
 
-    public ExporterModule() {
+    private final Settings settings;
+
+    public ExporterModule(Settings settings) {
+        this.settings = settings;
+
+        // TODO do we need to choose what exporters to bind based on settings?
+
         // Registers default exporter
         registerExporter(HttpESExporter.class);
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/LocalExporter.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.agent.exporter;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.component.Lifecycle;
+import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.marvel.shield.SecuredClient;
+
+import java.util.Collection;
+
+/**
+ *
+ */
+public class LocalExporter implements Exporter<LocalExporter> {
+
+    public static final String NAME = "local";
+
+    private final Client client;
+
+    @Inject
+    public LocalExporter(SecuredClient client) {
+        this.client = client;
+    }
+
+    @Override
+    public String name() {
+        return NAME;
+    }
+
+    @Override
+    public void export(Collection<MarvelDoc> marvelDocs) {
+
+    }
+
+    @Override
+    public Lifecycle.State lifecycleState() {
+        return null;
+    }
+
+    @Override
+    public void addLifecycleListener(LifecycleListener lifecycleListener) {
+
+    }
+
+    @Override
+    public void removeLifecycleListener(LifecycleListener lifecycleListener) {
+
+    }
+
+    @Override
+    public LocalExporter start() {
+        return null;
+    }
+
+    @Override
+    public LocalExporter stop() {
+        return null;
+    }
+
+    @Override
+    public void close() {
+
+    }
+}

marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelInternalUserHolder.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.shield;
+
+import org.elasticsearch.marvel.agent.settings.MarvelSettings;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.authz.Permission;
+import org.elasticsearch.shield.authz.Privilege;
+import org.elasticsearch.transport.TransportMessage;
+
+/**
+ *
+ */
+public class MarvelInternalUserHolder {
+
+    static final String NAME = "__marvel_user";
+    static final String[] ROLE_NAMES = new String[] { "__marvel_role" };
+
+    public static final Permission.Global.Role ROLE = Permission.Global.Role.builder(ROLE_NAMES[0])
+            .cluster(Privilege.Cluster.action("indices:admin/template/put"))
+
+            // we need all monitoring access
+            .cluster(Privilege.Cluster.MONITOR)
+            .add(Privilege.Index.MONITOR, "*")
+
+            // and full access to .marvel-* and .marvel-data indices
+            .add(Privilege.Index.ALL, MarvelSettings.MARVEL_INDICES_PREFIX + "*")
+
+            // note, we don't need _licenses permission as we're taking the licenses
+            // directly form the license service.
+
+            .build();
+
+    final User user = new User.Simple(NAME, ROLE_NAMES);
+
+    public void bindUser(TransportMessage<?> message) {
+
+    }
+}

marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelSettingsFilter.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.shield;
+
+import org.elasticsearch.common.inject.Inject;
+
+/**
+ *
+ */
+public interface MarvelSettingsFilter {
+
+    void filterOut(String... patterns);
+
+    class Noop implements MarvelSettingsFilter {
+
+        public static Noop INSTANCE = new Noop();
+
+        private Noop() {
+        }
+
+        @Override
+        public void filterOut(String... patterns) {
+        }
+    }
+
+    class Shield implements MarvelSettingsFilter {
+
+        private final MarvelShieldIntegration shieldIntegration;
+
+        @Inject
+        public Shield(MarvelShieldIntegration shieldIntegration) {
+            this.shieldIntegration = shieldIntegration;
+        }
+
+        @Override
+        public void filterOut(String... patterns) {
+            shieldIntegration.filterOutSettings(patterns);
+        }
+    }
+}

marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelShieldIntegration.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.shield;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.HasContext;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.inject.Injector;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.ShieldPlugin;
+import org.elasticsearch.shield.ShieldSettingsFilter;
+import org.elasticsearch.shield.authc.AuthenticationService;
+import org.elasticsearch.transport.TransportMessage;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class MarvelShieldIntegration {
+
+    private final Object authcService;
+    private final Object userHolder;
+    private final Object settingsFilter;
+
+    @Inject
+    public MarvelShieldIntegration(Settings settings, Injector injector) {
+        boolean enabled = enabled(settings);
+        authcService = enabled ? injector.getInstance(AuthenticationService.class) : null;
+        userHolder = enabled ? injector.getInstance(MarvelInternalUserHolder.class) : null;
+        settingsFilter = enabled ? injector.getInstance(ShieldSettingsFilter.class) : null;
+    }
+
+    public void bindInternalMarvelUser(TransportMessage message) {
+        if (authcService != null) {
+            try {
+                ((AuthenticationService) authcService).attachUserHeaderIfMissing(message, ((MarvelInternalUserHolder) userHolder).user);
+            } catch (IOException e) {
+                throw new ElasticsearchException("failed to attach watcher user to request", e);
+            }
+        }
+    }
+
+    public void filterOutSettings(String... patterns) {
+        if (settingsFilter != null) {
+            ((ShieldSettingsFilter) settingsFilter).filterOut(patterns);
+        }
+    }
+
+    static boolean installed() {
+        try {
+            MarvelShieldIntegration.class.getClassLoader().loadClass("org.elasticsearch.shield.ShieldPlugin");
+            return true;
+        } catch (ClassNotFoundException e) {
+            return false;
+        }
+    }
+
+    public static boolean enabled(Settings settings) {
+        return installed() && ShieldPlugin.shieldEnabled(settings);
+    }
+
+}

marvel/src/main/java/org/elasticsearch/marvel/shield/MarvelShieldModule.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.shield;
+
+import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.inject.util.Providers;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authz.Privilege;
+
+/**
+ *
+ */
+public class MarvelShieldModule extends AbstractModule {
+
+    private final MarvelInternalUserHolder userHolder;
+    private final boolean enabled;
+
+    public MarvelShieldModule(Settings settings) {
+        this.enabled = MarvelShieldIntegration.enabled(settings);
+        userHolder = enabled ? new MarvelInternalUserHolder() : null;
+    }
+
+    @Override
+    protected void configure() {
+        bind(MarvelShieldIntegration.class).asEagerSingleton();
+        bind(SecuredClient.class).asEagerSingleton();
+        bind(MarvelInternalUserHolder.class).toProvider(Providers.of(userHolder));
+        if (enabled) {
+            bind(MarvelSettingsFilter.Shield.class).asEagerSingleton();
+            bind(MarvelSettingsFilter.class).to(MarvelSettingsFilter.Shield.class);
+        } else {
+            bind(MarvelSettingsFilter.class).toInstance(MarvelSettingsFilter.Noop.INSTANCE);
+        }
+    }
+}

marvel/src/test/java/org/elasticsearch/marvel/MarvelF.java

@@ -23,14 +23,14 @@ import java.util.concurrent.CountDownLatch;
  *
  * It accepts collectors names as program arguments.
  */
-public class MarvelLauncher {
+public class MarvelF {
 
     public static void main(String[] args) throws Throwable {
         Settings.Builder settings = Settings.builder();
         settings.put("script.inline", "on");
         settings.put("security.manager.enabled", "false");
         settings.put("plugins.load_classpath_plugins", "false");
-        settings.put("cluster.name", MarvelLauncher.class.getSimpleName());
+        settings.put("cluster.name", MarvelF.class.getSimpleName());
         settings.put("marvel.agent.interval", "5s");
         if (!CollectionUtils.isEmpty(args)) {
             settings.putArray("marvel.agent.collectors", args);

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java

@@ -23,6 +23,8 @@ import org.elasticsearch.license.plugin.core.LicensesService;
 import org.elasticsearch.marvel.MarvelPlugin;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseService;
+import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
+import org.elasticsearch.marvel.shield.SecuredClient;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -60,6 +62,16 @@ public class AbstractCollectorTestCase extends ESIntegTestCase {
         enableLicense();
     }
 
+    public SecuredClient securedClient() {
+        MarvelShieldIntegration integration = internalCluster().getInstance(MarvelShieldIntegration.class);
+        return new SecuredClient(client(), integration);
+    }
+
+    public SecuredClient securedClient(String nodeId) {
+        MarvelShieldIntegration integration = internalCluster().getInstance(MarvelShieldIntegration.class);
+        return new SecuredClient(client(nodeId), integration);
+    }
+
     protected void assertCanCollect(AbstractCollector collector) {
         assertNotNull(collector);
         assertTrue("collector [" + collector.name() + "] should be able to collect data", collector.canCollect());

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollectorTests.java

@@ -98,6 +98,6 @@ public class ClusterInfoCollectorTests extends AbstractCollectorTestCase {
                 internalCluster().getInstance(MarvelSettings.class, nodeId),
                 internalCluster().getInstance(LicenseService.class, nodeId),
                 internalCluster().getInstance(ClusterName.class, nodeId),
-                client(nodeId));
+                securedClient(nodeId));
     }
 }

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java

@@ -161,6 +161,6 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
                 internalCluster().getInstance(ClusterService.class, nodeId),
                 internalCluster().getInstance(MarvelSettings.class, nodeId),
                 internalCluster().getInstance(LicenseService.class, nodeId),
-                client(nodeId));
+                securedClient(nodeId));
     }
 }

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollectorTests.java

@@ -82,6 +82,6 @@ public class ClusterStatsCollectorTests extends AbstractCollectorTestCase {
                 internalCluster().getInstance(ClusterService.class, nodeId),
                 internalCluster().getInstance(MarvelSettings.class, nodeId),
                 internalCluster().getInstance(LicenseService.class, nodeId),
-                client(nodeId));
+                securedClient(nodeId));
     }
 }

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java

@@ -158,6 +158,6 @@ public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase {
                 internalCluster().getInstance(ClusterService.class, nodeId),
                 internalCluster().getInstance(MarvelSettings.class, nodeId),
                 internalCluster().getInstance(LicenseService.class, nodeId),
-                client(nodeId));
+                securedClient(nodeId));
     }
 }

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java

@@ -216,6 +216,6 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
                 internalCluster().getInstance(ClusterService.class, nodeId),
                 internalCluster().getInstance(MarvelSettings.class, nodeId),
                 internalCluster().getInstance(LicenseService.class, nodeId),
-                client(nodeId));
+                securedClient(nodeId));
     }
 }

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/AbstractRendererTestCase.java

@@ -14,6 +14,7 @@ import org.elasticsearch.marvel.MarvelPlugin;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
@@ -30,18 +31,22 @@ public abstract class AbstractRendererTestCase extends ESIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
+        Settings.Builder builder = Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
                 .put(Node.HTTP_ENABLED, true)
                 .put(MarvelSettings.STARTUP_DELAY, "3s")
                 .put(MarvelSettings.INTERVAL, "1s")
-                .put(MarvelSettings.COLLECTORS, Strings.collectionToCommaDelimitedString(collectors()))
-                .build();
+                .put(MarvelSettings.COLLECTORS, Strings.collectionToCommaDelimitedString(collectors()));
+
+        // we need to remove this potential setting for shield
+        builder.remove("index.queries.cache.type");
+
+        return builder.build();
     }
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LicensePlugin.class, MarvelPlugin.class);
+        return Arrays.asList(LicensePlugin.class, MarvelPlugin.class, ShieldPlugin.class);
     }
 
     @Override