Fixed the integration tests

- prepared the ground for shield integration tests.
- for now, `shield` is disabled for all integration tests (once we have the `local` exporter we'll be able to turn shield on.

Original commit: elastic/x-pack-elasticsearch@cf5d660f58

marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterUtils.java

@@ -30,37 +30,6 @@ public class HttpExporterUtils {
     static final String MARVEL_VERSION_FIELD = "marvel_version";
     static final String VERSION_FIELD = "number";
 
-    public static String[] extractHostsFromAddress(BoundTransportAddress boundAddress, ESLogger logger) {
-        if (boundAddress == null || boundAddress.boundAddresses() == null) {
-            logger.debug("local http server is not yet started. can't connect");
-            return null;
-        }
-
-        TransportAddress[] boundAddresses = boundAddress.boundAddresses();
-        List<String> hosts = new ArrayList<>(boundAddresses.length);
-        for (TransportAddress transportAddress : boundAddresses) {
-            if (transportAddress.uniqueAddressTypeId() == 1) {
-                InetSocketTransportAddress address = (InetSocketTransportAddress) transportAddress;
-                InetSocketAddress inetSocketAddress = address.address();
-                InetAddress inetAddress = inetSocketAddress.getAddress();
-                if (inetAddress == null) {
-                    logger.error("failed to extract the ip address of from transport address [{}]", transportAddress);
-                    continue;
-                }
-                hosts.add(NetworkAddress.formatAddress(inetSocketAddress));
-            } else {
-                logger.error("local node http transport is not bound via a InetSocketTransportAddress. address is [{}] with typeId [{}]", transportAddress, transportAddress.uniqueAddressTypeId());
-            }
-        }
-
-        if (hosts.isEmpty()) {
-            logger.error("could not extract any hosts from bound address. can't connect");
-            return null;
-        }
-
-        return hosts.toArray(new String[hosts.size()]);
-    }
-
     public static URL parseHostWithPath(String host, String path) throws URISyntaxException, MalformedURLException {
 
         if (!host.contains("://")) {

marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java

@@ -39,7 +39,7 @@ public class MarvelPluginClientTests extends ESTestCase {
         MarvelPlugin plugin = new MarvelPlugin(settings);
         assertThat(plugin.isEnabled(), is(true));
         Collection<Module> modules = plugin.nodeModules();
-        assertThat(modules.size(), is(5));
+        assertThat(modules.size(), is(6));
     }
 
 }

marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginTests.java

@@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.plugin.LicensePlugin;
 import org.elasticsearch.marvel.agent.AgentService;
+import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginInfo;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -24,16 +25,13 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.hamcrest.Matchers.equalTo;
 
 @ClusterScope(scope = TEST, transportClientRatio = 0, numClientNodes = 0, numDataNodes = 0)
-public class MarvelPluginTests extends ESIntegTestCase {
+public class MarvelPluginTests extends MarvelIntegTestCase {
 
     @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
+    protected Settings nodeSettings(int nodeOrdinal) {
-        return Arrays.asList(LicensePlugin.class, MarvelPlugin.class);
+        return Settings.builder()
-    }
+                .put(super.nodeSettings(nodeOrdinal))
-
+                .build();
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return nodePlugins();
     }
 
     @Test

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java

@@ -25,6 +25,7 @@ import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.LicenseService;
 import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
 import org.elasticsearch.marvel.shield.SecuredClient;
+import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -36,17 +37,7 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 
 @ClusterScope(scope = ESIntegTestCase.Scope.SUITE, randomDynamicTemplates = false, transportClientRatio = 0.0)
-public class AbstractCollectorTestCase extends ESIntegTestCase {
+public class AbstractCollectorTestCase extends MarvelIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LicensePluginForCollectors.class, MarvelPlugin.class);
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return nodePlugins();
-    }
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/AbstractRendererTestCase.java

@@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.plugin.LicensePlugin;
 import org.elasticsearch.marvel.MarvelPlugin;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
+import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.ShieldPlugin;
@@ -27,7 +28,7 @@ import static org.hamcrest.Matchers.*;
 
 
 @ClusterScope(scope = ESIntegTestCase.Scope.SUITE, randomDynamicTemplates = false, transportClientRatio = 0.0)
-public abstract class AbstractRendererTestCase extends ESIntegTestCase {
+public abstract class AbstractRendererTestCase extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -44,16 +45,6 @@ public abstract class AbstractRendererTestCase extends ESIntegTestCase {
         return builder.build();
     }
 
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LicensePlugin.class, MarvelPlugin.class, ShieldPlugin.class);
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return nodePlugins();
-    }
-
     protected abstract Collection<String> collectors ();
 
     protected void waitForMarvelDocs(final String type) throws Exception {

marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java

@@ -10,6 +10,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.plugin.LicensePlugin;
 import org.elasticsearch.marvel.MarvelPlugin;
+import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -22,7 +23,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.hamcrest.Matchers.*;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
-public class MarvelSettingsTests extends ESIntegTestCase {
+public class MarvelSettingsTests extends MarvelIntegTestCase {
 
     private final TimeValue startUp = randomParsedTimeValue();
     private final TimeValue interval = randomParsedTimeValue();
@@ -44,16 +45,6 @@ public class MarvelSettingsTests extends ESIntegTestCase {
                 .build();
     }
 
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LicensePlugin.class, MarvelPlugin.class);
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return nodePlugins();
-    }
-
     private Settings marvelSettings() {
         return Settings.builder()
                 .put(MarvelSettings.STARTUP_DELAY, startUp)

marvel/src/test/java/org/elasticsearch/marvel/license/LicenseIntegrationTests.java

@@ -17,6 +17,7 @@ import org.elasticsearch.license.plugin.core.LicensesClientService;
 import org.elasticsearch.license.plugin.core.LicensesService;
 import org.elasticsearch.marvel.MarvelPlugin;
 import org.elasticsearch.marvel.mode.Mode;
+import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -28,16 +29,13 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.hamcrest.Matchers.*;
 
 @ClusterScope(scope = SUITE, transportClientRatio = 0, numClientNodes = 0)
-public class LicenseIntegrationTests extends ESIntegTestCase {
+public class LicenseIntegrationTests extends MarvelIntegTestCase {
 
     @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
+    protected Settings nodeSettings(int nodeOrdinal) {
-        return Arrays.asList(MockLicensePlugin.class, MarvelPlugin.class);
+        return Settings.builder()
-    }
+                .put(super.nodeSettings(nodeOrdinal))
-
+                .build();
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return nodePlugins();
     }
 
     @Test

marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.test;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.cache.IndexCacheModule;
+import org.elasticsearch.license.plugin.LicensePlugin;
+import org.elasticsearch.marvel.MarvelPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.shield.ShieldPlugin;
+import org.elasticsearch.shield.authc.esusers.ESUsersRealm;
+import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.crypto.InternalCryptoService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.TestCluster;
+import org.jboss.netty.util.internal.SystemPropertyUtil;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ *
+ */
+public class MarvelIntegTestCase extends ESIntegTestCase {
+
+    private static Boolean shieldEnabled;
+
+    @Override
+    protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException {
+        if (shieldEnabled == null) {
+            shieldEnabled = false; // enableShield();
+        }
+        return super.buildTestCluster(scope, seed);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                // we do this by default in core, but for marvel this isn't needed and only adds noise.
+                .put("index.store.mock.check_index_on_close", false)
+                .put(ShieldSettings.settings(shieldEnabled))
+                .build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        if (shieldEnabled) {
+            return Arrays.asList(LicensePlugin.class, MarvelPlugin.class, ShieldPlugin.class);
+        }
+        return Arrays.asList(LicensePlugin.class, MarvelPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return nodePlugins();
+    }
+
+    /**
+     * Override and returns {@code false} to force running without shield
+     */
+    protected boolean enableShield() {
+        return randomBoolean();
+    }
+
+    /** Shield related settings */
+
+    public static class ShieldSettings {
+
+        public static final String TEST_USERNAME = "test";
+        public static final String TEST_PASSWORD = "changeme";
+        private static final String TEST_PASSWORD_HASHED =  new String(Hasher.BCRYPT.hash(new SecuredString(TEST_PASSWORD.toCharArray())));
+
+        static boolean auditLogsEnabled = SystemPropertyUtil.getBoolean("tests.audit_logs", true);
+        static byte[] systemKey = generateKey(); // must be the same for all nodes
+
+        public static final String IP_FILTER = "allow: all\n";
+
+        public static final String USERS =
+                "transport_client:" + TEST_PASSWORD_HASHED + "\n" +
+                        TEST_USERNAME + ":" + TEST_PASSWORD_HASHED + "\n" +
+                        "admin:" + TEST_PASSWORD_HASHED + "\n" +
+                        "monitor:" + TEST_PASSWORD_HASHED;
+
+        public static final String USER_ROLES =
+                "transport_client:transport_client\n" +
+                        "test:test\n" +
+                        "admin:admin\n" +
+                        "monitor:monitor";
+
+        public static final String ROLES =
+                "test:\n" + // a user for the test infra.
+                        "  cluster: cluster:monitor/nodes/info, cluster:monitor/state, cluster:monitor/health, cluster:monitor/stats, cluster:admin/settings/update, cluster:admin/repository/delete, cluster:monitor/nodes/liveness, indices:admin/template/get, indices:admin/template/put, indices:admin/template/delete\n" +
+                        "  indices:\n" +
+                        "    '*': all\n" +
+                        "\n" +
+                        "admin:\n" +
+                        "  cluster: manage_watcher, cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n" +
+                        "transport_client:\n" +
+                        "  cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n" +
+                        "\n" +
+                        "monitor:\n" +
+                        "  cluster: monitor_watcher, cluster:monitor/nodes/info, cluster:monitor/nodes/liveness\n"
+                ;
+
+
+        public static Settings settings(boolean enabled)  {
+            Settings.Builder builder = Settings.builder();
+            if (!enabled) {
+                return builder.put("shield.enabled", false).build();
+            }
+            try {
+                Path folder = createTempDir().resolve("watcher_shield");
+                Files.createDirectories(folder);
+                return builder.put("shield.enabled", true)
+                        .put("shield.user", "test:changeme")
+                        .put("shield.authc.realms.esusers.type", ESUsersRealm.TYPE)
+                        .put("shield.authc.realms.esusers.order", 0)
+                        .put("shield.authc.realms.esusers.files.users", writeFile(folder, "users", USERS))
+                        .put("shield.authc.realms.esusers.files.users_roles", writeFile(folder, "users_roles", USER_ROLES))
+                        .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", ROLES))
+                        .put("shield.transport.n2n.ip_filter.file", writeFile(folder, "ip_filter.yml", IP_FILTER))
+                        .put("shield.system_key.file", writeFile(folder, "system_key.yml", systemKey))
+                        .put("shield.authc.sign_user_header", false)
+                        .put("shield.audit.enabled", auditLogsEnabled)
+                                // Test framework sometimes randomily selects the 'index' or 'none' cache and that makes the
+                                // validation in ShieldPlugin fail. Shield can only run with this query cache impl
+                        .put(IndexCacheModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE)
+                        .build();
+            } catch (IOException ex) {
+                throw new RuntimeException("failed to build settings for shield", ex);
+            }
+        }
+
+        static byte[] generateKey() {
+            try {
+                return InternalCryptoService.generateKey();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        public static String writeFile(Path folder, String name, String content) throws IOException {
+            Path file = folder.resolve(name);
+            try (BufferedWriter stream = Files.newBufferedWriter(file, StandardCharsets.UTF_8)) {
+                Streams.copy(content, stream);
+            } catch (IOException e) {
+                throw new ElasticsearchException("error writing file in test", e);
+            }
+            return file.toAbsolutePath().toString();
+        }
+
+        public static String writeFile(Path folder, String name, byte[] content) throws IOException {
+            Path file = folder.resolve(name);
+            try (OutputStream stream = Files.newOutputStream(file)) {
+                Streams.copy(content, stream);
+            } catch (IOException e) {
+                throw new ElasticsearchException("error writing file in test", e);
+            }
+            return file.toAbsolutePath().toString();
+        }
+    }
+}