Tribe node: add support for tribe node in shield

Disabled license check on the tribe node to make sure that the tribe node can start, otherwise license plugin would try to generate a new trial license which is not possible since the node has no master. License check still happens for tribes though. This will be improved once es core supports merging cluster level custom metadata, then the tribe node will see some license coming from its tribes and won't require any additional license.

Added integration test to verify basic functionality against a tribe node, which also validates the settings needed on the tribes.

Made sure that shield is loaded and enabled on very tribe if loaded and enabled on the tribe node. We want to make sure that nobody manages to use shield on the tribe node only for free (since we disabled liccensing there), with no shield on the tribes. If we forcibly enable and make the shield plugin mandatory on the tribe clients, it means that they will not be able to join their corresponding clusters unless they have shield loaded and enabled too. As a result, shield is supported in the tribe node as long as all the tribes have shield loaded and enabled too.

Relates to elastic/elasticsearch#311
Closes elastic/elasticsearch#584

Original commit: elastic/x-pack-elasticsearch@317add553f

src/main/java/org/elasticsearch/shield/ShieldPlugin.java

@@ -20,11 +20,11 @@ import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.authz.store.FileRolesStore;
 import org.elasticsearch.shield.license.LicenseService;
 import org.elasticsearch.shield.signature.InternalSignatureService;
-import org.elasticsearch.shield.signature.SignatureService;
 
 import java.io.File;
 import java.nio.file.Path;
 import java.util.Collection;
+import java.util.Map;
 
 /**
  *
@@ -33,6 +33,8 @@ public class ShieldPlugin extends AbstractPlugin {
 
     public static final String NAME = "shield";
 
+    public static final String ENABLED_SETTING_NAME = NAME + ".enabled";
+
     private final Settings settings;
     private final boolean enabled;
     private final boolean clientMode;
@@ -72,22 +74,73 @@ public class ShieldPlugin extends AbstractPlugin {
         if (!enabled) {
             return ImmutableSettings.EMPTY;
         }
-        String setting = Headers.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER;
-        if (settings.get(setting) != null) {
-            return ImmutableSettings.EMPTY;
+
+        ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder();
+        addUserSettings(settingsBuilder);
+        addTribeSettings(settingsBuilder);
+        return settingsBuilder.build();
+    }
+
+    private void addUserSettings(ImmutableSettings.Builder settingsBuilder) {
+        String authHeaderSettingName = Headers.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER;
+        if (settings.get(authHeaderSettingName) != null) {
+            return;
         }
-        String user = settings.get("shield.user");
-        if (user == null) {
-            return ImmutableSettings.EMPTY;
+        String userSetting = settings.get("shield.user");
+        if (userSetting == null) {
+            return;
         }
-        int i = user.indexOf(":");
-        if (i < 0 || i == user.length() - 1) {
+        int i = userSetting.indexOf(":");
+        if (i < 0 || i == userSetting.length() - 1) {
             throw new ShieldSettingsException("invalid [shield.user] settings. must be in the form of \"<username>:<password>\"");
         }
-        String username = user.substring(0, i);
-        String password = user.substring(i + 1);
-        return ImmutableSettings.builder()
-                .put(setting, UsernamePasswordToken.basicAuthHeaderValue(username, new SecuredString(password.toCharArray()))).build();
+        String username = userSetting.substring(0, i);
+        String password = userSetting.substring(i + 1);
+        settingsBuilder.put(authHeaderSettingName, UsernamePasswordToken.basicAuthHeaderValue(username, new SecuredString(password.toCharArray())));
+    }
+
+    /*
+     We inject additional settings on each tribe client if the current node is a tribe node, to make sure that every tribe has shield installed and enabled too:
+     - if shield is loaded on the tribe node we make sure it is also loaded on every tribe, by making it mandatory there
+     (this means that the tribe node will fail at startup if shield is not loaded on any tribe due to missing mandatory plugin)
+     - if shield is loaded and enabled on the tribe node, we make sure it is also enabled on every tribe, by forcibly enabling it
+       (that means it's not possible to disable shield on the tribe clients)
+     */
+    private void addTribeSettings(ImmutableSettings.Builder settingsBuilder) {
+        Map<String, Settings> tribesSettings = settings.getGroups("tribe", true);
+        if (tribesSettings.isEmpty()) {
+            return;
+        }
+
+        for (Map.Entry<String, Settings> tribeSettings : tribesSettings.entrySet()) {
+            String tribePrefix = "tribe." + tribeSettings.getKey() + ".";
+
+            //we copy over existing mandatory plugins under additional settings, as they would get overridden otherwise (arrays don't get merged)
+            String[] existingMandatoryPlugins = tribeSettings.getValue().getAsArray("plugin.mandatory", null);
+            if (existingMandatoryPlugins == null) {
+                //shield is mandatory on every tribe if installed and enabled on the tribe node
+                settingsBuilder.putArray(tribePrefix + "plugin.mandatory", NAME);
+            } else {
+                if (!isShieldMandatory(existingMandatoryPlugins)) {
+                    String[] updatedMandatoryPlugins = new String[existingMandatoryPlugins.length + 1];
+                    System.arraycopy(existingMandatoryPlugins, 0, updatedMandatoryPlugins, 0, existingMandatoryPlugins.length);
+                    updatedMandatoryPlugins[updatedMandatoryPlugins.length - 1] = NAME;
+                    //shield is mandatory on every tribe if installed and enabled on the tribe node
+                    settingsBuilder.putArray(tribePrefix + "plugin.mandatory", updatedMandatoryPlugins);
+                }
+            }
+            //shield must be enabled on every tribe if it's enabled on the tribe node
+            settingsBuilder.put(tribePrefix + ENABLED_SETTING_NAME, true);
+        }
+    }
+
+    private static boolean isShieldMandatory(String[] existingMandatoryPlugins) {
+        for (String existingMandatoryPlugin : existingMandatoryPlugins) {
+            if (NAME.equals(existingMandatoryPlugin)) {
+                return true;
+            }
+        }
+        return false;
     }
 
     public static Path configDir(Environment env) {
@@ -103,6 +156,6 @@ public class ShieldPlugin extends AbstractPlugin {
     }
 
     public static boolean shieldEnabled(Settings settings) {
-        return settings.getAsBoolean("shield.enabled", true);
+        return settings.getAsBoolean(ENABLED_SETTING_NAME, true);
     }
 }

src/main/java/org/elasticsearch/shield/license/LicenseService.java

@@ -42,7 +42,12 @@ public class LicenseService extends AbstractLifecycleComponent<LicenseService> {
 
     @Override
     protected void doStart() throws ElasticsearchException {
-        licensesClientService.register(FEATURE_NAME, TRIAL_LICENSE_OPTIONS, new InternalListener());
+        if (settings.getGroups("tribe", true).isEmpty()) {
+            licensesClientService.register(FEATURE_NAME, TRIAL_LICENSE_OPTIONS, new InternalListener());
+        } else {
+            //TODO currently we disable licensing on tribe node. remove this once es core supports merging cluster
+            new InternalListener().onEnabled();
+        }
     }
 
     @Override

src/test/java/org/elasticsearch/shield/ShieldPluginEnabledDisabledTests.java

@@ -54,7 +54,7 @@ public class ShieldPluginEnabledDisabledTests extends ShieldIntegrationTest {
         logger.info("******* shield is " + (enabled ? "enabled" : "disabled"));
         return ImmutableSettings.settingsBuilder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put("shield.enabled", enabled)
+                .put(ShieldPlugin.ENABLED_SETTING_NAME, enabled)
                 .put(InternalNode.HTTP_ENABLED, true)
                 .build();
     }
@@ -63,7 +63,7 @@ public class ShieldPluginEnabledDisabledTests extends ShieldIntegrationTest {
     protected Settings transportClientSettings() {
         return ImmutableSettings.settingsBuilder()
                 .put(super.transportClientSettings())
-                .put("shield.enabled", enabled)
+                .put(ShieldPlugin.ENABLED_SETTING_NAME, enabled)
                 .build();
     }
 

src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield;
+
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.Matchers.arrayContaining;
+
+public class ShieldPluginSettingsTests extends ElasticsearchTestCase {
+
+    private static final String TRIBE_T1_SHIELD_ENABLED = "tribe.t1." + ShieldPlugin.ENABLED_SETTING_NAME;
+    private static final String TRIBE_T2_SHIELD_ENABLED = "tribe.t2." + ShieldPlugin.ENABLED_SETTING_NAME;
+
+    @Test
+    public void testShieldIsMandatoryOnTribes() {
+        Settings settings = ImmutableSettings.builder().put("tribe.t1.cluster.name", "non_existing")
+                .put("tribe.t2.cluster.name", "non_existing").build();
+
+        ShieldPlugin shieldPlugin = new ShieldPlugin(settings);
+
+        Settings additionalSettings = shieldPlugin.additionalSettings();
+
+
+        assertThat(additionalSettings.getAsArray("tribe.t1.plugin.mandatory", null), arrayContaining(ShieldPlugin.NAME));
+        assertThat(additionalSettings.getAsArray("tribe.t2.plugin.mandatory", null), arrayContaining(ShieldPlugin.NAME));
+    }
+
+    @Test
+    public void testAdditionalMandatoryPluginsOnTribes() {
+        Settings settings = ImmutableSettings.builder().put("tribe.t1.cluster.name", "non_existing")
+                .putArray("tribe.t1.plugin.mandatory", "test_plugin").build();
+
+        ShieldPlugin shieldPlugin = new ShieldPlugin(settings);
+
+        //simulate what PluginsService#updatedSettings does to make sure we don't override existing mandatory plugins
+        Settings finalSettings = ImmutableSettings.builder().put(settings).put(shieldPlugin.additionalSettings()).build();
+
+        String[] finalMandatoryPlugins = finalSettings.getAsArray("tribe.t1.plugin.mandatory", null);
+        assertThat(finalMandatoryPlugins, notNullValue());
+        assertThat(finalMandatoryPlugins.length, equalTo(2));
+        assertThat(finalMandatoryPlugins[0], equalTo("test_plugin"));
+        assertThat(finalMandatoryPlugins[1], equalTo(ShieldPlugin.NAME));
+    }
+
+    @Test
+    public void testMandatoryPluginsOnTribesShieldAlreadyMandatory() {
+        Settings settings = ImmutableSettings.builder().put("tribe.t1.cluster.name", "non_existing")
+                .putArray("tribe.t1.plugin.mandatory", "test_plugin", ShieldPlugin.NAME).build();
+
+        ShieldPlugin shieldPlugin = new ShieldPlugin(settings);
+
+        //simulate what PluginsService#updatedSettings does to make sure we don't override existing mandatory plugins
+        Settings finalSettings = ImmutableSettings.builder().put(settings).put(shieldPlugin.additionalSettings()).build();
+
+        String[] finalMandatoryPlugins = finalSettings.getAsArray("tribe.t1.plugin.mandatory", null);
+        assertThat(finalMandatoryPlugins, notNullValue());
+        assertThat(finalMandatoryPlugins.length, equalTo(2));
+        assertThat(finalMandatoryPlugins[0], equalTo("test_plugin"));
+        assertThat(finalMandatoryPlugins[1], equalTo(ShieldPlugin.NAME));
+    }
+
+    @Test
+    public void testShieldAlwaysEnabledOnTribes() {
+        Settings settings = ImmutableSettings.builder().put("tribe.t1.cluster.name", "non_existing")
+                .put(TRIBE_T1_SHIELD_ENABLED, false)
+                .put("tribe.t2.cluster.name", "non_existing").build();
+
+        ShieldPlugin shieldPlugin = new ShieldPlugin(settings);
+
+        Settings additionalSettings = shieldPlugin.additionalSettings();
+
+        assertThat(additionalSettings.getAsBoolean(TRIBE_T1_SHIELD_ENABLED, null), equalTo(true));
+        assertThat(additionalSettings.getAsBoolean(TRIBE_T2_SHIELD_ENABLED, null), equalTo(true));
+
+        //simulate what PluginsService#updatedSettings does to make sure additional settings override existing settings with same name
+        Settings finalSettings = ImmutableSettings.builder().put(settings).put(shieldPlugin.additionalSettings()).build();
+        assertThat(finalSettings.getAsBoolean(TRIBE_T1_SHIELD_ENABLED, null), equalTo(true));
+        assertThat(finalSettings.getAsBoolean(TRIBE_T2_SHIELD_ENABLED, null), equalTo(true));
+    }
+
+    @Test
+    public void testShieldAlwaysEnabledOnTribesShieldAlreadyMandatory() {
+        Settings settings = ImmutableSettings.builder().put("tribe.t1.cluster.name", "non_existing")
+                .put(TRIBE_T1_SHIELD_ENABLED, false)
+                .put("tribe.t2.cluster.name", "non_existing")
+                .putArray("tribe.t1.plugin.mandatory", "test_plugin", ShieldPlugin.NAME).build();
+
+        ShieldPlugin shieldPlugin = new ShieldPlugin(settings);
+
+        Settings additionalSettings = shieldPlugin.additionalSettings();
+
+        assertThat(additionalSettings.getAsBoolean(TRIBE_T1_SHIELD_ENABLED, null), equalTo(true));
+        assertThat(additionalSettings.getAsBoolean(TRIBE_T2_SHIELD_ENABLED, null), equalTo(true));
+
+        //simulate what PluginsService#updatedSettings does to make sure additional settings override existing settings with same name
+        Settings finalSettings = ImmutableSettings.builder().put(settings).put(shieldPlugin.additionalSettings()).build();
+        assertThat(finalSettings.getAsBoolean(TRIBE_T1_SHIELD_ENABLED, null), equalTo(true));
+        assertThat(finalSettings.getAsBoolean(TRIBE_T2_SHIELD_ENABLED, null), equalTo(true));
+        String[] finalMandatoryPlugins = finalSettings.getAsArray("tribe.t1.plugin.mandatory", null);
+        assertThat(finalMandatoryPlugins, notNullValue());
+        assertThat(finalMandatoryPlugins.length, equalTo(2));
+        assertThat(finalMandatoryPlugins[0], equalTo("test_plugin"));
+        assertThat(finalMandatoryPlugins[1], equalTo(ShieldPlugin.NAME));
+    }
+}

src/test/java/org/elasticsearch/shield/VersionCompatibilityTests.java

@@ -46,5 +46,12 @@ public class VersionCompatibilityTests extends ElasticsearchTestCase {
          * see https://github.com/elasticsearch/elasticsearch/pull/9273 {@link org.elasticsearch.action.admin.indices.create.CreateIndexRequestHelper}
          */
         assertThat("Remove CreateIndexRequestHelper class, fixed in es core 1.5", Version.CURRENT.onOrBefore(Version.V_1_4_2), is(true));
+
+        /**
+         * see https://github.com/elasticsearch/elasticsearch/issues/9372 {@link org.elasticsearch.shield.license.LicenseService}
+         * Once es core supports merging cluster level custom metadata (licenses in our case), the tribe node will see some license coming from the tribe and everything will be ok.
+         *
+         */
+        assertThat("Remove workaround in LicenseService class when es core supports merging cluster level custom metadata", Version.CURRENT.onOrBefore(Version.V_1_4_2), is(true));
     }
 }

src/test/java/org/elasticsearch/shield/tribe/TribeShieldLoadedTests.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.tribe;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.license.plugin.LicensePlugin;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.shield.ShieldPlugin;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.containsString;
+
+/**
+ * This class tests different scenarios around tribe node configuration, to make sure that we properly validate
+ * tribes settings depending on how they will load shield or not. Main goal is to make sure that all tribes will run
+ * shield too if the tribe node does.
+ */
+public class TribeShieldLoadedTests extends ElasticsearchTestCase {
+
+    @Test
+    public void testShieldLoadedOnBothTribeNodeAndClients() {
+        //all good if the plugin is loaded on both tribe node and tribe clients, no matter how it gets loaded (manually or from classpath)
+        ImmutableSettings.Builder builder = defaultSettings();
+        if (randomBoolean()) {
+            builder.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false)
+                    .put("plugin.types", ShieldPlugin.class.getName() + "," + LicensePlugin.class.getName());
+        }
+        if (randomBoolean()) {
+            builder.put("tribe.t1.plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false)
+                    .put("tribe.t1.plugin.types", ShieldPlugin.class.getName() + "," + LicensePlugin.class.getName());
+        }
+
+        Node node = null;
+        try {
+            node = NodeBuilder.nodeBuilder().settings(builder.build()).build();
+            node.start();
+        } finally {
+            stop(node);
+        }
+    }
+
+    //this test causes leaking threads to be left behind
+    @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elasticsearch/elasticsearch/issues/9107")
+    @Test
+    public void testShieldLoadedOnTribeNodeOnly() {
+        //startup failure if any of the tribe clients doesn't have shield installed
+        ImmutableSettings.Builder builder = defaultSettings();
+        if (randomBoolean()) {
+            builder.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false)
+                    .put("plugin.types", ShieldPlugin.class.getName() + "," + LicensePlugin.class.getName());
+        }
+
+        builder.put("tribe.t1.plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false);
+
+        try {
+            NodeBuilder.nodeBuilder().settings(builder.build()).build();
+            fail("node initialization should have failed due to missing shield plugin");
+        } catch(Throwable t) {
+            assertThat(t.getMessage(), containsString("Missing mandatory plugins [shield]"));
+        }
+    }
+
+    @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elasticsearch/elasticsearch/issues/9107")
+    @Test
+    public void testShieldMustBeLoadedOnAllTribes() {
+        //startup failure if any of the tribe clients doesn't have shield installed
+        ImmutableSettings.Builder builder = addTribeSettings(defaultSettings(), "t2");
+        if (randomBoolean()) {
+            builder.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false)
+                    .put("plugin.types", ShieldPlugin.class.getName() + "," + LicensePlugin.class.getName());
+        }
+
+        //load shield explicitly on tribe t1
+        builder.put("tribe.t1.plugin.types", ShieldPlugin.class.getName() + "," + LicensePlugin.class.getName())
+                //disable loading from classpath on tribe t2 only
+                .put("tribe.t2.plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false);
+
+        try {
+            NodeBuilder.nodeBuilder().settings(builder.build()).build();
+            fail("node initialization should have failed due to missing shield plugin");
+        } catch(Throwable t) {
+            assertThat(t.getMessage(), containsString("Missing mandatory plugins [shield]"));
+        }
+    }
+
+    private static void stop(Node node) {
+        if (node != null) {
+            try {
+                node.stop();
+            } catch(Throwable t) {
+                //ignore
+            } finally {
+                node.close();
+            }
+        }
+    }
+
+    private static ImmutableSettings.Builder defaultSettings() {
+        return addTribeSettings(ImmutableSettings.builder().put("node.name", "tribe_node"), "t1");
+    }
+
+    private static ImmutableSettings.Builder addTribeSettings(ImmutableSettings.Builder settingsBuilder, String tribe) {
+        String tribePrefix = "tribe." + tribe + ".";
+        return settingsBuilder.put(tribePrefix + "cluster.name", "non_existing_cluster")
+                .put(tribePrefix + "discovery.type", "local")
+                .put(tribePrefix + "discovery.initial_state_timeout", 0);
+    }
+}

src/test/java/org/elasticsearch/shield/tribe/TribeTests.java

@@ -0,0 +1,231 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.tribe;
+
+import com.carrotsearch.randomizedtesting.LifecycleScope;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.client.support.Headers;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.collect.ImmutableMap;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.signature.InternalSignatureService;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.ShieldIntegrationTest;
+import org.elasticsearch.test.ShieldSettingsSource;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.tribe.TribeService;
+import org.junit.*;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.test.InternalTestCluster.clusterName;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.*;
+
+public class TribeTests extends ShieldIntegrationTest {
+
+    //use known suite prefix since their threads are already ignored via ElasticsearchThreadFilter
+    public static final String SECOND_CLUSTER_NODE_PREFIX =  SUITE_CLUSTER_NODE_PREFIX;
+    public static final String TRIBE_CLUSTER_NODE_PREFIX = "tribe_cluster_node_";
+
+    private static InternalTestCluster cluster2;
+    private static ShieldSettingsSource tribeSettingsSource;
+    private InternalTestCluster tribeNodeCluster;
+
+    @Before
+    public void setupSecondClusterAndTribeNode() throws Exception {
+        final Settings globalClusterSettings = internalCluster().getInstance(Settings.class);
+
+        //TODO tribe nodes and all of the tribes need to have either ssl disabled or enabled as a whole
+        //we read the randomized setting from the global cluster and apply it to the other cluster that we are going to start
+        //for simplicity the same certificates are used on all clusters
+        final boolean sslTransportEnabled = globalClusterSettings.getAsBoolean("shield.transport.ssl", null);
+
+        //we need to make sure that all clusters and the tribe node use the same system key, we just point to the same file on all clusters
+        byte[] systemKey = Files.readAllBytes(Paths.get(globalClusterSettings.get(InternalSignatureService.FILE_SETTING)));
+
+        //we run this part in @Before instead of beforeClass because we need to have the current cluster already assigned to global
+        //so that we can retrieve its settings and apply some of them the the second cluster (and tribe node too)
+        if (cluster2 == null) {
+            // create another cluster
+            String cluster2Name = clusterName(Scope.SUITE.name(), Integer.toString(CHILD_JVM_ID), randomLong());
+            //no port conflicts as this test uses the global cluster and a suite cluster that gets manually created
+            ShieldSettingsSource cluster2SettingsSource = new ShieldSettingsSource(2, sslTransportEnabled, systemKey, newTempDir(LifecycleScope.SUITE), Scope.SUITE);
+            cluster2 = new InternalTestCluster(randomLong(), 2, 2, cluster2Name, cluster2SettingsSource, 0, false, false, CHILD_JVM_ID, SECOND_CLUSTER_NODE_PREFIX);
+
+            assert tribeSettingsSource == null;
+            //given the low (2 and 1) number of nodes that the 2 SUITE clusters will have, we are not going to have port conflicts
+            tribeSettingsSource = new ShieldSettingsSource(1, sslTransportEnabled, systemKey, newTempDir(LifecycleScope.SUITE), Scope.SUITE) {
+                @Override
+                public Settings node(int nodeOrdinal) {
+                    Settings shieldSettings = super.node(nodeOrdinal);
+                    //all the settings are needed for the tribe node, some of them will also need to be copied to the tribe clients configuration
+                    ImmutableSettings.Builder builder = ImmutableSettings.builder().put(shieldSettings);
+                    //the tribe node itself won't join any cluster, no need for unicast discovery configuration
+                    builder.remove("discovery.type");
+                    builder.remove("discovery.zen.ping.multicast.enabled");
+                    //remove doesn't remove all the elements of an array, but we know it has only one element
+                    builder.remove("discovery.zen.ping.unicast.hosts.0");
+
+                    //copy the needed settings to the tribe clients configuration
+                    ImmutableMap<String, String> shieldSettingsAsMap = shieldSettings.getAsMap();
+                    for (Map.Entry<String, String> entry : shieldSettingsAsMap.entrySet()) {
+                        if (isSettingNeededForTribeClient(entry.getKey())) {
+                            builder.put("tribe.t1." + entry.getKey(), entry.getValue());
+                            builder.put("tribe.t2." + entry.getKey(), entry.getValue());
+                        }
+                    }
+
+                    return builder.put("tribe.t1.cluster.name", internalCluster().getClusterName())
+                            .putArray("tribe.t1.discovery.zen.ping.unicast.hosts", unicastHosts(internalCluster()))
+                            .put("tribe.t1.shield.transport.ssl", sslTransportEnabled)
+                            .put("tribe.t2.cluster.name", cluster2.getClusterName())
+                            .putArray("tribe.t2.discovery.zen.ping.unicast.hosts", unicastHosts(cluster2))
+                            .put("tribe.t2.shield.transport.ssl", sslTransportEnabled).build();
+                }
+
+                /**
+                 * Returns true if the setting is needed to setup a tribe client and needs to get forwarded to it, false otherwise.
+                 * Only some of the settings need to be forwarded e.g. realm configuration gets filtered out
+                 */
+                private boolean isSettingNeededForTribeClient(String settingKey) {
+                    if (settingKey.equals("transport.host")) {
+                        return true;
+                    }
+                    //discovery settings get forwarded to tribe clients to disable multicast discovery
+                    if (settingKey.equals("discovery.type") || settingKey.equals("discovery.zen.ping.multicast.enabled")) {
+                        return true;
+                    }
+                    //plugins need to be properly loaded on the tribe clients too
+                    if (settingKey.startsWith("plugin")) {
+                        return true;
+                    }
+                    //make sure node.mode is network on the tribe clients too
+                    if (settingKey.equals("node.mode")) {
+                        return true;
+                    }
+                    //forward the shield audit enabled to the tribe clients
+                    if (settingKey.equals("shield.audit.enabled")) {
+                        return true;
+                    }
+                    //forward the system key to the tribe clients, same file will be used
+                    if (settingKey.equals(InternalSignatureService.FILE_SETTING)) {
+                        return true;
+                    }
+                    //forward ssl settings to the tribe clients, same certificates will be used
+                    if (settingKey.startsWith("shield.ssl") || settingKey.equals("shield.transport.ssl") || settingKey.equals("shield.http.ssl")) {
+                        return true;
+                    }
+                    //forward the credentials to the tribe clients
+                    if (settingKey.equals("shield.user") || settingKey.equals(Headers.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER)) {
+                        return true;
+                    }
+                    return false;
+                }
+            };
+        }
+
+        cluster2.beforeTest(getRandom(), 0.5);
+
+        //we need to recreate the tribe node after each test otherwise ensureClusterSizeConsistency barfs
+        String tribeClusterName = clusterName(Scope.SUITE.name(), Integer.toString(CHILD_JVM_ID), randomLong());
+        tribeNodeCluster = new InternalTestCluster(randomLong(), 1, 1, tribeClusterName, tribeSettingsSource, 0, false, false, CHILD_JVM_ID, TRIBE_CLUSTER_NODE_PREFIX);
+        tribeNodeCluster.beforeTest(getRandom(), 0.5);
+        awaitSameNodeCounts();
+    }
+
+    private static String[] unicastHosts(InternalTestCluster testCluster) {
+        Iterable<Transport> transports = testCluster.getInstances(Transport.class);
+        List<String> unicastHosts = new ArrayList<>();
+        for (Transport transport : transports) {
+            TransportAddress transportAddress = transport.boundAddress().boundAddress();
+            assertThat(transportAddress, is(instanceOf(InetSocketTransportAddress.class)));
+            InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) transportAddress;
+            unicastHosts.add("localhost:" + inetSocketTransportAddress.address().getPort());
+        }
+        return unicastHosts.toArray(new String[unicastHosts.size()]);
+    }
+
+    @After
+    public void afterTest() throws IOException {
+        //we need to close the tribe node after each test otherwise ensureClusterSizeConsistency barfs
+        if (tribeNodeCluster != null) {
+            try {
+                tribeNodeCluster.close();
+            } finally {
+                tribeNodeCluster = null;
+            }
+        }
+        //and clean up the second cluster that we manually started
+        if (cluster2 != null) {
+            try {
+                cluster2.wipe();
+            } finally {
+                cluster2.afterTest();
+            }
+        }
+    }
+
+    @AfterClass
+    public static void tearDownSecondCluster() {
+        if (cluster2 != null) {
+            try {
+                cluster2.close();
+            } finally {
+                cluster2 = null;
+                tribeSettingsSource = null;
+            }
+        }
+    }
+
+    @Test
+    public void testIndexRefreshAndSearch() throws Exception {
+        internalCluster().client().admin().indices().prepareCreate("test1").get();
+        cluster2.client().admin().indices().prepareCreate("test2").get();
+        assertThat(tribeNodeCluster.client().admin().cluster().prepareHealth().setWaitForGreenStatus().get().getStatus(), equalTo(ClusterHealthStatus.GREEN));
+
+        tribeNodeCluster.client().prepareIndex("test1", "type1", "1").setSource("field1", "value1").get();
+        tribeNodeCluster.client().prepareIndex("test2", "type1", "1").setSource("field1", "value1").get();
+        assertNoFailures(tribeNodeCluster.client().admin().indices().prepareRefresh().get());
+
+        assertHitCount(tribeNodeCluster.client().prepareSearch().get(), 2l);
+    }
+
+    private void awaitSameNodeCounts() throws Exception {
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                DiscoveryNodes tribeNodes = tribeNodeCluster.client().admin().cluster().prepareState().get().getState().getNodes();
+                assertThat(countDataNodesForTribe("t1", tribeNodes), equalTo(internalCluster().client().admin().cluster().prepareState().get().getState().getNodes().dataNodes().size()));
+                assertThat(countDataNodesForTribe("t2", tribeNodes), equalTo(cluster2.client().admin().cluster().prepareState().get().getState().getNodes().dataNodes().size()));
+            }
+        });
+    }
+
+    private int countDataNodesForTribe(String tribeName, DiscoveryNodes nodes) {
+        int count = 0;
+        for (DiscoveryNode node : nodes) {
+            if (!node.dataNode()) {
+                continue;
+            }
+            if (tribeName.equals(node.getAttributes().get(TribeService.TRIBE_NAME))) {
+                count++;
+            }
+        }
+        return count;
+    }
+}

src/test/java/org/elasticsearch/test/ShieldSettingsSource.java

@@ -37,6 +37,11 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
  */
 public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZen {
 
+    public static final Settings DEFAULT_SETTINGS = ImmutableSettings.builder()
+            .put("node.mode", "network")
+            .put("plugins.load_classpath_plugins", false)
+            .build();
+
     public static final String DEFAULT_USER_NAME = "test_user";
     public static final String DEFAULT_PASSWORD = "changeme";
     public static final String DEFAULT_ROLE = "user";
@@ -73,16 +78,26 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ
      * Creates a new {@link org.elasticsearch.test.SettingsSource} for the shield configuration.
      *
      * @param numOfNodes the number of nodes for proper unicast configuration (can be more than actually available)
+     * @param sslTransportEnabled whether ssl should be enabled on the transport layer or not
      * @param parentFolder the parent folder that will contain all of the configuration files that need to be created
      * @param scope the scope of the test that is requiring an instance of ShieldSettingsSource
      */
     public ShieldSettingsSource(int numOfNodes, boolean sslTransportEnabled, File parentFolder, ElasticsearchIntegrationTest.Scope scope) {
-        super(numOfNodes, ImmutableSettings.builder()
-                .put("node.mode", "network")
-                .put("plugins.load_classpath_plugins", false)
-                .build(),
-                scope);
-        this.systemKey = generateKey();
+        this(numOfNodes, sslTransportEnabled, generateKey(), parentFolder, scope);
+    }
+
+    /**
+     * Creates a new {@link org.elasticsearch.test.SettingsSource} for the shield configuration.
+     *
+     * @param numOfNodes the number of nodes for proper unicast configuration (can be more than actually available)
+     * @param sslTransportEnabled whether ssl should be enabled on the transport layer or not
+     * @param systemKey the system key that all of the nodes will use to sign messages
+     * @param parentFolder the parent folder that will contain all of the configuration files that need to be created
+     * @param scope the scope of the test that is requiring an instance of ShieldSettingsSource
+     */
+    public ShieldSettingsSource(int numOfNodes, boolean sslTransportEnabled, byte[] systemKey, File parentFolder, ElasticsearchIntegrationTest.Scope scope) {
+        super(numOfNodes, DEFAULT_SETTINGS, scope);
+        this.systemKey = systemKey;
         this.parentFolder = parentFolder;
         this.subfolderPrefix = scope.name();
         this.sslTransportEnabled = sslTransportEnabled;