Merge pull request #16893 from rjernst/more_tribe_node_settings

Passthrough environment and network settings to tribe client nodes

core/src/main/java/org/elasticsearch/tribe/TribeService.java

@@ -41,6 +41,8 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -52,7 +54,10 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.transport.TransportSettings;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -154,6 +159,15 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
     public static final Set<String> TRIBE_SETTING_KEYS = Sets.newHashSet(TRIBE_NAME_SETTING.getKey(), ON_CONFLICT_SETTING.getKey(),
             BLOCKS_METADATA_INDICES_SETTING.getKey(), BLOCKS_METADATA_SETTING.getKey(), BLOCKS_READ_INDICES_SETTING.getKey(), BLOCKS_WRITE_INDICES_SETTING.getKey(), BLOCKS_WRITE_SETTING.getKey());
 
+    // these settings should be passed through to each tribe client, if they are not set explicitly
+    private static final List<Setting<?>> PASS_THROUGH_SETTINGS = Arrays.asList(
+        NetworkService.GLOBAL_NETWORK_HOST_SETTING,
+        NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING,
+        NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING,
+        TransportSettings.HOST,
+        TransportSettings.BIND_HOST,
+        TransportSettings.PUBLISH_HOST
+    );
     private final String onConflict;
     private final Set<String> droppedIndices = ConcurrentCollections.newConcurrentSet();
 
@@ -167,18 +181,8 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
         nodesSettings.remove("blocks"); // remove prefix settings that don't indicate a client
         nodesSettings.remove("on_conflict"); // remove prefix settings that don't indicate a client
         for (Map.Entry<String, Settings> entry : nodesSettings.entrySet()) {
-            Settings.Builder sb = Settings.builder().put(entry.getValue());
+            Settings clientSettings = buildClientSettings(entry.getKey(), settings, entry.getValue());
-            sb.put("node.name", settings.get("node.name") + "/" + entry.getKey());
+            nodes.add(new TribeClientNode(clientSettings));
-            sb.put(Environment.PATH_HOME_SETTING.getKey(), Environment.PATH_HOME_SETTING.get(settings)); // pass through ES home dir
-            if (Environment.PATH_CONF_SETTING.exists(settings)) {
-                sb.put(Environment.PATH_CONF_SETTING.getKey(), Environment.PATH_CONF_SETTING.get(settings));
-            }
-            sb.put(TRIBE_NAME_SETTING.getKey(), entry.getKey());
-            if (sb.get("http.enabled") == null) {
-                sb.put("http.enabled", false);
-            }
-            sb.put(Node.NODE_CLIENT_SETTING.getKey(), true);
-            nodes.add(new TribeClientNode(sb.build()));
         }
 
         this.blockIndicesMetadata = BLOCKS_METADATA_INDICES_SETTING.get(settings).toArray(Strings.EMPTY_ARRAY);
@@ -197,6 +201,46 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
         this.onConflict = ON_CONFLICT_SETTING.get(settings);
     }
 
+    // pkg private for testing
+    /**
+     * Builds node settings for a tribe client node from the tribe node's global settings,
+     * combined with tribe specific settings.
+     */
+    static Settings buildClientSettings(String tribeName, Settings globalSettings, Settings tribeSettings) {
+        for (String tribeKey : tribeSettings.getAsMap().keySet()) {
+            if (tribeKey.startsWith("path.")) {
+                throw new IllegalArgumentException("Setting [" + tribeKey + "] not allowed in tribe client [" + tribeName + "]");
+            }
+        }
+        Settings.Builder sb = Settings.builder().put(tribeSettings);
+        sb.put("node.name", globalSettings.get("node.name") + "/" + tribeName);
+        sb.put(Environment.PATH_HOME_SETTING.getKey(), Environment.PATH_HOME_SETTING.get(globalSettings)); // pass through ES home dir
+        if (Environment.PATH_CONF_SETTING.exists(globalSettings)) {
+            sb.put(Environment.PATH_CONF_SETTING.getKey(), Environment.PATH_CONF_SETTING.get(globalSettings));
+        }
+        if (Environment.PATH_PLUGINS_SETTING.exists(globalSettings)) {
+            sb.put(Environment.PATH_PLUGINS_SETTING.getKey(), Environment.PATH_PLUGINS_SETTING.get(globalSettings));
+        }
+        if (Environment.PATH_LOGS_SETTING.exists(globalSettings)) {
+            sb.put(Environment.PATH_LOGS_SETTING.getKey(), Environment.PATH_LOGS_SETTING.get(globalSettings));
+        }
+        if (Environment.PATH_SCRIPTS_SETTING.exists(globalSettings)) {
+            sb.put(Environment.PATH_SCRIPTS_SETTING.getKey(), Environment.PATH_SCRIPTS_SETTING.get(globalSettings));
+        }
+        for (Setting<?> passthrough : PASS_THROUGH_SETTINGS) {
+            if (passthrough.exists(tribeSettings) == false && passthrough.exists(globalSettings)) {
+                sb.put(passthrough.getKey(), globalSettings.get(passthrough.getKey()));
+            }
+        }
+        sb.put(TRIBE_NAME_SETTING.getKey(), tribeName);
+        if (sb.get(NetworkModule.HTTP_ENABLED.getKey()) == null) {
+            sb.put(NetworkModule.HTTP_ENABLED.getKey(), false);
+        }
+        sb.put(Node.NODE_CLIENT_SETTING.getKey(), true);
+        return sb.build();
+    }
+
+
     @Override
     protected void doStart() {
         if (nodes.isEmpty() == false) {

core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tribe;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+public class TribeServiceTests extends ESTestCase {
+    public void testMinimalSettings() {
+        Settings globalSettings = Settings.builder()
+            .put("node.name", "nodename")
+            .put("path.home", "some/path").build();
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        assertEquals("some/path", clientSettings.get("path.home"));
+        assertEquals("nodename/tribe1", clientSettings.get("node.name"));
+        assertEquals("tribe1", clientSettings.get("tribe.name"));
+        assertEquals("false", clientSettings.get("http.enabled"));
+        assertEquals("true", clientSettings.get("node.client"));
+        assertEquals(5, clientSettings.getAsMap().size());
+    }
+
+    public void testEnvironmentSettings() {
+        Settings globalSettings = Settings.builder()
+            .put("node.name", "nodename")
+            .put("path.home", "some/path")
+            .put("path.conf", "conf/path")
+            .put("path.plugins", "plugins/path")
+            .put("path.scripts", "scripts/path")
+            .put("path.logs", "logs/path").build();
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        assertEquals("some/path", clientSettings.get("path.home"));
+        assertEquals("conf/path", clientSettings.get("path.conf"));
+        assertEquals("plugins/path", clientSettings.get("path.plugins"));
+        assertEquals("scripts/path", clientSettings.get("path.scripts"));
+        assertEquals("logs/path", clientSettings.get("path.logs"));
+
+        Settings tribeSettings = Settings.builder()
+            .put("path.home", "alternate/path").build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            TribeService.buildClientSettings("tribe1", globalSettings, tribeSettings);
+        });
+        assertTrue(e.getMessage(), e.getMessage().contains("Setting [path.home] not allowed in tribe client"));
+    }
+
+    public void testPassthroughSettings() {
+        Settings globalSettings = Settings.builder()
+            .put("node.name", "nodename")
+            .put("path.home", "some/path")
+            .put("network.host", "0.0.0.0")
+            .put("network.bind_host", "1.1.1.1")
+            .put("network.publish_host", "2.2.2.2")
+            .put("transport.host", "3.3.3.3")
+            .put("transport.bind_host", "4.4.4.4")
+            .put("transport.publish_host", "5.5.5.5").build();
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        assertEquals("0.0.0.0", clientSettings.get("network.host"));
+        assertEquals("1.1.1.1", clientSettings.get("network.bind_host"));
+        assertEquals("2.2.2.2", clientSettings.get("network.publish_host"));
+        assertEquals("3.3.3.3", clientSettings.get("transport.host"));
+        assertEquals("4.4.4.4", clientSettings.get("transport.bind_host"));
+        assertEquals("5.5.5.5", clientSettings.get("transport.publish_host"));
+
+        // per tribe client overrides still work
+        Settings tribeSettings = Settings.builder()
+            .put("network.host", "3.3.3.3")
+            .put("network.bind_host", "4.4.4.4")
+            .put("network.publish_host", "5.5.5.5")
+            .put("transport.host", "6.6.6.6")
+            .put("transport.bind_host", "7.7.7.7")
+            .put("transport.publish_host", "8.8.8.8").build();
+        clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, tribeSettings);
+        assertEquals("3.3.3.3", clientSettings.get("network.host"));
+        assertEquals("4.4.4.4", clientSettings.get("network.bind_host"));
+        assertEquals("5.5.5.5", clientSettings.get("network.publish_host"));
+        assertEquals("6.6.6.6", clientSettings.get("transport.host"));
+        assertEquals("7.7.7.7", clientSettings.get("transport.bind_host"));
+        assertEquals("8.8.8.8", clientSettings.get("transport.publish_host"));
+    }
+}