fix shield settings to not rely on iteration order

This removes the use of group setting for `shield.` and introduces some individual settings
and some group settings that should not overlap and cause issues when iteration order
changes.

See elastic/elasticsearch#1520

Original commit: elastic/x-pack-elasticsearch@193e937193

elasticsearch/qa/smoke-test-plugins-ssl/build.gradle

@@ -39,11 +39,8 @@ processTestResources.dependsOn(createKey)
 ext.pluginsCount = 1 // we install xpack explicitly
 project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each { subproj ->
   // need to get a non-decorated project object, so must re-lookup the project by path
-  // FIXME - fix shield settings to not rely on iteration order so this doesn't break!
+  integTest.cluster.plugin(subproj.name, project(subproj.path))
-  if (subproj.name.equals("discovery-ec2") == false) {
+  pluginsCount += 1
-    integTest.cluster.plugin(subproj.name, project(subproj.path))
-    pluginsCount += 1
-  }
 }
 
 integTest {

elasticsearch/qa/smoke-test-plugins/build.gradle

@@ -9,11 +9,8 @@ dependencies {
 ext.pluginsCount = 1 // we install xpack explicitly
 project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each { subproj ->
   // need to get a non-decorated project object, so must re-lookup the project by path
-  // FIXME - fix shield settings to not rely on iteration order so this doesn't break!
+  integTest.cluster.plugin(subproj.name, project(subproj.path))
-  if (subproj.name.equals("discovery-ec2") == false) {
+  pluginsCount += 1
-    integTest.cluster.plugin(subproj.name, project(subproj.path))
-    pluginsCount += 1
-  }
 }
 
 integTest {

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceClusterTests.java

@@ -58,7 +58,6 @@ public class LicensesServiceClusterTests extends AbstractLicensesIntegrationTest
     private Settings.Builder nodeSettingsBuilder(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put("plugins.load_classpath_plugins", false)
                 .put("node.data", true)
                 // this setting is only used in tests
                 .put("_trial_license_duration_in_seconds", 9)

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MonitoringF.java

@@ -32,7 +32,6 @@ public class MonitoringF {
         Settings.Builder settings = Settings.builder();
         settings.put("script.inline", "true");
         settings.put("security.manager.enabled", "false");
-        settings.put("plugins.load_classpath_plugins", "false");
         settings.put("cluster.name", MonitoringF.class.getSimpleName());
         settings.put("xpack.monitoring.agent.interval", "5s");
         if (!CollectionUtils.isEmpty(args)) {

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java

@@ -419,7 +419,6 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
                         .put("shield.authc.realms.esusers.files.users", writeFile(folder, "users", USERS))
                         .put("shield.authc.realms.esusers.files.users_roles", writeFile(folder, "users_roles", USER_ROLES))
                         .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", ROLES))
-                        .put("shield.transport.n2n.ip_filter.file", writeFile(folder, "ip_filter.yml", IP_FILTER))
                         .put("shield.system_key.file", writeFile(folder, "system_key.yml", systemKey))
                         .put("shield.authc.sign_user_header", false)
                         .put("shield.audit.enabled", auditLogsEnabled)

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Shield.java

@@ -77,6 +77,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  *
@@ -173,11 +174,27 @@ public class Shield {
         settingsModule.registerSetting(IPFilter.HTTP_FILTER_DENY_SETTING);
         settingsModule.registerSetting(IPFilter.TRANSPORT_FILTER_ALLOW_SETTING);
         settingsModule.registerSetting(IPFilter.TRANSPORT_FILTER_DENY_SETTING);
-        settingsModule.registerSetting(Setting.boolSetting("plugins.load_classpath_plugins", true, false, Setting.Scope.CLUSTER));
+        XPackPlugin.registerFeatureEnabledSettings(settingsModule, NAME, true);
-        // TODO add real settings for this wildcard here
+        XPackPlugin.registerFeatureEnabledSettings(settingsModule, DLS_FLS_FEATURE, true);
-        settingsModule.registerSetting(Setting.groupSetting("shield.", false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.groupSetting("shield.audit.", false, Setting.Scope.CLUSTER));
-        // TODO please let's just drop the old settings before releasing
+        settingsModule.registerSetting(Setting.listSetting("shield.hide_settings", Collections.emptyList(), Function.identity(), false,
-        settingsModule.registerSetting(Setting.groupSetting("xpack.shield.", false, Setting.Scope.CLUSTER));
+                Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.groupSetting("shield.ssl.", false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.groupSetting("shield.authc.", false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.simpleString("shield.authz.store.files.roles", false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.simpleString("shield.system_key.file", false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.boolSetting(ShieldNettyHttpServerTransport.HTTP_SSL_SETTING,
+                ShieldNettyHttpServerTransport.HTTP_SSL_DEFAULT, false, Setting.Scope.CLUSTER));
+        // FIXME need to register a real setting with the defaults here
+        settingsModule.registerSetting(Setting.simpleString(ShieldNettyHttpServerTransport.HTTP_CLIENT_AUTH_SETTING,
+                false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.boolSetting(ShieldNettyTransport.TRANSPORT_SSL_SETTING,
+                ShieldNettyTransport.TRANSPORT_SSL_DEFAULT, false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.simpleString(ShieldNettyTransport.TRANSPORT_CLIENT_AUTH_SETTING, false,
+                Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.simpleString("shield.user", false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.simpleString("shield.encryption_key.algorithm", false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.simpleString("shield.encryption.algorithm", false, Setting.Scope.CLUSTER));
 
         String[] asArray = settings.getAsArray("shield.hide_settings");
         for (String pattern : asArray) {

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java

@@ -47,7 +47,6 @@ public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZ
 
     public static final Settings DEFAULT_SETTINGS = settingsBuilder()
             .put("node.mode", "network")
-            .put("plugins.load_classpath_plugins", false)
             .build();
 
     public static final String DEFAULT_USER_NAME = "test_user";

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java

@@ -171,7 +171,7 @@ public class XPackPlugin extends Plugin {
      */
     public static boolean featureEnabled(Settings settings, String featureName, boolean defaultValue) {
         return settings.getAsBoolean(featureEnabledSetting(featureName),
-                settings.getAsBoolean(featureName + ".enabled", defaultValue)); // for bwc
+                settings.getAsBoolean(legacyFeatureEnabledSetting(featureName), defaultValue)); // for bwc
     }
 
     public static String featureEnabledSetting(String featureName) {
@@ -181,4 +181,23 @@ public class XPackPlugin extends Plugin {
     public static String featureSettingPrefix(String featureName) {
         return NAME + "." + featureName;
     }
+
+    public static String legacyFeatureEnabledSetting(String featureName) {
+        return featureName + ".enabled";
+    }
+
+    /**
+     * A consistent way to register the settings used to enable disable features, supporting the following format:
+     *
+     *          {@code "xpack.<feature>.enabled": true | false}
+     *
+     *  Also supports the following setting as a fallback (for BWC with 1.x/2.x):
+     *
+     *          {@code "<feature>.enabled": true | false}
+     */
+    public static void registerFeatureEnabledSettings(SettingsModule settingsModule, String featureName, boolean defaultValue) {
+        settingsModule.registerSetting(Setting.boolSetting(featureEnabledSetting(featureName), defaultValue, false, Setting.Scope.CLUSTER));
+        settingsModule.registerSetting(Setting.boolSetting(legacyFeatureEnabledSetting(featureName),
+                defaultValue, false, Setting.Scope.CLUSTER));
+    }
 }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java

@@ -715,7 +715,6 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase
                         .put("shield.authc.realms.esusers.files.users", writeFile(folder, "users", USERS))
                         .put("shield.authc.realms.esusers.files.users_roles", writeFile(folder, "users_roles", USER_ROLES))
                         .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", ROLES))
-                        .put("shield.transport.n2n.ip_filter.file", writeFile(folder, "ip_filter.yml", IP_FILTER))
                         .put("shield.system_key.file", writeFile(folder, "system_key.yml", systemKey))
                         .put("shield.authc.sign_user_header", false)
                         .put("shield.audit.enabled", auditLogsEnabled)