[ML] Enable machine learning in x-pack (elastic/x-pack-elasticsearch#683)

Original commit: elastic/x-pack-elasticsearch@d718b6f7e8

plugin/src/main/java/org/elasticsearch/xpack/XPackSettings.java

@@ -42,7 +42,7 @@ public class XPackSettings {
     public static final Setting<Boolean> GRAPH_ENABLED = enabledSetting(XPackPlugin.GRAPH, true);
 
     /** Setting for enabling or disabling machine learning. Defaults to false. */
-    public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = enabledSetting(XPackPlugin.MACHINE_LEARNING, false);
+    public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = enabledSetting(XPackPlugin.MACHINE_LEARNING, true);
 
     /** Setting for enabling or disabling auditing. Defaults to false. */
     public static final Setting<Boolean> AUDIT_ENABLED = enabledSetting(XPackPlugin.SECURITY + ".audit", false);

plugin/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java

@@ -19,6 +19,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.ml.MachineLearning;
 
 public abstract class AbstractLicensesIntegrationTestCase extends ESIntegTestCase {
 
@@ -29,6 +30,8 @@ public abstract class AbstractLicensesIntegrationTestCase extends ESIntegTestCas
                 .put(XPackSettings.MONITORING_ENABLED.getKey(), false)
                 .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
                 .put(XPackSettings.GRAPH_ENABLED.getKey(), false)
+                .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false)
+                .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
                 .build();
     }
 

plugin/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java

@@ -37,7 +37,9 @@ public class LicensesManagerServiceTests extends XPackSingleNodeTestCase {
             .put(XPackSettings.SECURITY_ENABLED.getKey(), false)
             .put(XPackSettings.MONITORING_ENABLED.getKey(), false)
             .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
-            .put(XPackSettings.GRAPH_ENABLED.getKey(), false).build();
+            .put(XPackSettings.GRAPH_ENABLED.getKey(), false)
+            .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false)
+            .build();
     }
 
     @Override

plugin/src/test/java/org/elasticsearch/license/TribeTransportTestCase.java

@@ -33,6 +33,7 @@ import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.transport.MockTcpTransportPlugin;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -63,6 +64,8 @@ public abstract class TribeTransportTestCase extends ESIntegTestCase {
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), enabledFeatures.contains(XPackPlugin.MONITORING));
         builder.put(XPackSettings.WATCHER_ENABLED.getKey(), enabledFeatures.contains(XPackPlugin.WATCHER));
         builder.put(XPackSettings.GRAPH_ENABLED.getKey(), enabledFeatures.contains(XPackPlugin.GRAPH));
+        builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), enabledFeatures.contains(XPackPlugin.MACHINE_LEARNING));
+        builder.put(MachineLearning.AUTODETECT_PROCESS.getKey(), false);
         return builder.build();
     }
 
@@ -137,8 +140,14 @@ public abstract class TribeTransportTestCase extends ESIntegTestCase {
                 .put(NetworkModule.HTTP_ENABLED.getKey(), false)
                 .put(internalCluster().getDefaultSettings())
                 .put(XPackSettings.SECURITY_ENABLED.getKey(), false) // otherwise it conflicts with mock transport
+                .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false)
+                .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
                 .put("tribe.t1." + XPackSettings.SECURITY_ENABLED.getKey(), false)
                 .put("tribe.t2." + XPackSettings.SECURITY_ENABLED.getKey(), false)
+                .put("tribe.t1." + XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false)
+                .put("tribe.t2." + XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false)
+                .put("tribe.t1." + MachineLearning.AUTODETECT_PROCESS.getKey(), false)
+                .put("tribe.t2." + MachineLearning.AUTODETECT_PROCESS.getKey(), false)
                 .put("node.name", "tribe_node") // make sure we can identify threads from this node
                 .put("transport.type", MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)
                 .build();

plugin/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java

@@ -17,6 +17,7 @@ import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
 import org.elasticsearch.transport.Netty4Plugin;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail;
 import org.elasticsearch.xpack.security.authc.esnative.NativeRealm;
@@ -121,6 +122,8 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
                 //TODO: for now isolate security tests from watcher & monitoring (randomize this later)
                 .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
                 .put(XPackSettings.MONITORING_ENABLED.getKey(), false)
+                .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false)
+                .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
                 .put(XPackSettings.AUDIT_ENABLED.getKey(), randomBoolean())
                 .put(LoggingAuditTrail.HOST_ADDRESS_SETTING.getKey(), randomBoolean())
                 .put(LoggingAuditTrail.HOST_NAME_SETTING.getKey(), randomBoolean())

plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java

@@ -39,23 +39,22 @@ public class MachineLearningFeatureSetTests extends ESTestCase {
     }
 
     public void testEnabled() throws Exception {
-        boolean enabled = randomBoolean();
+        boolean useDefault = randomBoolean();
+        boolean enabled = true;
         Settings.Builder settings = Settings.builder();
-        if (enabled) {
+        if (useDefault == false) {
-            settings.put("xpack.ml.enabled", enabled);
+            enabled = randomBoolean();
-        } else {
-            if (randomBoolean()) {
             settings.put("xpack.ml.enabled", enabled);
         }
-        }
+        boolean expected = enabled || useDefault;
         MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(settings.build(), licenseState);
-        assertThat(featureSet.enabled(), is(enabled));
+        assertThat(featureSet.enabled(), is(expected));
-        assertThat(featureSet.usage().enabled(), is(enabled));
+        assertThat(featureSet.usage().enabled(), is(expected));
 
         BytesStreamOutput out = new BytesStreamOutput();
         featureSet.usage().writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new MachineLearningFeatureSet.Usage(out.bytes().streamInput());
-        assertThat(serializedUsage.enabled(), is(enabled));
+        assertThat(serializedUsage.enabled(), is(expected));
     }
 
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java

@@ -113,6 +113,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
                 .put(XPackSettings.WATCHER_ENABLED.getKey(), watcherEnabled)
                 // Disable native ML autodetect_process as the c++ controller won't be available
                 .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
+                .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false)
                 // we do this by default in core, but for monitoring this isn't needed and only adds noise.
                 .put("index.store.mock.check_index_on_close", false);
 
@@ -467,6 +468,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
                 writeFile(xpackConf, "system_key", systemKey);
 
                 builder.put("xpack.security.enabled", true)
+                        .put("xpack.ml.autodetect_process", false)
                         .put("xpack.security.authc.realms.esusers.type", FileRealm.TYPE)
                         .put("xpack.security.authc.realms.esusers.order", 0)
                         .put("xpack.security.authc.sign_user_header", false)

plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java

@@ -39,6 +39,7 @@ import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.Message;
 import org.elasticsearch.xpack.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.security.crypto.CryptoService;
@@ -143,6 +144,8 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase {
             public Settings nodeSettings(int nodeOrdinal) {
                 Settings.Builder builder = Settings.builder()
                         .put(super.nodeSettings(nodeOrdinal))
+                        // Disable native ML autodetect_process as the c++ controller won't be available
+                        .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
                         .put(XPackSettings.SECURITY_ENABLED.getKey(), useSecurity);
                 if (useSecurity == false && builder.get(NetworkModule.TRANSPORT_TYPE_KEY) == null) {
                     builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME);

plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java

@@ -16,6 +16,7 @@ import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.junit.After;
@@ -94,6 +95,8 @@ public class RemoteIndexAuditTrailStartingTests extends SecurityIntegTestCase {
             public Settings nodeSettings(int nodeOrdinal) {
                 Settings.Builder builder = Settings.builder()
                         .put(super.nodeSettings(nodeOrdinal))
+                        // Disable native ML autodetect_process as the c++ controller won't be available
+                        .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
                         .put("xpack.security.audit.enabled", true)
                         .put("xpack.security.audit.outputs", randomFrom("index", "index,logfile"))
                         .putArray("xpack.security.audit.index.client.hosts", addresses.toArray(new String[addresses.size()]))

plugin/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java

@@ -100,6 +100,7 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
                 .put(NetworkModule.HTTP_ENABLED.getKey(), false)
                 .put(Node.NODE_MASTER_SETTING.getKey(), false)
                 .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false)
+                .put("xpack.ml.autodetect_process", false)
                 .build();
         try (Node node = new MockNode(nodeSettings, Arrays.asList(XPackPlugin.class, TestZenDiscovery.TestPlugin.class))) {
             node.start();
@@ -137,6 +138,7 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
                 .put("path.home", home)
                 .put(Node.NODE_MASTER_SETTING.getKey(), false)
                 .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false)
+                .put("xpack.ml.autodetect_process", false)
                 .build();
         try (Node node = new MockNode(nodeSettings, Arrays.asList(XPackPlugin.class, TestZenDiscovery.TestPlugin.class))) {
             node.start();

plugin/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java

@@ -109,6 +109,7 @@ public class IPFilterTests extends ESTestCase {
     @Network // requires network for name resolution
     public void testThatHostnamesCanBeProcessed() throws Exception {
         Settings settings = Settings.builder()
+                .put("xpack.ml.autodetect_process", false)
                 .put("xpack.security.transport.filter.allow", "127.0.0.1")
                 .put("xpack.security.transport.filter.deny", "*.google.com")
                 .build();
@@ -240,6 +241,7 @@ public class IPFilterTests extends ESTestCase {
                 .put("path.home", createTempDir())
                 .put("xpack.security.transport.filter.enabled", randomBoolean())
                 .put("xpack.security.http.filter.enabled", randomBoolean())
+                .put("xpack.ml.autodetect_process", false)
                 .build();
         try (Node node = new MockNode(settings, Collections.singletonList(XPackPlugin.class))) {
             assertNotNull(node);

plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginDisableTests.java

@@ -23,6 +23,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolInfo;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor;
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
@@ -42,6 +43,8 @@ public class WatcherPluginDisableTests extends ESIntegTestCase {
                 .put(XPackSettings.MONITORING_ENABLED.getKey(), false)
 
                 .put(NetworkModule.HTTP_ENABLED.getKey(), true)
+                // Disable native ML autodetect_process as the c++ controller won't be available
+                .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
                 .build();
     }
 

plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java

@@ -50,6 +50,7 @@ import org.elasticsearch.xpack.XPackClient;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.common.http.HttpClient;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.notification.email.Authentication;
 import org.elasticsearch.xpack.notification.email.Email;
 import org.elasticsearch.xpack.notification.email.EmailService;
@@ -145,6 +146,8 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase
                 .put(SecuritySettings.settings(securityEnabled))
                 .put("xpack.watcher.trigger.schedule.engine", scheduleEngineName)
                 .put("script.inline", "true")
+                // Disable native ML autodetect_process as the c++ controller won't be available
+                .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false)
                 .build();
     }