From 746eb2ead460474dfde7e5845e2f218e2b80c539 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 2 Mar 2017 13:51:50 +0000 Subject: [PATCH] [ML] Enable machine learning in x-pack (elastic/x-pack-elasticsearch#683) Original commit: elastic/x-pack-elasticsearch@d718b6f7e826074c67375040cee2be5e8806866e --- .../org/elasticsearch/xpack/XPackSettings.java | 2 +- .../AbstractLicensesIntegrationTestCase.java | 3 +++ .../license/LicensesManagerServiceTests.java | 4 +++- .../license/TribeTransportTestCase.java | 9 +++++++++ .../test/SecuritySettingsSource.java | 3 +++ .../ml/MachineLearningFeatureSetTests.java | 17 ++++++++--------- .../test/MonitoringIntegTestCase.java | 2 ++ .../audit/index/IndexAuditTrailTests.java | 3 +++ .../RemoteIndexAuditTrailStartingTests.java | 3 +++ .../ServerTransportFilterIntegrationTests.java | 2 ++ .../transport/filter/IPFilterTests.java | 2 ++ .../watcher/WatcherPluginDisableTests.java | 3 +++ .../AbstractWatcherIntegrationTestCase.java | 3 +++ 13 files changed, 45 insertions(+), 11 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/XPackSettings.java b/plugin/src/main/java/org/elasticsearch/xpack/XPackSettings.java index acc7712e302..828b3e09e0b 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/XPackSettings.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/XPackSettings.java @@ -42,7 +42,7 @@ public class XPackSettings { public static final Setting GRAPH_ENABLED = enabledSetting(XPackPlugin.GRAPH, true); /** Setting for enabling or disabling machine learning. Defaults to false. */ - public static final Setting MACHINE_LEARNING_ENABLED = enabledSetting(XPackPlugin.MACHINE_LEARNING, false); + public static final Setting MACHINE_LEARNING_ENABLED = enabledSetting(XPackPlugin.MACHINE_LEARNING, true); /** Setting for enabling or disabling auditing. Defaults to false. */ public static final Setting AUDIT_ENABLED = enabledSetting(XPackPlugin.SECURITY + ".audit", false); diff --git a/plugin/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java b/plugin/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java index 883ebc444a5..ace2bbea51c 100644 --- a/plugin/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; +import org.elasticsearch.xpack.ml.MachineLearning; public abstract class AbstractLicensesIntegrationTestCase extends ESIntegTestCase { @@ -29,6 +30,8 @@ public abstract class AbstractLicensesIntegrationTestCase extends ESIntegTestCas .put(XPackSettings.MONITORING_ENABLED.getKey(), false) .put(XPackSettings.WATCHER_ENABLED.getKey(), false) .put(XPackSettings.GRAPH_ENABLED.getKey(), false) + .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false) .build(); } diff --git a/plugin/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java b/plugin/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java index a277bda8dde..0a98f991ab6 100644 --- a/plugin/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java @@ -37,7 +37,9 @@ public class LicensesManagerServiceTests extends XPackSingleNodeTestCase { .put(XPackSettings.SECURITY_ENABLED.getKey(), false) .put(XPackSettings.MONITORING_ENABLED.getKey(), false) .put(XPackSettings.WATCHER_ENABLED.getKey(), false) - .put(XPackSettings.GRAPH_ENABLED.getKey(), false).build(); + .put(XPackSettings.GRAPH_ENABLED.getKey(), false) + .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .build(); } @Override diff --git a/plugin/src/test/java/org/elasticsearch/license/TribeTransportTestCase.java b/plugin/src/test/java/org/elasticsearch/license/TribeTransportTestCase.java index 48fadc77499..62abb7ef7f0 100644 --- a/plugin/src/test/java/org/elasticsearch/license/TribeTransportTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/license/TribeTransportTestCase.java @@ -33,6 +33,7 @@ import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.transport.MockTcpTransportPlugin; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; +import org.elasticsearch.xpack.ml.MachineLearning; import java.util.ArrayList; import java.util.Arrays; @@ -63,6 +64,8 @@ public abstract class TribeTransportTestCase extends ESIntegTestCase { builder.put(XPackSettings.MONITORING_ENABLED.getKey(), enabledFeatures.contains(XPackPlugin.MONITORING)); builder.put(XPackSettings.WATCHER_ENABLED.getKey(), enabledFeatures.contains(XPackPlugin.WATCHER)); builder.put(XPackSettings.GRAPH_ENABLED.getKey(), enabledFeatures.contains(XPackPlugin.GRAPH)); + builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), enabledFeatures.contains(XPackPlugin.MACHINE_LEARNING)); + builder.put(MachineLearning.AUTODETECT_PROCESS.getKey(), false); return builder.build(); } @@ -137,8 +140,14 @@ public abstract class TribeTransportTestCase extends ESIntegTestCase { .put(NetworkModule.HTTP_ENABLED.getKey(), false) .put(internalCluster().getDefaultSettings()) .put(XPackSettings.SECURITY_ENABLED.getKey(), false) // otherwise it conflicts with mock transport + .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false) .put("tribe.t1." + XPackSettings.SECURITY_ENABLED.getKey(), false) .put("tribe.t2." + XPackSettings.SECURITY_ENABLED.getKey(), false) + .put("tribe.t1." + XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put("tribe.t2." + XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put("tribe.t1." + MachineLearning.AUTODETECT_PROCESS.getKey(), false) + .put("tribe.t2." + MachineLearning.AUTODETECT_PROCESS.getKey(), false) .put("node.name", "tribe_node") // make sure we can identify threads from this node .put("transport.type", MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME) .build(); diff --git a/plugin/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/plugin/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index de29722aac5..9596a741fad 100644 --- a/plugin/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/plugin/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -17,6 +17,7 @@ import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration; import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail; import org.elasticsearch.xpack.security.authc.esnative.NativeRealm; @@ -121,6 +122,8 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas //TODO: for now isolate security tests from watcher & monitoring (randomize this later) .put(XPackSettings.WATCHER_ENABLED.getKey(), false) .put(XPackSettings.MONITORING_ENABLED.getKey(), false) + .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false) .put(XPackSettings.AUDIT_ENABLED.getKey(), randomBoolean()) .put(LoggingAuditTrail.HOST_ADDRESS_SETTING.getKey(), randomBoolean()) .put(LoggingAuditTrail.HOST_NAME_SETTING.getKey(), randomBoolean()) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java index 35fecdd617f..dec48c6e134 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java @@ -39,23 +39,22 @@ public class MachineLearningFeatureSetTests extends ESTestCase { } public void testEnabled() throws Exception { - boolean enabled = randomBoolean(); + boolean useDefault = randomBoolean(); + boolean enabled = true; Settings.Builder settings = Settings.builder(); - if (enabled) { + if (useDefault == false) { + enabled = randomBoolean(); settings.put("xpack.ml.enabled", enabled); - } else { - if (randomBoolean()) { - settings.put("xpack.ml.enabled", enabled); - } } + boolean expected = enabled || useDefault; MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(settings.build(), licenseState); - assertThat(featureSet.enabled(), is(enabled)); - assertThat(featureSet.usage().enabled(), is(enabled)); + assertThat(featureSet.enabled(), is(expected)); + assertThat(featureSet.usage().enabled(), is(expected)); BytesStreamOutput out = new BytesStreamOutput(); featureSet.usage().writeTo(out); XPackFeatureSet.Usage serializedUsage = new MachineLearningFeatureSet.Usage(out.bytes().streamInput()); - assertThat(serializedUsage.enabled(), is(enabled)); + assertThat(serializedUsage.enabled(), is(expected)); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java index 70ca87f474e..4ee9a40854f 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java @@ -113,6 +113,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase { .put(XPackSettings.WATCHER_ENABLED.getKey(), watcherEnabled) // Disable native ML autodetect_process as the c++ controller won't be available .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false) + .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) // we do this by default in core, but for monitoring this isn't needed and only adds noise. .put("index.store.mock.check_index_on_close", false); @@ -467,6 +468,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase { writeFile(xpackConf, "system_key", systemKey); builder.put("xpack.security.enabled", true) + .put("xpack.ml.autodetect_process", false) .put("xpack.security.authc.realms.esusers.type", FileRealm.TYPE) .put("xpack.security.authc.realms.esusers.order", 0) .put("xpack.security.authc.sign_user_header", false) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java index 1b1cb3f0a3f..1c51bee93c6 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.Message; import org.elasticsearch.xpack.security.authc.AuthenticationToken; import org.elasticsearch.xpack.security.crypto.CryptoService; @@ -143,6 +144,8 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase { public Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) + // Disable native ML autodetect_process as the c++ controller won't be available + .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false) .put(XPackSettings.SECURITY_ENABLED.getKey(), useSecurity); if (useSecurity == false && builder.get(NetworkModule.TRANSPORT_TYPE_KEY) == null) { builder.put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java index 680e03053c4..811ac8ba2b0 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.junit.After; @@ -94,6 +95,8 @@ public class RemoteIndexAuditTrailStartingTests extends SecurityIntegTestCase { public Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) + // Disable native ML autodetect_process as the c++ controller won't be available + .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false) .put("xpack.security.audit.enabled", true) .put("xpack.security.audit.outputs", randomFrom("index", "index,logfile")) .putArray("xpack.security.audit.index.client.hosts", addresses.toArray(new String[addresses.size()])) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java index 1ca4f7a6670..1d0f6fc667b 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java @@ -100,6 +100,7 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase .put(NetworkModule.HTTP_ENABLED.getKey(), false) .put(Node.NODE_MASTER_SETTING.getKey(), false) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false) + .put("xpack.ml.autodetect_process", false) .build(); try (Node node = new MockNode(nodeSettings, Arrays.asList(XPackPlugin.class, TestZenDiscovery.TestPlugin.class))) { node.start(); @@ -137,6 +138,7 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase .put("path.home", home) .put(Node.NODE_MASTER_SETTING.getKey(), false) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false) + .put("xpack.ml.autodetect_process", false) .build(); try (Node node = new MockNode(nodeSettings, Arrays.asList(XPackPlugin.class, TestZenDiscovery.TestPlugin.class))) { node.start(); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java index cb8a088bdd9..97d5a3a0974 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java @@ -109,6 +109,7 @@ public class IPFilterTests extends ESTestCase { @Network // requires network for name resolution public void testThatHostnamesCanBeProcessed() throws Exception { Settings settings = Settings.builder() + .put("xpack.ml.autodetect_process", false) .put("xpack.security.transport.filter.allow", "127.0.0.1") .put("xpack.security.transport.filter.deny", "*.google.com") .build(); @@ -240,6 +241,7 @@ public class IPFilterTests extends ESTestCase { .put("path.home", createTempDir()) .put("xpack.security.transport.filter.enabled", randomBoolean()) .put("xpack.security.http.filter.enabled", randomBoolean()) + .put("xpack.ml.autodetect_process", false) .build(); try (Node node = new MockNode(settings, Collections.singletonList(XPackPlugin.class))) { assertNotNull(node); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginDisableTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginDisableTests.java index 2e115fdb50f..9de8d166857 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginDisableTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginDisableTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPoolInfo; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; @@ -42,6 +43,8 @@ public class WatcherPluginDisableTests extends ESIntegTestCase { .put(XPackSettings.MONITORING_ENABLED.getKey(), false) .put(NetworkModule.HTTP_ENABLED.getKey(), true) + // Disable native ML autodetect_process as the c++ controller won't be available + .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false) .build(); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index c36d3e928b5..44464875bf7 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.XPackClient; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.common.http.HttpClient; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.notification.email.Authentication; import org.elasticsearch.xpack.notification.email.Email; import org.elasticsearch.xpack.notification.email.EmailService; @@ -145,6 +146,8 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase .put(SecuritySettings.settings(securityEnabled)) .put("xpack.watcher.trigger.schedule.engine", scheduleEngineName) .put("script.inline", "true") + // Disable native ML autodetect_process as the c++ controller won't be available + .put(MachineLearning.AUTODETECT_PROCESS.getKey(), false) .build(); }