diff --git a/elasticsearch/build.gradle b/elasticsearch/build.gradle index 1565469d379..29a89f6137e 100644 --- a/elasticsearch/build.gradle +++ b/elasticsearch/build.gradle @@ -1,10 +1,18 @@ import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.test.NodeInfo +import org.gradle.plugins.ide.eclipse.model.SourceFolder +import org.elasticsearch.gradle.precommit.LicenseHeadersTask +import org.elasticsearch.gradle.VersionProperties +import com.bettercloud.vault.Vault +import com.bettercloud.vault.VaultConfig +import com.bettercloud.vault.response.LogicalResponse import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardCopyOption +import java.nio.file.attribute.PosixFilePermission +import java.nio.file.attribute.PosixFilePermissions group 'org.elasticsearch.plugin' @@ -16,6 +24,65 @@ esplugin { } archivesBaseName = 'x-pack' // for api jar +buildscript { + repositories { + mavenCentral() + } + dependencies { + classpath group: 'com.bettercloud', name: 'vault-java-driver', version:"1.1.0" + } +} + +// Vault auth to get keys for access to cpp artifacts + +// first need to get an authentication token with vault +String homePath = System.properties['user.home'] +File githubToken = file("${homePath}/.elastic/github.token") +final String VAULT_URL = 'https://secrets.elastic.co:8200' +final String VAULT_ROLE_ID = "8e90dd88-5a8e-9c12-0da9-5439f293ff97" +final String VAULT_SECRET_ID = System.env.VAULT_SECRET_ID +String authBody = null +URL vaultUrl = null +if (githubToken.exists()) { + Set perms = Files.getPosixFilePermissions(githubToken.toPath()) + if (perms.equals(PosixFilePermissions.fromString("rw-------")) == false) { + throw new GradleException('github.token must have 600 permissions') + } + vaultUrl = new URL(VAULT_URL + '/v1/auth/github/login') + authBody = "{\"token\": \"${githubToken.getText('UTF-8').trim()}\"}" +} else if (VAULT_SECRET_ID != null) { + vaultUrl = new URL(VAULT_URL + '/v1/auth/approle/login') + authBody = "{\"role_id\": \"${VAULT_ROLE_ID}\", \"secret_id\": \"${VAULT_SECRET_ID}\"}" +} else { + throw new GradleException('Missing ~/.elastic/github.token file or VAULT_SECRET_ID environment variable, needed to authenticate with vault for secrets') +} +HttpURLConnection vaultConn = (HttpURLConnection) vaultUrl.openConnection() +vaultConn.setRequestProperty('Content-Type', 'application/json') +vaultConn.setRequestMethod('PUT') +vaultConn.setDoOutput(true) +vaultConn.outputStream.withWriter('UTF-8') { writer -> + writer.write(authBody) +} +vaultConn.connect() +Object authResponse = new groovy.json.JsonSlurper().parseText(vaultConn.content.text) +VaultConfig config = new VaultConfig(VAULT_URL, authResponse.auth.client_token) +Vault vault = new Vault(config) +LogicalResponse secret = vault.logical().read("aws-dev/creds/prelertartifacts") +String mlAwsAccessKey = secret.data.get('access_key') +String mlAwsSecretKey = secret.data.get('secret_key') +// Sleeping to give AWS a chance to propagate the credentials +sleep(3000) + +repositories { + maven { + url "s3://prelert-artifacts/maven" + credentials(AwsCredentials) { + accessKey "${mlAwsAccessKey}" + secretKey "${mlAwsSecretKey}" + } + } +} + // TODO: fix this! https://github.com/elastic/x-plugins/issues/1066 ext.compactProfile = 'full' @@ -26,6 +93,19 @@ licenseHeaders { additionalLicense 'BCRYP', 'BCrypt (BSD-like)', 'Copyright (c) 2006 Damien Miller ' } +configurations { + nativeBundle +} + +if (findProject(':machine-learning-cpp') != null) { + configurations.nativeBundle { + resolutionStrategy.dependencySubstitution { + substitute module("org.elasticsearch.ml:ml-cpp") with project(":machine-learning-cpp") + } + } + bundlePlugin.dependsOn ':machine-learning-cpp:buildUberZip' +} + dependencies { // security deps compile project(path: ':modules:transport-netty4', configuration: 'runtime') @@ -50,6 +130,11 @@ dependencies { compile "org.elasticsearch.client:rest:${version}" compile "org.elasticsearch.client:sniffer:${version}" + // ml deps + compile group: 'net.sf.supercsv', name: 'super-csv', version:'2.4.0' + nativeBundle group: 'org.elasticsearch.ml', name: 'ml-cpp', version:"${project.version}", ext: 'zip' + testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' + // common test deps testCompile 'org.elasticsearch:securemock:1.2' testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}" @@ -92,6 +177,11 @@ forbiddenPatterns { // TODO: standardize packaging config for plugins bundlePlugin { + for (outputFile in configurations.nativeBundle) { + from(zipTree(outputFile)) { + duplicatesStrategy 'exclude' + } + } from(project(':x-pack').projectDir) { include 'LICENSE.txt' } @@ -142,6 +232,7 @@ integTest { // TODO: fix this rest test to not depend on a hardcoded port! systemProperty 'tests.rest.blacklist', 'getting_started/10_monitor_cluster_health/*,bulk/10_basic/*' cluster { + setting 'xpack.ml.enabled', 'true' setting 'xpack.monitoring.collection.interval', '3s' waitCondition = { NodeInfo node, AntBuilder ant -> File tmpFile = new File(node.cwd, 'wait.success') @@ -241,6 +332,7 @@ thirdPartyAudit.excludes = [ ] run { + setting 'xpack.ml.enabled', 'true' setting 'xpack.graph.enabled', 'true' setting 'xpack.security.enabled', 'true' setting 'xpack.monitoring.enabled', 'true' diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/XPackPlugin.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/XPackPlugin.java index 3b8830a96cc..f600faffa14 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/XPackPlugin.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/XPackPlugin.java @@ -68,6 +68,7 @@ import org.elasticsearch.xpack.extensions.XPackExtension; import org.elasticsearch.xpack.extensions.XPackExtensionsService; import org.elasticsearch.xpack.graph.Graph; import org.elasticsearch.xpack.graph.GraphFeatureSet; +import org.elasticsearch.xpack.ml.MlPlugin; import org.elasticsearch.xpack.monitoring.Monitoring; import org.elasticsearch.xpack.monitoring.MonitoringFeatureSet; import org.elasticsearch.xpack.monitoring.MonitoringSettings; @@ -135,6 +136,9 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I /** Name constant for the graph feature. */ public static final String GRAPH = "graph"; + /** Name constant for the machine learning feature. */ + public static final String MACHINE_LEARNING = "ml"; + // inside of YAML settings we still use xpack do not having handle issues with dashes private static final String SETTINGS_NAME = "xpack"; @@ -183,6 +187,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I protected Monitoring monitoring; protected Watcher watcher; protected Graph graph; + protected MlPlugin machineLearning; public XPackPlugin(Settings settings) throws IOException, CertificateException, UnrecoverableKeyException, NoSuchAlgorithmException, KeyStoreException, DestroyFailedException, OperatorCreationException { @@ -197,6 +202,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I this.monitoring = new Monitoring(settings, licenseState); this.watcher = new Watcher(settings); this.graph = new Graph(settings); + this.machineLearning = new MlPlugin(settings, env); // Check if the node is a transport client. if (transportClientMode == false) { this.extensionsService = new XPackExtensionsService(settings, resolveXPackExtensionsFile(env), getExtensions()); @@ -223,6 +229,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I modules.addAll(monitoring.nodeModules()); modules.addAll(watcher.nodeModules()); modules.addAll(graph.createGuiceModules()); + modules.addAll(machineLearning.createGuiceModules()); if (transportClientMode) { modules.add(b -> b.bind(XPackLicenseState.class).toProvider(Providers.of(null))); @@ -271,6 +278,9 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I httpClient, httpTemplateParser, threadPool, clusterService, security.getCryptoService(), xContentRegistry, components)); + components.addAll(machineLearning.createComponents(internalClient, clusterService, threadPool, resourceWatcherService, + scriptService, xContentRegistry)); + // just create the reloader as it will pull all of the loaded ssl configurations and start watching them new SSLConfigurationReloader(settings, env, sslService, resourceWatcherService); return components; @@ -303,6 +313,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I Settings.Builder builder = Settings.builder(); builder.put(security.additionalSettings()); builder.put(watcher.additionalSettings()); + builder.put(machineLearning.additionalSettings()); return builder.build(); } @@ -332,6 +343,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I settings.addAll(Security.getSettings(transportClientMode, extensionsService)); settings.addAll(MonitoringSettings.getSettings()); settings.addAll(watcher.getSettings()); + settings.addAll(machineLearning.getSettings()); settings.addAll(licensing.getSettings()); settings.addAll(XPackSettings.getAllSettings()); @@ -364,6 +376,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I filters.add("xpack.notification.hipchat.account.*.auth_token"); filters.addAll(security.getSettingsFilter()); filters.addAll(MonitoringSettings.getSettingsFilter()); + filters.addAll(machineLearning.getSettingsFilter()); if (transportClientMode == false) { for (XPackExtension extension : extensionsService.getExtensions()) { filters.addAll(extension.getSettingsFilter()); @@ -374,7 +387,10 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I @Override public List> getExecutorBuilders(final Settings settings) { - return watcher.getExecutorBuilders(settings); + List> executorBuilders = new ArrayList>(); + executorBuilders.addAll(watcher.getExecutorBuilders(settings)); + executorBuilders.addAll(machineLearning.getExecutorBuilders(settings)); + return executorBuilders; } @Override @@ -387,6 +403,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I actions.addAll(security.getActions()); actions.addAll(watcher.getActions()); actions.addAll(graph.getActions()); + actions.addAll(machineLearning.getActions()); return actions; } @@ -397,6 +414,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I filters.addAll(monitoring.getActionFilters()); filters.addAll(security.getActionFilters()); filters.addAll(watcher.getActionFilters()); + filters.addAll(machineLearning.getActionFilters()); return filters; } @@ -417,6 +435,8 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I indexNameExpressionResolver, nodesInCluster)); handlers.addAll(graph.getRestHandlers(settings, restController, clusterSettings, indexScopedSettings, settingsFilter, indexNameExpressionResolver, nodesInCluster)); + handlers.addAll(machineLearning.getRestHandlers(settings, restController, clusterSettings, indexScopedSettings, settingsFilter, + indexNameExpressionResolver, nodesInCluster)); return handlers; } @@ -433,6 +453,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, MONITORING, MonitoringFeatureSet.Usage::new)); entries.add(new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, GRAPH, GraphFeatureSet.Usage::new)); entries.addAll(watcher.getNamedWriteables()); + entries.addAll(machineLearning.getNamedWriteables()); entries.addAll(licensing.getNamedWriteables()); return entries; } @@ -441,6 +462,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I public List getNamedXContent() { List entries = new ArrayList<>(); entries.addAll(watcher.getNamedXContent()); + entries.addAll(machineLearning.getNamedXContent()); entries.addAll(licensing.getNamedXContent()); return entries; diff --git a/elasticsearch/src/test/java/org/elasticsearch/transport/KnownActionsTests.java b/elasticsearch/src/test/java/org/elasticsearch/transport/KnownActionsTests.java index ae6421bbfa9..efd1e87f80e 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/transport/KnownActionsTests.java +++ b/elasticsearch/src/test/java/org/elasticsearch/transport/KnownActionsTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.graph.Graph; +import org.elasticsearch.xpack.ml.MlPlugin; import org.elasticsearch.xpack.security.action.SecurityActionModule; import org.junit.BeforeClass; @@ -140,6 +141,9 @@ public class KnownActionsTests extends SecurityIntegTestCase { // also loading all actions from the graph plugin loadActions(collectSubClasses(Action.class, Graph.class), actions); + // also loading all actions from the machine learning plugin + loadActions(collectSubClasses(Action.class, MlPlugin.class), actions); + return unmodifiableSet(actions); } diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java index ecef03d16cc..9b97685839b 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java @@ -15,12 +15,16 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.SecurityIntegTestCase; +import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.ml.MlPlugin; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.datafeed.DatafeedState; @@ -32,6 +36,7 @@ import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.persistent.PersistentActionResponse; import org.elasticsearch.xpack.persistent.RemovePersistentTaskAction; import org.junit.After; @@ -45,15 +50,25 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static org.elasticsearch.xpack.ml.integration.TooManyJobsIT.ensureClusterStateConsistencyWorkAround; +import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @ESIntegTestCase.ClusterScope(numDataNodes = 1) -public class DatafeedJobsIT extends ESIntegTestCase { +public class DatafeedJobsIT extends SecurityIntegTestCase { + + @Override + protected Settings externalClusterClientSettings() { + return Settings.builder().put(super.externalClusterClientSettings()).put("transport.type", "security4") + .put(MlPlugin.ML_ENABLED.getKey(), true) + .put(ThreadContext.PREFIX + ".Authorization", basicAuthHeaderValue("elastic", new SecuredString("changeme".toCharArray()))) + .build(); + } + @Override protected Collection> nodePlugins() { - return Collections.singleton(MlPlugin.class); + return Collections.singleton(XPackPlugin.class); } @Override diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java index 3807389bb8a..3228bc5903a 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java @@ -7,8 +7,12 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.XPackPlugin; +import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.ml.action.util.QueryPage; import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.ml.job.config.Detector; @@ -38,6 +42,7 @@ import org.elasticsearch.xpack.ml.job.results.ModelDebugOutputTests; import org.junit.Before; import java.util.ArrayList; +import java.util.Collection; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -60,6 +65,17 @@ public class AutodetectResultProcessorIT extends ESSingleNodeTestCase { private JobResultsPersister jobResultsPersister; private JobProvider jobProvider; + + @Override + protected Settings nodeSettings() { + return Settings.builder().put(super.nodeSettings()).put(XPackSettings.SECURITY_ENABLED.getKey(), false).build(); + } + + @Override + protected Collection> getPlugins() { + return Collections.singleton(XPackPlugin.class); + } + @Before public void createComponents() { renormalizer = new NoOpRenormalizer(); diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobIT.java index eab0a30c2ab..7ef63ff42ac 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobIT.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobIT.java @@ -9,8 +9,11 @@ import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.junit.After; import org.junit.Before; @@ -21,11 +24,20 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class DatafeedJobIT extends ESRestTestCase { + + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("elastic", new SecuredString("changeme".toCharArray())); + + @Override + protected Settings restClientSettings() { + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); + } + @Before public void setUpData() throws Exception { // Create empty index diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 7d8bf831a07..7c21b76eafe 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -8,9 +8,12 @@ package org.elasticsearch.xpack.ml.integration; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.ml.MlPlugin; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.junit.After; import java.io.BufferedReader; @@ -23,6 +26,7 @@ import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.isEmptyString; @@ -30,6 +34,13 @@ import static org.hamcrest.Matchers.not; public class MlJobIT extends ESRestTestCase { + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("elastic", new SecuredString("changeme".toCharArray())); + + @Override + protected Settings restClientSettings() { + return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); + } + private static final String RESULT_MAPPING = "{ \"mappings\": {\"result\": { \"properties\": { " + "\"result_type\": { \"type\" : \"keyword\" }," + "\"timestamp\": { \"type\" : \"date\" }, " + diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index 862200a1a79..c804029371a 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -11,9 +11,12 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.SecurityIntegTestCase; + import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.ml.MlPlugin; import org.elasticsearch.xpack.ml.action.DatafeedJobsIT; import org.elasticsearch.xpack.ml.action.GetJobsStatsAction; @@ -26,6 +29,7 @@ import org.elasticsearch.xpack.ml.job.config.Detector; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.JobState; import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import org.elasticsearch.xpack.persistent.PersistentActionRequest; import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; @@ -40,13 +44,24 @@ import java.util.Map; import static org.elasticsearch.test.XContentTestUtils.convertToMap; import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; +import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @ESIntegTestCase.ClusterScope(numDataNodes = 1) -public class TooManyJobsIT extends ESIntegTestCase { +public class TooManyJobsIT extends SecurityIntegTestCase { + + @Override + protected Settings externalClusterClientSettings() { + return Settings.builder().put(super.externalClusterClientSettings()).put("transport.type", "security4") + .put(MlPlugin.ML_ENABLED.getKey(), true) + .put(ThreadContext.PREFIX + ".Authorization", + basicAuthHeaderValue("elastic", new SecuredString("changeme".toCharArray()))) + .build(); + } + @Override protected Collection> nodePlugins() { - return Collections.singleton(MlPlugin.class); + return Collections.singleton(XPackPlugin.class); } @Override diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index ef8bb04e8f6..2827931de97 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -8,7 +8,9 @@ package org.elasticsearch.xpack.test.rest; import org.apache.http.HttpStatus; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; +import org.elasticsearch.xpack.ml.integration.MlRestTestStateCleaner; import org.elasticsearch.xpack.security.SecurityTemplateService; +import org.junit.After; import org.junit.Before; import java.io.IOException; @@ -22,6 +24,11 @@ import static java.util.Collections.singletonMap; /** Runs rest tests against external cluster */ public class XPackRestIT extends XPackRestTestCase { + @After + public void clearMlState() throws IOException { + new MlRestTestStateCleaner(client(), this).clearMlMetadata(); + } + public XPackRestIT(ClientYamlTestCandidate testCandidate) { super(testCandidate); } diff --git a/elasticsearch/src/test/resources/org/elasticsearch/transport/actions b/elasticsearch/src/test/resources/org/elasticsearch/transport/actions index 73ffb3b4dfa..c27609bc184 100644 --- a/elasticsearch/src/test/resources/org/elasticsearch/transport/actions +++ b/elasticsearch/src/test/resources/org/elasticsearch/transport/actions @@ -105,3 +105,39 @@ cluster:admin/ingest/pipeline/delete cluster:admin/ingest/pipeline/get cluster:admin/ingest/pipeline/put cluster:admin/ingest/pipeline/simulate +cluster:admin/ml/filters/get +cluster:admin/ml/anomaly_detectors/internal_open +cluster:admin/ml/anomaly_detectors/results/categories/get +cluster:admin/ml/anomaly_detectors/stats/get +cluster:admin/ml/anomaly_detectors/results/buckets/get +cluster:admin/ml/anomaly_detectors/model_snapshots/get +cluster:admin/ml/anomaly_detectors/results/records/get +cluster:admin/ml/anomaly_detectors/results/influencers/get +cluster:admin/ml/datafeeds/put +cluster:admin/ml/anomaly_detectors/model_snapshots/delete +cluster:admin/ml/anomaly_detectors/validate/detector +cluster:admin/ml/anomaly_detectors/validate +cluster:admin/ml/anomaly_detectors/delete +cluster:admin/ml/anomaly_detectors/model_snapshots/revert +cluster:admin/ml/datafeeds/delete +cluster:admin/ml/anomaly_detectors/data/post +cluster:admin/ml/anomaly_detectors/close +cluster:admin/ml/filters/put +cluster:admin/ml/anomaly_detectors/put +cluster:admin/ml/anomaly_detectors/get +cluster:admin/ml/datafeeds/get +cluster:admin/ml/anomaly_detectors/model_snapshots/update +cluster:admin/ml/anomaly_detectors/flush +cluster:admin/ml/filters/delete +cluster:admin/ml/datafeeds/stats/get +cluster:admin/ml/datafeeds/stop +cluster:admin/ml/datafeeds/start +cluster:admin/ml/anomaly_detectors/open +cluster:admin/ml/anomaly_detectors/state/update +cluster:admin/ml/job/update +indices:data/write/delete/mlbyquery +cluster:admin/ml/job/update/process +cluster:admin/persistent/start +cluster:admin/persistent/completion +cluster:admin/persistent/update_status +cluster:admin/persistent/remove