diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 296232a0f09..03e96e733a2 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -60,6 +60,7 @@ import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetFiltersAction; import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction; +import org.elasticsearch.xpack.core.ml.action.MlInfoAction; import org.elasticsearch.xpack.core.ml.action.GetJobsAction; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction; @@ -207,6 +208,7 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl // ML GetJobsAction.INSTANCE, GetJobsStatsAction.INSTANCE, + MlInfoAction.INSTANCE, PutJobAction.INSTANCE, UpdateJobAction.INSTANCE, DeleteJobAction.INSTANCE, diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlInfoAction.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlInfoAction.java new file mode 100644 index 00000000000..53a0758bcf8 --- /dev/null +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlInfoAction.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +public class MlInfoAction extends Action { + + public static final MlInfoAction INSTANCE = new MlInfoAction(); + public static final String NAME = "cluster:monitor/xpack/ml/info/get"; + + private MlInfoAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest { + + public Request() { + super(); + } + + public Request(StreamInput in) throws IOException { + super(in); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, MlInfoAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private Map info; + + public Response(Map info) { + this.info = info; + } + + public Response() { + this.info = Collections.emptyMap(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + info = in.readMap(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeMap(info); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.map(info); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(info); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(info, other.info); + } + } +} diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 3a8d89f5bcb..713601c3368 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -55,6 +55,8 @@ import java.util.concurrent.TimeUnit; */ public class DatafeedConfig extends AbstractDiffable implements ToXContentObject { + public static final int DEFAULT_SCROLL_SIZE = 1000; + private static final int SECONDS_IN_MINUTE = 60; private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE; private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE; @@ -427,7 +429,6 @@ public class DatafeedConfig extends AbstractDiffable implements public static class Builder { - private static final int DEFAULT_SCROLL_SIZE = 1000; private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); private static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java index e24f18fd8a6..569d62a02cf 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java @@ -38,10 +38,10 @@ public class AnalysisLimits implements ToXContentObject, Writeable { * is now 1GB and defined here in the Java code. Prior to 6.3, a value of null means that * the old default value should be used. From 6.3 onwards, the value will always be explicit. */ - static final long DEFAULT_MODEL_MEMORY_LIMIT_MB = 1024L; + public static final long DEFAULT_MODEL_MEMORY_LIMIT_MB = 1024L; static final long PRE_6_1_DEFAULT_MODEL_MEMORY_LIMIT_MB = 4096L; - static final long DEFAULT_CATEGORIZATION_EXAMPLES_LIMIT = 4; + public static final long DEFAULT_CATEGORIZATION_EXAMPLES_LIMIT = 4; /** * Serialisation field names diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 3bdb1586990..c898415b5f7 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -94,6 +94,8 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO public static final TimeValue MIN_BACKGROUND_PERSIST_INTERVAL = TimeValue.timeValueHours(1); public static final ByteSizeValue PROCESS_MEMORY_OVERHEAD = new ByteSizeValue(100, ByteSizeUnit.MB); + public static final long DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS = 1; + static { PARSERS.put(MlParserType.METADATA, METADATA_PARSER); PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER); @@ -688,7 +690,7 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO private ModelPlotConfig modelPlotConfig; private Long renormalizationWindowDays; private TimeValue backgroundPersistInterval; - private Long modelSnapshotRetentionDays = 1L; + private Long modelSnapshotRetentionDays = DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS; private Long resultsRetentionDays; private Map customSettings; private String modelSnapshotId; diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlInfoActionResponseTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlInfoActionResponseTests.java new file mode 100644 index 00000000000..7fb216d445d --- /dev/null +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlInfoActionResponseTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.core.ml.action.MlInfoAction.Response; + +import java.util.HashMap; +import java.util.Map; + +public class MlInfoActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int size = randomInt(10); + Map info = new HashMap<>(); + for (int j = 0; j < size; j++) { + info.put(randomAlphaOfLength(20), randomAlphaOfLength(20)); + } + return new Response(info); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } +} diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 22b48b5bdff..5d61816317a 100644 --- a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -40,6 +40,7 @@ import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.monitor.os.OsProbe; import org.elasticsearch.monitor.os.OsStats; +import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; @@ -74,6 +75,7 @@ import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetFiltersAction; import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction; +import org.elasticsearch.xpack.core.ml.action.MlInfoAction; import org.elasticsearch.xpack.core.ml.action.GetJobsAction; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction; @@ -103,7 +105,6 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.notifications.AuditMessage; import org.elasticsearch.xpack.core.ml.notifications.AuditorField; -import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.xpack.core.template.TemplateUtils; import org.elasticsearch.xpack.ml.action.TransportCloseJobAction; import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarAction; @@ -124,6 +125,7 @@ import org.elasticsearch.xpack.ml.action.TransportGetDatafeedsAction; import org.elasticsearch.xpack.ml.action.TransportGetDatafeedsStatsAction; import org.elasticsearch.xpack.ml.action.TransportGetFiltersAction; import org.elasticsearch.xpack.ml.action.TransportGetInfluencersAction; +import org.elasticsearch.xpack.ml.action.TransportMlInfoAction; import org.elasticsearch.xpack.ml.action.TransportGetJobsAction; import org.elasticsearch.xpack.ml.action.TransportGetJobsStatsAction; import org.elasticsearch.xpack.ml.action.TransportGetModelSnapshotsAction; @@ -172,6 +174,7 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerProcessFactory; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction; +import org.elasticsearch.xpack.ml.rest.RestMlInfoAction; import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarAction; import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarEventAction; import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarJobAction; @@ -448,6 +451,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu return Arrays.asList( new RestGetJobsAction(settings, restController), new RestGetJobStatsAction(settings, restController), + new RestMlInfoAction(settings, restController), new RestPutJobAction(settings, restController), new RestPostJobUpdateAction(settings, restController), new RestDeleteJobAction(settings, restController), @@ -498,6 +502,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu return Arrays.asList( new ActionHandler<>(GetJobsAction.INSTANCE, TransportGetJobsAction.class), new ActionHandler<>(GetJobsStatsAction.INSTANCE, TransportGetJobsStatsAction.class), + new ActionHandler<>(MlInfoAction.INSTANCE, TransportMlInfoAction.class), new ActionHandler<>(PutJobAction.INSTANCE, TransportPutJobAction.class), new ActionHandler<>(UpdateJobAction.INSTANCE, TransportUpdateJobAction.class), new ActionHandler<>(DeleteJobAction.INSTANCE, TransportDeleteJobAction.class), diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java new file mode 100644 index 00000000000..f799e204799 --- /dev/null +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ml.MachineLearningField; +import org.elasticsearch.xpack.core.ml.action.MlInfoAction; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; +import org.elasticsearch.xpack.core.ml.job.config.Job; + +import java.util.HashMap; +import java.util.Map; + +public class TransportMlInfoAction extends HandledTransportAction { + + private final ClusterService clusterService; + + @Inject + public TransportMlInfoAction(Settings settings, ThreadPool threadPool, TransportService transportService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + ClusterService clusterService) { + super(settings, MlInfoAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + MlInfoAction.Request::new); + this.clusterService = clusterService; + } + + @Override + protected void doExecute(MlInfoAction.Request request, ActionListener listener) { + Map info = new HashMap<>(); + info.put("defaults", defaults()); + info.put("limits", limits()); + listener.onResponse(new MlInfoAction.Response(info)); + } + + private Map defaults() { + Map defaults = new HashMap<>(); + defaults.put("anomaly_detectors", anomalyDetectorsDefaults()); + defaults.put("datafeeds", datafeedsDefaults()); + return defaults; + } + + private Map anomalyDetectorsDefaults() { + Map defaults = new HashMap<>(); + defaults.put(AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName(), + new ByteSizeValue(AnalysisLimits.DEFAULT_MODEL_MEMORY_LIMIT_MB, ByteSizeUnit.MB)); + defaults.put(AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), AnalysisLimits.DEFAULT_CATEGORIZATION_EXAMPLES_LIMIT); + defaults.put(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), Job.DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS); + return defaults; + } + + private Map datafeedsDefaults() { + Map anomalyDetectorsDefaults = new HashMap<>(); + anomalyDetectorsDefaults.put(DatafeedConfig.SCROLL_SIZE.getPreferredName(), DatafeedConfig.DEFAULT_SCROLL_SIZE); + return anomalyDetectorsDefaults; + } + + private Map limits() { + Map limits = new HashMap<>(); + ByteSizeValue maxModelMemoryLimit = clusterService.getClusterSettings().get(MachineLearningField.MAX_MODEL_MEMORY_LIMIT); + if (maxModelMemoryLimit != null && maxModelMemoryLimit.getBytes() > 0) { + limits.put("max_model_memory_limit", maxModelMemoryLimit); + } + return limits; + } +} diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestMlInfoAction.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestMlInfoAction.java new file mode 100644 index 00000000000..ddb15086e2a --- /dev/null +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestMlInfoAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.ml.action.MlInfoAction; +import org.elasticsearch.xpack.ml.MachineLearning; + +import java.io.IOException; + +public class RestMlInfoAction extends BaseRestHandler { + + public RestMlInfoAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "info", this); + } + + @Override + public String getName() { + return "xpack_ml_info_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + return channel -> client.execute(MlInfoAction.INSTANCE, new MlInfoAction.Request(), new RestToXContentListener<>(channel)); + } +} diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.info.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.info.json new file mode 100644 index 00000000000..770438251f6 --- /dev/null +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.info.json @@ -0,0 +1,11 @@ +{ + "xpack.ml.info": { + "methods": [ "GET" ], + "url": { + "path": "/_xpack/ml/info", + "paths": [ "/_xpack/ml/info" ], + "parts": {}, + "body": null + } + } +} diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/ml_info.yml b/plugin/src/test/resources/rest-api-spec/test/ml/ml_info.yml new file mode 100644 index 00000000000..1ec2b5ea7a6 --- /dev/null +++ b/plugin/src/test/resources/rest-api-spec/test/ml/ml_info.yml @@ -0,0 +1,23 @@ +--- +"Test ml info": + - do: + xpack.ml.info: {} + - match: { defaults.anomaly_detectors.model_memory_limit: "1gb" } + - match: { defaults.anomaly_detectors.categorization_examples_limit: 4 } + - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 } + - match: { defaults.datafeeds.scroll_size: 1000 } + - match: { limits: {} } + + - do: + cluster.put_settings: + body: + persistent: + xpack.ml.max_model_memory_limit: "4gb" + + - do: + xpack.ml.info: {} + - match: { defaults.anomaly_detectors.model_memory_limit: "1gb" } + - match: { defaults.anomaly_detectors.categorization_examples_limit: 4 } + - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 } + - match: { defaults.datafeeds.scroll_size: 1000 } + - match: { limits.max_model_memory_limit: "4gb" } diff --git a/qa/smoke-test-ml-with-security/build.gradle b/qa/smoke-test-ml-with-security/build.gradle index 5c805543687..2f7c49424ee 100644 --- a/qa/smoke-test-ml-with-security/build.gradle +++ b/qa/smoke-test-ml-with-security/build.gradle @@ -74,6 +74,7 @@ integTestRunner { 'ml/job_groups/Test put job with group that matches its id', 'ml/job_groups/Test put job with id that matches an existing group', 'ml/job_groups/Test put job with invalid group', + 'ml/ml_info/Test ml info', 'ml/post_data/Test Flush data with invalid parameters', 'ml/post_data/Test flushing and posting a closed job', 'ml/post_data/Test open and close with non-existent job id',