[ML] Add ML info endpoint providing defaults and limits (elastic/x-pack-elasticsearch#4154)

This commit adds an info API to ML. The API returns information
about default values and limits so that implementors can be
aware of such values and deal with them accordingly.

relates elastic/x-pack-elasticsearch#4135

Original commit: elastic/x-pack-elasticsearch@a969221032

plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -60,6 +60,7 @@ import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetFiltersAction;
 import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction;
+import org.elasticsearch.xpack.core.ml.action.MlInfoAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction;
@@ -207,6 +208,7 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
                 // ML
                 GetJobsAction.INSTANCE,
                 GetJobsStatsAction.INSTANCE,
+                MlInfoAction.INSTANCE,
                 PutJobAction.INSTANCE,
                 UpdateJobAction.INSTANCE,
                 DeleteJobAction.INSTANCE,

plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlInfoAction.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+
+public class MlInfoAction extends Action<MlInfoAction.Request, MlInfoAction.Response, MlInfoAction.RequestBuilder> {
+
+    public static final MlInfoAction INSTANCE = new MlInfoAction();
+    public static final String NAME = "cluster:monitor/xpack/ml/info/get";
+
+    private MlInfoAction() {
+        super(NAME);
+    }
+
+    @Override
+    public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new RequestBuilder(client, this);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Request extends ActionRequest {
+
+        public Request() {
+            super();
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+    }
+
+    public static class RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder> {
+
+        public RequestBuilder(ElasticsearchClient client, MlInfoAction action) {
+            super(client, action, new Request());
+        }
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private Map<String, Object> info;
+
+        public Response(Map<String, Object> info) {
+            this.info = info;
+        }
+
+        public Response() {
+            this.info = Collections.emptyMap();
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            info = in.readMap();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeMap(info);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.map(info);
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(info);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Response other = (Response) obj;
+            return Objects.equals(info, other.info);
+        }
+    }
+}

plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java

@@ -55,6 +55,8 @@ import java.util.concurrent.TimeUnit;
  */
 public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements ToXContentObject {
 
+    public static final int DEFAULT_SCROLL_SIZE = 1000;
+
     private static final int SECONDS_IN_MINUTE = 60;
     private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE;
     private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE;
@@ -427,7 +429,6 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
 
     public static class Builder {
 
-        private static final int DEFAULT_SCROLL_SIZE = 1000;
         private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1);
         private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2);
         private static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000;

plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java

@@ -38,10 +38,10 @@ public class AnalysisLimits implements ToXContentObject, Writeable {
      * is now 1GB and defined here in the Java code.  Prior to 6.3, a value of <code>null</code> means that
      * the old default value should be used. From 6.3 onwards, the value will always be explicit.
      */
-    static final long DEFAULT_MODEL_MEMORY_LIMIT_MB = 1024L;
+    public static final long DEFAULT_MODEL_MEMORY_LIMIT_MB = 1024L;
     static final long PRE_6_1_DEFAULT_MODEL_MEMORY_LIMIT_MB = 4096L;
 
-    static final long DEFAULT_CATEGORIZATION_EXAMPLES_LIMIT = 4;
+    public static final long DEFAULT_CATEGORIZATION_EXAMPLES_LIMIT = 4;
 
     /**
      * Serialisation field names

plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java

@@ -94,6 +94,8 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
     public static final TimeValue MIN_BACKGROUND_PERSIST_INTERVAL = TimeValue.timeValueHours(1);
     public static final ByteSizeValue PROCESS_MEMORY_OVERHEAD = new ByteSizeValue(100, ByteSizeUnit.MB);
 
+    public static final long DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS = 1;
+
     static {
         PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
         PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
@@ -688,7 +690,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
         private ModelPlotConfig modelPlotConfig;
         private Long renormalizationWindowDays;
         private TimeValue backgroundPersistInterval;
-        private Long modelSnapshotRetentionDays = 1L;
+        private Long modelSnapshotRetentionDays = DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS;
         private Long resultsRetentionDays;
         private Map<String, Object> customSettings;
         private String modelSnapshotId;

plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlInfoActionResponseTests.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.ml.action.MlInfoAction.Response;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MlInfoActionResponseTests extends AbstractStreamableTestCase<Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        int size = randomInt(10);
+        Map<String, Object> info = new HashMap<>();
+        for (int j = 0; j < size; j++) {
+            info.put(randomAlphaOfLength(20), randomAlphaOfLength(20));
+        }
+        return new Response(info);
+    }
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+}

plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -40,6 +40,7 @@ import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.monitor.os.OsProbe;
 import org.elasticsearch.monitor.os.OsStats;
+import org.elasticsearch.persistent.PersistentTasksExecutor;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.PersistentTaskPlugin;
@@ -74,6 +75,7 @@ import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetFiltersAction;
 import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction;
+import org.elasticsearch.xpack.core.ml.action.MlInfoAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction;
@@ -103,7 +105,6 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
 import org.elasticsearch.xpack.core.ml.notifications.AuditMessage;
 import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
-import org.elasticsearch.persistent.PersistentTasksExecutor;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 import org.elasticsearch.xpack.ml.action.TransportCloseJobAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarAction;
@@ -124,6 +125,7 @@ import org.elasticsearch.xpack.ml.action.TransportGetDatafeedsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetDatafeedsStatsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetFiltersAction;
 import org.elasticsearch.xpack.ml.action.TransportGetInfluencersAction;
+import org.elasticsearch.xpack.ml.action.TransportMlInfoAction;
 import org.elasticsearch.xpack.ml.action.TransportGetJobsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetJobsStatsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetModelSnapshotsAction;
@@ -172,6 +174,7 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory;
 import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerProcessFactory;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction;
+import org.elasticsearch.xpack.ml.rest.RestMlInfoAction;
 import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarAction;
 import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarEventAction;
 import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarJobAction;
@@ -448,6 +451,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
         return Arrays.asList(
             new RestGetJobsAction(settings, restController),
             new RestGetJobStatsAction(settings, restController),
+            new RestMlInfoAction(settings, restController),
             new RestPutJobAction(settings, restController),
             new RestPostJobUpdateAction(settings, restController),
             new RestDeleteJobAction(settings, restController),
@@ -498,6 +502,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
         return Arrays.asList(
                 new ActionHandler<>(GetJobsAction.INSTANCE, TransportGetJobsAction.class),
                 new ActionHandler<>(GetJobsStatsAction.INSTANCE, TransportGetJobsStatsAction.class),
+                new ActionHandler<>(MlInfoAction.INSTANCE, TransportMlInfoAction.class),
                 new ActionHandler<>(PutJobAction.INSTANCE, TransportPutJobAction.class),
                 new ActionHandler<>(UpdateJobAction.INSTANCE, TransportUpdateJobAction.class),
                 new ActionHandler<>(DeleteJobAction.INSTANCE, TransportDeleteJobAction.class),

plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.MachineLearningField;
+import org.elasticsearch.xpack.core.ml.action.MlInfoAction;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TransportMlInfoAction extends HandledTransportAction<MlInfoAction.Request, MlInfoAction.Response> {
+
+    private final ClusterService clusterService;
+
+    @Inject
+    public TransportMlInfoAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                 ClusterService clusterService) {
+        super(settings, MlInfoAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
+                MlInfoAction.Request::new);
+        this.clusterService = clusterService;
+    }
+
+    @Override
+    protected void doExecute(MlInfoAction.Request request, ActionListener<MlInfoAction.Response> listener) {
+        Map<String, Object> info = new HashMap<>();
+        info.put("defaults", defaults());
+        info.put("limits", limits());
+        listener.onResponse(new MlInfoAction.Response(info));
+    }
+
+    private Map<String, Object> defaults() {
+        Map<String, Object> defaults = new HashMap<>();
+        defaults.put("anomaly_detectors", anomalyDetectorsDefaults());
+        defaults.put("datafeeds", datafeedsDefaults());
+        return defaults;
+    }
+
+    private Map<String, Object> anomalyDetectorsDefaults() {
+        Map<String, Object> defaults = new HashMap<>();
+        defaults.put(AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName(),
+                new ByteSizeValue(AnalysisLimits.DEFAULT_MODEL_MEMORY_LIMIT_MB, ByteSizeUnit.MB));
+        defaults.put(AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), AnalysisLimits.DEFAULT_CATEGORIZATION_EXAMPLES_LIMIT);
+        defaults.put(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), Job.DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS);
+        return defaults;
+    }
+
+    private Map<String, Object> datafeedsDefaults() {
+        Map<String, Object> anomalyDetectorsDefaults = new HashMap<>();
+        anomalyDetectorsDefaults.put(DatafeedConfig.SCROLL_SIZE.getPreferredName(), DatafeedConfig.DEFAULT_SCROLL_SIZE);
+        return anomalyDetectorsDefaults;
+    }
+
+    private Map<String, Object> limits() {
+        Map<String, Object> limits = new HashMap<>();
+        ByteSizeValue maxModelMemoryLimit = clusterService.getClusterSettings().get(MachineLearningField.MAX_MODEL_MEMORY_LIMIT);
+        if (maxModelMemoryLimit != null && maxModelMemoryLimit.getBytes() > 0) {
+            limits.put("max_model_memory_limit", maxModelMemoryLimit);
+        }
+        return limits;
+    }
+}

plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestMlInfoAction.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.ml.action.MlInfoAction;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+
+public class RestMlInfoAction extends BaseRestHandler {
+
+    public RestMlInfoAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "info", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_ml_info_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        return channel -> client.execute(MlInfoAction.INSTANCE, new MlInfoAction.Request(), new RestToXContentListener<>(channel));
+    }
+}

plugin/src/test/resources/rest-api-spec/api/xpack.ml.info.json

@@ -0,0 +1,11 @@
+{
+  "xpack.ml.info": {
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/_xpack/ml/info",
+      "paths": [ "/_xpack/ml/info" ],
+      "parts": {},
+      "body": null
+    }
+  }
+}

plugin/src/test/resources/rest-api-spec/test/ml/ml_info.yml

@@ -0,0 +1,23 @@
+---
+"Test ml info":
+  - do:
+      xpack.ml.info: {}
+  - match: { defaults.anomaly_detectors.model_memory_limit: "1gb" }
+  - match: { defaults.anomaly_detectors.categorization_examples_limit: 4 }
+  - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 }
+  - match: { defaults.datafeeds.scroll_size: 1000 }
+  - match: { limits: {} }
+
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            xpack.ml.max_model_memory_limit: "4gb"
+
+  - do:
+      xpack.ml.info: {}
+  - match: { defaults.anomaly_detectors.model_memory_limit: "1gb" }
+  - match: { defaults.anomaly_detectors.categorization_examples_limit: 4 }
+  - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 }
+  - match: { defaults.datafeeds.scroll_size: 1000 }
+  - match: { limits.max_model_memory_limit: "4gb" }

qa/smoke-test-ml-with-security/build.gradle

@@ -74,6 +74,7 @@ integTestRunner {
     'ml/job_groups/Test put job with group that matches its id',
     'ml/job_groups/Test put job with id that matches an existing group',
     'ml/job_groups/Test put job with invalid group',
+    'ml/ml_info/Test ml info',
     'ml/post_data/Test Flush data with invalid parameters',
     'ml/post_data/Test flushing and posting a closed job',
     'ml/post_data/Test open and close with non-existent job id',