[ML] Add description to DF analytics (#45774) (#46019)

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java

@@ -48,6 +48,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
     }
 
     private static final ParseField ID = new ParseField("id");
+    private static final ParseField DESCRIPTION = new ParseField("description");
     private static final ParseField SOURCE = new ParseField("source");
     private static final ParseField DEST = new ParseField("dest");
     private static final ParseField ANALYSIS = new ParseField("analysis");
@@ -60,6 +61,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
 
     static {
         PARSER.declareString(Builder::setId, ID);
+        PARSER.declareString(Builder::setDescription, DESCRIPTION);
         PARSER.declareObject(Builder::setSource, (p, c) -> DataFrameAnalyticsSource.fromXContent(p), SOURCE);
         PARSER.declareObject(Builder::setDest, (p, c) -> DataFrameAnalyticsDest.fromXContent(p), DEST);
         PARSER.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p), ANALYSIS);
@@ -95,6 +97,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
     }
 
     private final String id;
+    private final String description;
     private final DataFrameAnalyticsSource source;
     private final DataFrameAnalyticsDest dest;
     private final DataFrameAnalysis analysis;
@@ -103,10 +106,12 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
     private final Instant createTime;
     private final Version version;
 
-    private DataFrameAnalyticsConfig(@Nullable String id, @Nullable DataFrameAnalyticsSource source, @Nullable DataFrameAnalyticsDest dest,
+    private DataFrameAnalyticsConfig(@Nullable String id, @Nullable String description, @Nullable DataFrameAnalyticsSource source,
-                                     @Nullable DataFrameAnalysis analysis, @Nullable FetchSourceContext analyzedFields,
+                                     @Nullable DataFrameAnalyticsDest dest, @Nullable DataFrameAnalysis analysis,
-                                     @Nullable ByteSizeValue modelMemoryLimit, @Nullable Instant createTime, @Nullable Version version) {
+                                     @Nullable FetchSourceContext analyzedFields, @Nullable ByteSizeValue modelMemoryLimit,
+                                     @Nullable Instant createTime, @Nullable Version version) {
         this.id = id;
+        this.description = description;
         this.source = source;
         this.dest = dest;
         this.analysis = analysis;
@@ -120,6 +125,10 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
         return id;
     }
 
+    public String getDescription() {
+        return description;
+    }
+
     public DataFrameAnalyticsSource getSource() {
         return source;
     }
@@ -154,6 +163,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
         if (id != null) {
             builder.field(ID.getPreferredName(), id);
         }
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
         if (source != null) {
             builder.field(SOURCE.getPreferredName(), source);
         }
@@ -189,6 +201,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
 
         DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o;
         return Objects.equals(id, other.id)
+            && Objects.equals(description, other.description)
             && Objects.equals(source, other.source)
             && Objects.equals(dest, other.dest)
             && Objects.equals(analysis, other.analysis)
@@ -200,7 +213,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
+        return Objects.hash(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
     }
 
     @Override
@@ -211,6 +224,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
     public static class Builder {
 
         private String id;
+        private String description;
         private DataFrameAnalyticsSource source;
         private DataFrameAnalyticsDest dest;
         private DataFrameAnalysis analysis;
@@ -226,6 +240,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
             return this;
         }
 
+        public Builder setDescription(String description) {
+            this.description = description;
+            return this;
+        }
+
         public Builder setSource(DataFrameAnalyticsSource source) {
             this.source = Objects.requireNonNull(source);
             return this;
@@ -262,7 +281,8 @@ public class DataFrameAnalyticsConfig implements ToXContentObject {
         }
 
         public DataFrameAnalyticsConfig build() {
-            return new DataFrameAnalyticsConfig(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
+            return new DataFrameAnalyticsConfig(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime,
+                version);
         }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -1257,6 +1257,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
                 .setIndex("put-test-dest-index")
                 .build())
             .setAnalysis(OutlierDetection.createDefault())
+            .setDescription("some description")
             .build();
 
         createIndex("put-test-source-index", defaultMappingForTest());
@@ -1273,6 +1274,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
         assertThat(createdConfig.getAnalysis(), equalTo(config.getAnalysis()));
         assertThat(createdConfig.getAnalyzedFields(), equalTo(config.getAnalyzedFields()));
         assertThat(createdConfig.getModelMemoryLimit(), equalTo(ByteSizeValue.parseBytesSizeValue("1gb", "")));  // default value
+        assertThat(createdConfig.getDescription(), equalTo("some description"));
     }
 
     public void testGetDataFrameAnalyticsConfig_SingleConfig() throws Exception {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -2950,6 +2950,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
                 .setAnalysis(outlierDetection) // <4>
                 .setAnalyzedFields(analyzedFields) // <5>
                 .setModelMemoryLimit(new ByteSizeValue(5, ByteSizeUnit.MB)) // <6>
+                .setDescription("this is an example description") // <7>
                 .build();
             // end::put-data-frame-analytics-config
 

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java

@@ -49,6 +49,9 @@ public class DataFrameAnalyticsConfigTests extends AbstractXContentTestCase<Data
                 .setSource(randomSourceConfig())
                 .setDest(randomDestConfig())
                 .setAnalysis(randomOutlierDetection());
+        if (randomBoolean()) {
+            builder.setDescription(randomAlphaOfLength(20));
+        }
         if (randomBoolean()) {
             builder.setAnalyzedFields(new FetchSourceContext(true,
                 generateRandomStringArray(10, 10, false, false),

docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc

@@ -36,6 +36,7 @@ include-tagged::{doc-tests-file}[{api}-config]
 <4> The analysis to be performed
 <5> The fields to be included in / excluded from the analysis
 <6> The memory limit for the model created as part of the analysis process
+<7> Optionally, a human-readable description
 
 [id="{upid}-{api}-query-config"]
 

docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc

@@ -42,6 +42,9 @@ PUT _ml/data_frame/analytics/loganalytics
 // CONSOLE
 // TEST[setup:setup_logdata]
 
+`description`::
+  (Optional, string) A description of the job.
+
 `dest`::
   (object) The destination configuration of the analysis. The `index` property 
   (string) is the name of the index in which to store the results of the 

docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc

@@ -65,7 +65,10 @@ and mappings.
   (Optional, object) You can specify both `includes` and/or `excludes` patterns. If 
   `analyzed_fields` is not set, only the relevant fields will be included. For 
   example, all the numeric fields for {oldetection}.
-  
+
+`description`::
+  (Optional, string) A description of the job.
+
 `dest`::
   (Required, object) The destination configuration, consisting of `index` and 
   optionally `results_field` (`ml` by default). See 
@@ -94,6 +97,7 @@ type is `outlier_detection`:
 --------------------------------------------------
 PUT _ml/data_frame/analytics/loganalytics
 {
+  "description": "Outlier detection on log data",
   "source": {
     "index": "logdata"
   },
@@ -115,6 +119,7 @@ The API returns the following result:
 ----
 {
   "id" : "loganalytics",
+  "description": "Outlier detection on log data",
   "source" : {
     "index" : [
       "logdata"

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java

@@ -44,6 +44,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
     public static final ByteSizeValue PROCESS_MEMORY_OVERHEAD = new ByteSizeValue(20, ByteSizeUnit.MB);
 
     public static final ParseField ID = new ParseField("id");
+    public static final ParseField DESCRIPTION = new ParseField("description");
     public static final ParseField SOURCE = new ParseField("source");
     public static final ParseField DEST = new ParseField("dest");
     public static final ParseField ANALYSIS = new ParseField("analysis");
@@ -62,6 +63,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
 
         parser.declareString((c, s) -> {}, CONFIG_TYPE);
         parser.declareString(Builder::setId, ID);
+        parser.declareString(Builder::setDescription, DESCRIPTION);
         parser.declareObject(Builder::setSource, DataFrameAnalyticsSource.createParser(ignoreUnknownFields), SOURCE);
         parser.declareObject(Builder::setDest, DataFrameAnalyticsDest.createParser(ignoreUnknownFields), DEST);
         parser.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p, ignoreUnknownFields), ANALYSIS);
@@ -100,6 +102,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
     }
 
     private final String id;
+    private final String description;
     private final DataFrameAnalyticsSource source;
     private final DataFrameAnalyticsDest dest;
     private final DataFrameAnalysis analysis;
@@ -117,10 +120,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
     private final Instant createTime;
     private final Version version;
 
-    public DataFrameAnalyticsConfig(String id, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest,
+    public DataFrameAnalyticsConfig(String id, String description, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest,
                                     DataFrameAnalysis analysis, Map<String, String> headers, ByteSizeValue modelMemoryLimit,
                                     FetchSourceContext analyzedFields, Instant createTime, Version version) {
         this.id = ExceptionsHelper.requireNonNull(id, ID);
+        this.description = description;
         this.source = ExceptionsHelper.requireNonNull(source, SOURCE);
         this.dest = ExceptionsHelper.requireNonNull(dest, DEST);
         this.analysis = ExceptionsHelper.requireNonNull(analysis, ANALYSIS);
@@ -133,6 +137,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
 
     public DataFrameAnalyticsConfig(StreamInput in) throws IOException {
         this.id = in.readString();
+        if (in.getVersion().onOrAfter(Version.V_7_4_0)) {
+            description = in.readOptionalString();
+        } else {
+            description = null;
+        }
         this.source = new DataFrameAnalyticsSource(in);
         this.dest = new DataFrameAnalyticsDest(in);
         this.analysis = in.readNamedWriteable(DataFrameAnalysis.class);
@@ -152,6 +161,10 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
         return id;
     }
 
+    public String getDescription() {
+        return description;
+    }
+
     public DataFrameAnalyticsSource getSource() {
         return source;
     }
@@ -188,6 +201,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(ID.getPreferredName(), id);
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
         builder.field(SOURCE.getPreferredName(), source);
         builder.field(DEST.getPreferredName(), dest);
 
@@ -218,6 +234,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(id);
+        if (out.getVersion().onOrAfter(Version.V_7_4_0)) {
+            out.writeOptionalString(description);
+        }
         source.writeTo(out);
         dest.writeTo(out);
         out.writeNamedWriteable(analysis);
@@ -242,6 +261,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
 
         DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o;
         return Objects.equals(id, other.id)
+            && Objects.equals(description, other.description)
             && Objects.equals(source, other.source)
             && Objects.equals(dest, other.dest)
             && Objects.equals(analysis, other.analysis)
@@ -254,7 +274,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields, createTime, version);
+        return Objects.hash(id, description, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields, createTime, version);
     }
 
     @Override
@@ -269,6 +289,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
     public static class Builder {
 
         private String id;
+        private String description;
         private DataFrameAnalyticsSource source;
         private DataFrameAnalyticsDest dest;
         private DataFrameAnalysis analysis;
@@ -287,6 +308,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
 
         public Builder(DataFrameAnalyticsConfig config, ByteSizeValue maxModelMemoryLimit) {
             this.id = config.id;
+            this.description = config.description;
             this.source = new DataFrameAnalyticsSource(config.source);
             this.dest = new DataFrameAnalyticsDest(config.dest);
             this.analysis = config.analysis;
@@ -304,6 +326,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
             return id;
         }
 
+        public Builder setDescription(String description) {
+            this.description = description;
+            return this;
+        }
+
         public Builder setId(String id) {
             this.id = ExceptionsHelper.requireNonNull(id, ID);
             return this;
@@ -354,7 +381,8 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
          */
         public DataFrameAnalyticsConfig build() {
             applyMaxModelMemoryLimit();
-            return new DataFrameAnalyticsConfig(id, source, dest, analysis, headers, modelMemoryLimit, analyzedFields, createTime, version);
+            return new DataFrameAnalyticsConfig(id, description, source, dest, analysis, headers, modelMemoryLimit, analyzedFields,
+                createTime, version);
         }
 
         /**
@@ -365,6 +393,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
         public DataFrameAnalyticsConfig buildForMemoryEstimation() {
             return new DataFrameAnalyticsConfig(
                 id != null ? id : "dummy",
+                description,
                 source,
                 dest != null ? dest : new DataFrameAnalyticsDest("dummy", null),
                 analysis,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java

@@ -286,6 +286,7 @@ public final class ReservedFieldNames {
             ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(),
 
             DataFrameAnalyticsConfig.ID.getPreferredName(),
+            DataFrameAnalyticsConfig.DESCRIPTION.getPreferredName(),
             DataFrameAnalyticsConfig.SOURCE.getPreferredName(),
             DataFrameAnalyticsConfig.DEST.getPreferredName(),
             DataFrameAnalyticsConfig.ANALYSIS.getPreferredName(),

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java

@@ -113,6 +113,9 @@ public class DataFrameAnalyticsConfigTests extends AbstractSerializingTestCase<D
         if (randomBoolean()) {
             builder.setModelMemoryLimit(new ByteSizeValue(randomIntBetween(1, 16), randomFrom(ByteSizeUnit.MB, ByteSizeUnit.GB)));
         }
+        if (randomBoolean()) {
+            builder.setDescription(randomAlphaOfLength(20));
+        }
         if (withGeneratedFields) {
             if (randomBoolean()) {
                 builder.setCreateTime(Instant.now());

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml

@@ -1182,3 +1182,36 @@ setup:
   }}
   - is_true: create_time
   - is_true: version
+
+---
+"Test put with description":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "with-description"
+        body: >
+          {
+            "description": "This is a described config",
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {
+              "regression": {
+                "dependent_variable": "foo"
+              }
+            }
+          }
+  - match: { id: "with-description" }
+  - match: { description: "This is a described config" }
+  - match: { source.index: ["index-source"] }
+  - match: { dest.index: "index-dest" }
+  - match: { analysis: {
+    "regression":{
+      "dependent_variable": "foo"
+    }
+  }}
+  - is_true: create_time
+  - is_true: version